diff --git a/.dockerignore b/.dockerignore index 9c4c55bc..ed39380a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,11 @@ docs/ specs/ *.md LICENSE +**/.env +**/.env.* +**/node_modules +**/__pycache__ +**/.venv +**/.pulumi +.DS_Store +**/.DS_Store diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml index bb231ba4..6d99c439 100644 --- a/.github/actions/load-image/action.yml +++ b/.github/actions/load-image/action.yml @@ -5,7 +5,7 @@ runs: using: "composite" steps: - name: Download image artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: rustack-image path: /tmp diff --git a/.github/workflows/promote-docker.yml b/.github/workflows/promote-docker.yml new file mode 100644 index 00000000..9fc7363e --- /dev/null +++ b/.github/workflows/promote-docker.yml @@ -0,0 +1,60 @@ +name: promote-docker + +on: + workflow_dispatch: + inputs: + candidate_run_id: + description: Successful same-commit release-docker candidate run + required: true + type: string + qualification_run_id: + description: Successful same-commit qualify-workflows run + required: true + type: string + version: + description: Numeric major.minor.patch version to promote + required: true + type: string + +permissions: + contents: read + actions: read + +jobs: + promote: + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + permissions: + contents: read + actions: read + packages: write + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_RUN_ID: ${{ inputs.candidate_run_id }} + QUALIFICATION_RUN_ID: ${{ inputs.qualification_run_id }} + RELEASE_VERSION: ${{ inputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + run-id: ${{ inputs.qualification_run_id }} + name: workflows-qualified-release + github-token: ${{ github.token }} + path: ${{ runner.temp }}/qualification + - name: Recheck raw qualification before registry login + run: make workflows-release-preflight WORKFLOWS_RELEASE_BUNDLE="${{ runner.temp }}/qualification" + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Promote the qualified archives without rebuilding + run: make workflows-release-promote WORKFLOWS_RELEASE_BUNDLE="${{ runner.temp }}/qualification" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: workflows-promotion-receipt + path: ${{ runner.temp }}/qualification/promotion.json + if-no-files-found: error diff --git a/.github/workflows/qualify-workflows.yml b/.github/workflows/qualify-workflows.yml new file mode 100644 index 00000000..3b93f8bd --- /dev/null +++ b/.github/workflows/qualify-workflows.yml @@ -0,0 +1,54 @@ +name: qualify-workflows + +on: + workflow_dispatch: + inputs: + candidate_run_id: + description: Successful same-commit release-docker run containing the exact archives + required: true + type: string + evidence_run_id: + description: Successful same-commit run containing the complete workflows-evidence artifact + required: true + type: string + +permissions: + contents: read + actions: read + +jobs: + assemble: + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Fence source runs + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_RUN_ID: ${{ inputs.candidate_run_id }} + EVIDENCE_RUN_ID: ${{ inputs.evidence_run_id }} + run: make workflows-release-input-runs + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + run-id: ${{ inputs.evidence_run_id }} + name: workflows-evidence + github-token: ${{ github.token }} + path: ${{ runner.temp }}/qualification + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + run-id: ${{ inputs.candidate_run_id }} + pattern: release-candidate-* + merge-multiple: true + github-token: ${{ github.token }} + path: ${{ runner.temp }}/qualification + - name: Require complete raw evidence and exact candidates + run: make verify-workflows WORKFLOWS_RELEASE_BUNDLE="${{ runner.temp }}/qualification" WORKFLOWS_RELEASE_REPORT="${{ runner.temp }}/qualification/release-report.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: workflows-qualified-release + path: ${{ runner.temp }}/qualification/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index b36155ae..9393a652 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -2,71 +2,63 @@ name: release-docker on: push: - tags: - - v* + tags: [v*] workflow_dispatch: +# Candidates are local Docker archives. Publication belongs to promote-docker. permissions: contents: read - packages: write - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository_owner }}/rustack jobs: - # Wait for the build workflow to pass before pushing the Docker image. - wait-for-build: - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest + candidate: + strategy: + fail-fast: false + matrix: + include: + - architecture: arm64 + runner: ubuntu-24.04-arm + - architecture: amd64 + runner: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 steps: - - name: Wait for build workflow - uses: lewagon/wait-on-check-action@v1.3.4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.ref }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - check-name: build-rust (ubuntu-latest) - wait-interval: 15 - allowed-conclusions: success - - build-and-push: - needs: [wait-for-build] - if: always() && (needs.wait-for-build.result == 'success' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU (for multi-arch builds) - uses: docker/setup-qemu-action@v3 - - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker metadata - id: docker-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern={{major}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=latest - - - name: Build and push - uses: docker/build-push-action@v6 + persist-credentials: false + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Allocate candidate directory + run: mkdir -m 700 "${{ runner.temp }}/candidate" + - name: Build without publication + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker-meta.outputs.tags }} - labels: ${{ steps.docker-meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: linux/${{ matrix.architecture }} + push: false + provenance: false + tags: rustack/workflows-release:integration-${{ matrix.architecture }} + outputs: type=docker,dest=${{ runner.temp }}/candidate/release-${{ matrix.architecture }}.tar + cache-from: type=gha,scope=release-${{ matrix.architecture }} + cache-to: type=gha,mode=max,scope=release-${{ matrix.architecture }} + - name: Load and seal exact archive + env: + ARCHITECTURE: ${{ matrix.architecture }} + CANDIDATE: ${{ runner.temp }}/candidate + run: | + docker image load --input "$CANDIDATE/release-$ARCHITECTURE.tar" + make workflows-release-seal WORKFLOWS_ARCH="$ARCHITECTURE" WORKFLOWS_RELEASE_BUNDLE="$CANDIDATE" + - name: Record execution scope + env: + ARCHITECTURE: ${{ matrix.architecture }} + run: | + if [ "$ARCHITECTURE" = amd64 ]; then + echo 'D100: distribution build only; Linux x86_64 execution is waived.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'Candidate awaits complete exact-image platform qualification; no promotion occurred.' >> "$GITHUB_STEP_SUMMARY" + fi + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-candidate-${{ matrix.architecture }} + path: ${{ runner.temp }}/candidate/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 8e5334cd..a1bc9c9e 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -31,13 +31,13 @@ jobs: name: "Build Docker image" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build Docker image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . load: true @@ -49,7 +49,7 @@ jobs: run: docker save "$IMAGE" | gzip > /tmp/rustack-image.tar.gz - name: Upload image artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: rustack-image path: /tmp/rustack-image.tar.gz @@ -63,7 +63,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -247,6 +247,59 @@ jobs: aws lambda get-account-settings | jq -e '.AccountLimit' echo "Lambda OK" + - name: "Step Functions: execute pure ASL and verify exact history" + timeout-minutes: 2 + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=3s 60s bash <<'SMOKE' + set -euo pipefail + [[ "$AWS_ENDPOINT_URL" =~ ^http://localhost:[0-9]{1,5}$ ]] + unset AWS_PROFILE AWS_DEFAULT_PROFILE AWS_SESSION_TOKEN AWS_SECURITY_TOKEN AWS_ENDPOINT_URL_STEPFUNCTIONS + export AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null + export AWS_MAX_ATTEMPTS=1 AWS_EC2_METADATA_DISABLED=true AWS_PAGER="" + sfn() { + timeout --signal=TERM --kill-after=2s 8s aws \ + --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" \ + --cli-connect-timeout 2 --cli-read-timeout 3 --no-cli-pager --no-paginate \ + stepfunctions "$@" + } + machine="" + cleanup() { + status=$? + trap - EXIT + if [ -n "$machine" ]; then + sfn delete-state-machine --state-machine-arn "$machine" || status=1 + fi + exit "$status" + } + trap cleanup EXIT + name="action-pure-$(python3 -c 'import uuid; print(uuid.uuid4().hex)')" + machine=$(sfn create-state-machine --name "$name" --type STANDARD \ + --role-arn arn:aws:iam::000000000000:role/ActionPure \ + --definition '{"StartAt":"Double","States":{"Double":{"Type":"Pass","Parameters":{"result.$":"States.MathAdd($.n, $.n)"},"End":true}}}' \ + --query stateMachineArn --output text) + execution=$(sfn start-execution --state-machine-arn "$machine" --input '{"n":21}' \ + --query executionArn --output text) + for attempt in $(seq 1 20); do + result=$(sfn describe-execution --execution-arn "$execution") + state=$(jq -er '.status' <<<"$result") + [ "$state" != RUNNING ] && break + sleep 0.2 + done + jq -e '.status == "SUCCEEDED" and (.output | fromjson) == {result:42}' <<<"$result" + history=$(sfn get-execution-history --execution-arn "$execution" --max-results 10) + jq -e ' + (has("nextToken") | not) and + [.events[].type] == ["ExecutionStarted","PassStateEntered","PassStateExited","ExecutionSucceeded"] and + [.events[].id] == [1,2,3,4] and [.events[].previousEventId] == [0,0,2,3] and + (.events[0].executionStartedEventDetails.input | fromjson) == {n:21} and + .events[1].stateEnteredEventDetails.name == "Double" and + (.events[2].stateExitedEventDetails.output | fromjson) == {result:42} and + (.events[3].executionSucceededEventDetails.output | fromjson) == {result:42} + ' <<<"$history" + echo "Step Functions pure ASL 21 -> 42 and exact four-event history OK" + SMOKE + # ── CloudWatch smoke test ───────────────────────────────────────── - name: "CloudWatch: put metric + list metrics" run: | @@ -462,7 +515,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -514,7 +567,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -557,7 +610,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -596,7 +649,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -671,7 +724,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -712,7 +765,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image @@ -741,7 +794,7 @@ jobs: needs: build-image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Load Docker image uses: ./.github/actions/load-image diff --git a/.github/workflows/workflows-evidence.yml b/.github/workflows/workflows-evidence.yml new file mode 100644 index 00000000..03a760b0 --- /dev/null +++ b/.github/workflows/workflows-evidence.yml @@ -0,0 +1,36 @@ +name: workflows-evidence + +on: + workflow_dispatch: + inputs: + candidate_run_id: + description: Candidate run whose exact archives were qualified on every required lane + required: true + type: string + +permissions: + contents: read + +jobs: + retain: + # Operators stage the complete same-source receipts on this dedicated runner. + # Missing Desktop/reference/rootless receipts fail; this never waives a lane. + runs-on: [self-hosted, rustack-qualification] + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Pack only fully verified evidence + env: + CANDIDATE_RUN_ID: ${{ inputs.candidate_run_id }} + WORKFLOWS_EVIDENCE_ROOT: ${{ vars.WORKFLOWS_EVIDENCE_ROOT }} + WORKFLOWS_EVIDENCE_OUTPUT: ${{ runner.temp }}/workflows-evidence + run: make workflows-release-evidence + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: workflows-evidence + path: ${{ runner.temp }}/workflows-evidence/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 diff --git a/.gitignore b/.gitignore index 80a40a5e..e5f4bcdd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules/ .pulumi/ .pulumi-home/ +.DS_Store diff --git a/.gitmodules b/.gitmodules index bcc7d616..732c2408 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,12 @@ [submodule "vendors/localstack"] path = vendors/localstack url = https://github.com/localstack/localstack.git +[submodule "vendors/bollard"] + path = vendors/bollard + url = https://github.com/fussybeaver/bollard.git +[submodule "vendors/moby"] + path = vendors/moby + url = https://github.com/moby/moby.git +[submodule "vendors/pulumi"] + path = vendors/pulumi + url = https://github.com/pulumi/pulumi.git diff --git a/Cargo.lock b/Cargo.lock index 885ed16c..4f32638c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,45 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -102,9 +141,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" dependencies = [ "aws-credential-types", "aws-runtime", @@ -113,10 +152,10 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", @@ -133,9 +172,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -169,9 +208,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -206,11 +245,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -232,13 +271,13 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.0", "aws-types", "fastrand", "http 0.2.12", @@ -260,11 +299,11 @@ dependencies = [ "aws-smithy-cbor", "aws-smithy-compression", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "fastrand", @@ -289,11 +328,11 @@ dependencies = [ "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -315,11 +354,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -342,11 +381,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -368,11 +407,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -394,14 +433,14 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", - "aws-smithy-query", + "aws-smithy-query 0.62.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.0", "aws-types", "fastrand", "http 0.2.12", @@ -422,11 +461,11 @@ dependencies = [ "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -448,11 +487,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -475,11 +514,11 @@ dependencies = [ "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -504,25 +543,25 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.0", "aws-types", "bytes", "fastrand", "hex", - "hmac 0.13.0", + "hmac", "http 0.2.12", "http 1.5.0", "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2", "tracing", "url", ] @@ -538,11 +577,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -564,14 +603,14 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", - "aws-smithy-query", + "aws-smithy-query 0.62.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.0", "aws-types", "fastrand", "http 0.2.12", @@ -580,6 +619,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-sfn" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c149c59d61748e843ef40e47db88a884148300dfa560c0d84b0fcf04f0c85c2f" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json 0.65.0", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.5.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sns" version = "1.111.0" @@ -591,14 +655,14 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", - "aws-smithy-query", + "aws-smithy-query 0.62.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.0", "aws-types", "fastrand", "http 0.2.12", @@ -618,11 +682,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -644,11 +708,11 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -661,25 +725,24 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.117.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "5316813498fa9c403877007934ae448aa44b18136f58051f8fa3c5b79b44bdc6" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -687,25 +750,24 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "31db35e5662787b786daba15f431e4393c7ea512b70fa112bd048d1f3ecf877a" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -713,26 +775,25 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.122.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", - "aws-smithy-query", + "aws-smithy-query 0.64.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.64.0", "aws-types", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -740,9 +801,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.5.1" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -750,26 +811,21 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hmac 0.13.0", - "http 0.2.12", + "hmac", "http 1.5.0", - "p256", "percent-encoding", - "sha2 0.11.0", - "subtle", + "sha2", "time", "tracing", - "zeroize", ] [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" dependencies = [ "futures-util", "pin-project-lite", @@ -783,7 +839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f069169abdc367533ea89b869b7dfaadd4134a9bc287c13dbe5ff364ff13228" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "minicbor", "num-bigint", @@ -806,7 +862,7 @@ dependencies = [ "md-5", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "tracing", ] @@ -830,9 +886,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.61.2" +version = "0.61.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f" +checksum = "14c8a643d0e114cae39e45415e3225793a6396698b7779a0c9e3ede731d8fc71" dependencies = [ "aws-smithy-types", "bytes", @@ -841,9 +897,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.64.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -857,15 +913,14 @@ dependencies = [ "http-body-util", "percent-encoding", "pin-project-lite", - "pin-utils", "tracing", ] [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -892,7 +947,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", "aws-smithy-types", ] @@ -912,43 +978,53 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", + "aws-smithy-xml 0.62.0", + "urlencoding", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.64.0", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", "http-body 1.1.0", "http-body-util", "pin-project-lite", - "pin-utils", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -984,11 +1060,22 @@ dependencies = [ "http 1.5.0", ] +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" dependencies = [ "base64-simd", "bytes", @@ -1002,7 +1089,6 @@ dependencies = [ "itoa", "num-integer", "pin-project-lite", - "pin-utils", "ryu", "serde", "time", @@ -1017,21 +1103,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "rustc_version", "tracing", @@ -1085,12 +1183,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -1103,6 +1195,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1119,6 +1217,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitcode" version = "0.6.9" @@ -1176,6 +1283,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "bollard-stubs" +version = "1.53.1-rc.29.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +dependencies = [ + "serde", + "serde_json", + "serde_repr", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -1239,7 +1357,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -1320,12 +1438,6 @@ dependencies = [ "yaml-rust2", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -1460,28 +1572,16 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.7.5" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" dependencies = [ "cpubits", "ctutils", "getrandom 0.4.3", "hybrid-array", "num-traits", - "rand_core 0.10.1", + "rand_core", "subtle", "zeroize", ] @@ -1504,14 +1604,14 @@ checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "getrandom 0.4.3", "hybrid-array", - "rand_core 0.10.1", + "rand_core", ] [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" dependencies = [ "cmov", "subtle", @@ -1532,15 +1632,10 @@ dependencies = [ ] [[package]] -name = "der" -version = "0.7.10" +name = "data-encoding" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "der" @@ -1548,10 +1643,24 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid 0.10.2", + "const-oid", "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1565,9 +1674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.6", - "subtle", ] [[package]] @@ -1577,7 +1684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -1599,32 +1706,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.8.2", + "der", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", - "signature 3.0.0", - "spki 0.8.0", + "elliptic-curve", + "rfc6979", + "signature", + "spki", "zeroize", ] @@ -1634,42 +1727,22 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pem-rfc7468", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", - "pkcs8 0.11.0", - "rand_core 0.10.1", - "sec1 0.8.1", + "pkcs8", + "rand_core", + "sec1", "subtle", "zeroize", ] @@ -1713,22 +1786,22 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "rand_core 0.6.4", + "rand_core", "subtle", ] [[package]] -name = "ff" -version = "0.14.0" +name = "filetime" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ - "rand_core 0.10.1", - "subtle", + "cfg-if", + "libc", ] [[package]] @@ -1871,7 +1944,6 @@ checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -1897,7 +1969,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "rand_core 0.10.1", + "rand_core", "wasm-bindgen", ] @@ -1908,15 +1980,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e762b9b188634fc508d4ec54b62ea3d352c43fd431d18d05d46f559f217f4725" [[package]] -name = "group" -version = "0.13.0" +name = "glob" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", -] +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "group" @@ -1924,16 +1991,16 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", - "rand_core 0.10.1", + "ff", + "rand_core", "subtle", ] [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -2000,15 +2067,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0" @@ -2098,9 +2156,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -2137,16 +2195,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http 1.5.0", "http-body 1.1.0", + "httparse", "hyper", "ipnet", "libc", @@ -2387,11 +2446,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f50113171a713f4a4231ef82eb26703607139b35dcb56241f0ceab2ae1f7d8" dependencies = [ "cpubits", - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", - "primeorder 0.14.0", - "sha2 0.11.0", - "signature 3.0.0", + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", + "signature", "wnaf", ] @@ -2495,6 +2554,12 @@ dependencies = [ "half", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -2544,6 +2609,28 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2602,6 +2689,15 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2620,18 +2716,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2661,15 +2745,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2682,30 +2757,14 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", -] - [[package]] name = "pkcs8" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.2", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -2735,30 +2794,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", - "rand_core 0.10.1", + "ff", + "rand_core", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -2774,6 +2824,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand", + "rand_xorshift", + "regex-syntax", + "unarray", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -2805,9 +2871,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -2863,16 +2929,7 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", + "rand_core", ] [[package]] @@ -2887,7 +2944,29 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.10.1", + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "aws-lc-rs", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", ] [[package]] @@ -2934,13 +3013,19 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "http 1.5.0", @@ -2978,7 +3063,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3012,24 +3097,14 @@ dependencies = [ "web-sys", ] -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] @@ -3046,12 +3121,38 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b376e512f30a7686a8dbc6aeaace99b3ade1c26c1c63473473a817ade226dafc" +dependencies = [ + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88512dbfb5b73a0dfa38aef7bc73b5657035fce477b923faf12d332dd83051ea" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustack-apigatewayv2-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3105,15 +3206,15 @@ dependencies = [ name = "rustack-auth" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "digest 0.11.3", "hex", - "hmac 0.13.0", + "hmac", "http 1.5.0", "percent-encoding", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "subtle", "thiserror", "tokio", @@ -3129,16 +3230,23 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-dynamodb", + "aws-sdk-lambda", "aws-sdk-s3", + "aws-sdk-sfn", "aws-sdk-sqs", "aws-smithy-runtime-api", + "base64 0.22.1", "bytes", "ciborium", "crc32fast", + "getrandom 0.4.3", "http 1.5.0", "http-body-util", "hyper", "hyper-util", + "nix", + "proptest", + "rstest", "rustack-apigatewayv2-core", "rustack-apigatewayv2-http", "rustack-auth", @@ -3163,6 +3271,7 @@ dependencies = [ "rustack-kms-http", "rustack-lambda-core", "rustack-lambda-http", + "rustack-lambda-model", "rustack-logs-core", "rustack-logs-http", "rustack-s3-core", @@ -3179,15 +3288,24 @@ dependencies = [ "rustack-sqs-model", "rustack-ssm-core", "rustack-ssm-http", + "rustack-stepfunctions-core", + "rustack-stepfunctions-http", + "rustack-stepfunctions-model", "rustack-sts-core", "rustack-sts-http", "serde", "serde_json", + "sha2", + "tar", "tempfile", + "thiserror", "tokio", "tokio-test", "tracing", "tracing-subscriber", + "uuid", + "yaml-rust2", + "zip", "zstd", ] @@ -3237,7 +3355,7 @@ dependencies = [ name = "rustack-cloudfront-http" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -3336,6 +3454,8 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", + "proptest", + "rstest", "serde", "serde_json", "thiserror", @@ -3395,7 +3515,7 @@ dependencies = [ name = "rustack-dynamodb-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3595,7 +3715,7 @@ dependencies = [ "aws-sdk-ssm", "aws-sdk-sts", "aws-smithy-runtime-api", - "base64", + "base64 0.22.1", "bytes", "futures", "rand", @@ -3619,7 +3739,7 @@ name = "rustack-kinesis-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3665,7 +3785,7 @@ dependencies = [ name = "rustack-kinesis-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3679,7 +3799,7 @@ version = "0.11.0" dependencies = [ "anyhow", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3723,7 +3843,7 @@ dependencies = [ name = "rustack-kms-model" version = "0.10.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3737,28 +3857,42 @@ version = "0.11.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", + "bollard-stubs", "bytes", "chrono", "dashmap", + "form_urlencoded", + "hex", "http 1.5.0", "http-body-util", "hyper", "hyper-util", + "nix", "parking_lot", + "percent-encoding", + "proptest", + "rstest", "rustack-auth", "rustack-core", "rustack-lambda-http", "rustack-lambda-model", + "rustack-lambda-runtime", + "secrecy", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "squib", + "tar", "tempfile", "thiserror", "tokio", "tracing", + "tracing-subscriber", + "typed-builder", "uuid", + "winnow", + "yaml-rust2", "zip", ] @@ -3766,13 +3900,21 @@ dependencies = [ name = "rustack-lambda-echo-bootstrap" version = "0.9.0" dependencies = [ - "base64", + "anyhow", + "aws-credential-types", + "aws-sdk-sfn", + "base64 0.22.1", "rustack-lambda-core", "rustack-lambda-model", + "secrecy", + "serde", "serde_json", "tempfile", "tokio", + "typed-builder", "ureq", + "url", + "uuid", "zip", ] @@ -3807,6 +3949,49 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-lambda-runtime" +version = "0.9.0" +dependencies = [ + "anyhow", + "bytes", + "ciborium", + "getrandom 0.4.3", + "http 1.5.0", + "http-body-util", + "hyper", + "hyper-util", + "proptest", + "rcgen", + "rstest", + "rustls", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-rustls", + "tracing", + "tracing-core", + "tracing-subscriber", + "typed-builder", + "url", + "uuid", + "webpki-roots", +] + +[[package]] +name = "rustack-lambda-runtime-verifier" +version = "0.9.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "rustack-lambda-core", + "rustack-lambda-model", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "rustack-logs-core" version = "0.11.0" @@ -3865,7 +4050,7 @@ name = "rustack-s3-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc32c", @@ -3887,7 +4072,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "tempfile", "thiserror", "tokio", @@ -3918,7 +4103,7 @@ dependencies = [ "rustack-s3-core", "rustack-s3-model", "rustack-s3-xml", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tokio-test", @@ -3952,7 +4137,7 @@ dependencies = [ name = "rustack-secretsmanager-core" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3994,7 +4179,7 @@ dependencies = [ name = "rustack-secretsmanager-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -4007,7 +4192,7 @@ name = "rustack-ses-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4064,7 +4249,7 @@ version = "0.11.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4077,7 +4262,7 @@ dependencies = [ "rustack-sns-model", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tracing", @@ -4119,7 +4304,7 @@ name = "rustack-sqs-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4133,7 +4318,7 @@ dependencies = [ "rustack-sqs-model", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tracing", @@ -4166,7 +4351,7 @@ dependencies = [ name = "rustack-sqs-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "http 1.5.0", "serde", @@ -4178,7 +4363,7 @@ name = "rustack-ssm-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4226,11 +4411,75 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-stepfunctions-core" +version = "0.9.0" +dependencies = [ + "bytes", + "chrono", + "ciborium", + "getrandom 0.4.3", + "hex", + "proptest", + "regex", + "rstest", + "rustack-core", + "rustack-stepfunctions-model", + "secrecy", + "serde", + "serde_json", + "sha2", + "subtle", + "thiserror", + "tokio", + "tracing", + "uuid", + "winnow", +] + +[[package]] +name = "rustack-stepfunctions-http" +version = "0.9.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "hyper", + "hyper-util", + "rstest", + "rustack-auth", + "rustack-core", + "rustack-stepfunctions-model", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "rustack-stepfunctions-model" +version = "0.9.0" +dependencies = [ + "chrono", + "http 1.5.0", + "rstest", + "serde", + "serde_json", + "thiserror", + "typed-builder", +] + [[package]] name = "rustack-sts-core" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4291,6 +4540,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4306,9 +4564,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -4370,9 +4628,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4418,29 +4676,24 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sec1" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", + "base16ct", + "ctutils", + "der", + "hybrid-array", "subtle", "zeroize", ] [[package]] -name = "sec1" -version = "0.8.1" +name = "secrecy" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" dependencies = [ - "base16ct 1.0.0", - "ctutils", - "der 0.8.2", - "hybrid-array", - "subtle", "zeroize", ] @@ -4531,6 +4784,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4549,7 +4813,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -4581,17 +4845,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -4628,16 +4881,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "3.0.0" @@ -4645,7 +4888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ "digest 0.11.3", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -4701,16 +4944,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -4718,7 +4951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.2", + "der", ] [[package]] @@ -4857,8 +5090,6 @@ dependencies = [ [[package]] name = "squib-loader" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd137eb148a011bc2debae200a3c428d5e6c0934c242c64e1c89c374af1a69c" dependencies = [ "flate2", "squib-arch", @@ -4873,7 +5104,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "312eb6febdc4336a85e5be8bd92e207e729651ce0ee7380225fcb9d8960d5069" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "parking_lot", "serde_json", @@ -5038,6 +5269,16 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", +] + [[package]] name = "target-features" version = "0.1.6" @@ -5143,9 +5384,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -5170,9 +5411,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -5202,9 +5443,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -5363,9 +5604,9 @@ dependencies = [ [[package]] name = "typed-path" -version = "0.12.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" [[package]] name = "typenum" @@ -5373,6 +5614,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -5397,7 +5644,7 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64", + "base64 0.22.1", "log", "once_cell", "url", @@ -5475,9 +5722,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -5565,18 +5812,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -5757,8 +6004,8 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", "primefield", ] @@ -5769,6 +6016,24 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror", + "time", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -5786,6 +6051,16 @@ dependencies = [ "hashlink", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -5891,15 +6166,18 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "518b847fd864440c379748ad684c449f293af68978a485a4ff16f7c901c1f7c4" dependencies = [ "crc32fast", "flate2", + "getrandom 0.4.3", "indexmap", "memchr", + "nix", "typed-path", + "windows-sys 0.61.2", "zopfli", ] @@ -5929,27 +6207,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index f0c41f55..30e68679 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,8 +4,9 @@ members = [ "apps/*", "tests/integration", "tests/lambda-fixtures/echo-bootstrap", + "tests/lambda-fixtures/runtime-verifier", ] -exclude = ["codegen", "tools/lambda-squib-agent"] +exclude = ["codegen", "tools/lambda-squib-agent", "patches/squib-loader"] resolver = "3" [workspace.package] @@ -16,6 +17,7 @@ authors = ["Tyr Chen "] repository = "https://github.com/tyrchen/rustack" [workspace.lints.rust] +rust_2024_compatibility = "warn" missing_docs = "warn" missing_debug_implementations = "warn" @@ -50,6 +52,10 @@ rustack-sns-core = { version = "0.11.0", path = "crates/rustack-sns-core" } rustack-lambda-model = { version = "0.9.0", path = "crates/rustack-lambda-model" } rustack-lambda-http = { version = "0.11.0", path = "crates/rustack-lambda-http" } rustack-lambda-core = { version = "0.11.0", path = "crates/rustack-lambda-core" } +rustack-lambda-runtime = { version = "0.9.0", path = "crates/rustack-lambda-runtime" } +rustack-stepfunctions-model = { version = "0.9.0", path = "crates/rustack-stepfunctions-model" } +rustack-stepfunctions-http = { version = "0.9.0", path = "crates/rustack-stepfunctions-http" } +rustack-stepfunctions-core = { version = "0.9.0", path = "crates/rustack-stepfunctions-core" } rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } rustack-logs-http = { version = "0.11.0", path = "crates/rustack-logs-http" } @@ -90,7 +96,7 @@ rustack-cloudfront-dataplane = { version = "0.11.0", path = "crates/rustack-clou squib = "0.2.0" # Async runtime -tokio = { version = "1.53", features = [ +tokio = { version = "1.54", features = [ "rt-multi-thread", "macros", "signal", @@ -103,7 +109,7 @@ tokio = { version = "1.53", features = [ ] } # HTTP / networking -hyper = { version = "1.11", features = ["server", "http1", "http2"] } +hyper = { version = "1.12", features = ["server", "client", "http1", "http2"] } hyper-util = { version = "0.1", features = [ "tokio", "server-auto", @@ -115,18 +121,29 @@ http-body-util = "0.1" # Validated runtime configuration (YAML only). config = { version = "0.15.25", default-features = false, features = ["yaml"] } +# Reuse config's audited event-parser cohort for shallow private credentials. +yaml-rust2 = { version = "~0.11.1", default-features = false } +# Owned endpoint probe TLS; explicitly select aws-lc despite feature unification. +rustls = { version = "~0.23.46", default-features = false, features = ["std", "aws_lc_rs", "tls12"] } +tokio-rustls = { version = "~0.26.6", default-features = false, features = ["aws_lc_rs", "tls12"] } +webpki-roots = "~1.0.11" +rcgen = { version = "~0.14.10", default-features = false, features = ["crypto", "aws_lc_rs"] } url = "2.5.8" # Serialization +bollard-stubs = { version = "=1.53.1-rc.29.3.1", default-features = false } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["float_roundtrip"] } +getrandom = { version = "0.4.3", features = ["std"] } ciborium = "0.2" quick-xml = "0.41" -zstd = { version = "0.13.3", default-features = false } +zstd = { version = "~0.14.3", default-features = false } # Error handling anyhow = "1.0" thiserror = "2.0" +# Safe relative-descriptor ownership IO; also resolved by ZIP9. +nix = { version = "0.31.3", default-features = false, features = ["fs", "user", "dir"] } # Concurrent data structures dashmap = "6.2" @@ -144,6 +161,7 @@ hex = "0.4" hmac = "0.13" digest = "0.11" subtle = "2.6" +secrecy = "~0.10.3" # Time / IDs chrono = { version = "0.4", features = ["serde"] } @@ -152,6 +170,7 @@ uuid = { version = "1.26", features = ["v4"] } # Logging / tracing tracing = "0.1" +tracing-core = "0.1.36" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } # Async utilities @@ -164,17 +183,21 @@ regex = "1.13" bytes = "1.12" rand = "0.10" tempfile = "3.27" -zip = { version = "8", default-features = false, features = ["deflate"] } +zip = { version = "9.1.1", default-features = false, features = ["deflate"] } +tar = { version = "0.4.46", default-features = false } percent-encoding = "2.3" mime = "0.3" futures = "0.3" pin-project-lite = "0.2" typed-builder = "0.23" +winnow = { version = "1.0", default-features = false, features = ["std", "parser"] } form_urlencoded = "1.2" # Testing +rstest = { version = "0.28", default-features = false } +proptest = { version = "1.12", default-features = false, features = ["std"] } tokio-test = "0.4" -aws-config = { version = "1.12", default-features = false, features = [ +aws-config = { version = "1.13", default-features = false, features = [ "credentials-process", "default-https-client", "rt-tokio", @@ -184,7 +207,6 @@ aws-sdk-s3 = { version = "1.145", default-features = false, features = [ "default-https-client", "http-1x", "rt-tokio", - "sigv4a", ] } aws-sdk-dynamodb = { version = "1.124", default-features = false, features = [ "default-https-client", @@ -194,6 +216,10 @@ aws-sdk-dynamodbstreams = { version = "1", default-features = false, features = "default-https-client", "rt-tokio", ] } +aws-sdk-sfn = { version = "1.123", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } aws-sdk-sqs = { version = "1.109", default-features = false, features = [ "default-https-client", "rt-tokio", @@ -221,7 +247,6 @@ aws-sdk-cloudwatchlogs = { version = "1.150", default-features = false, features aws-sdk-eventbridge = { version = "1.116", default-features = false, features = [ "default-https-client", "rt-tokio", - "sigv4a", ] } aws-sdk-kms = { version = "1", default-features = false, features = [ "default-https-client", @@ -250,7 +275,6 @@ aws-sdk-iam = { version = "1", default-features = false, features = [ aws-sdk-sts = { version = "1", default-features = false, features = [ "default-https-client", "rt-tokio", - "sigv4a", ] } aws-sdk-cloudfront = { version = "1", default-features = false, features = [ "default-https-client", @@ -259,3 +283,7 @@ aws-sdk-cloudfront = { version = "1", default-features = false, features = [ reqwest = { version = "0.13", features = ["json"] } aws-credential-types = "1.3" aws-smithy-runtime-api = "1.16" + +[patch.crates-io] +# D69: source-identical loader with the patched compression dependency. +squib-loader = { path = "patches/squib-loader" } diff --git a/Dockerfile b/Dockerfile index 6f43b209..d34ef4ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,43 @@ # syntax=docker/dockerfile:1 # Match the repository toolchain pin and add musl targets to that toolchain. -FROM rust:1.99.0-slim AS builder +FROM rust:1.99.1-slim AS builder RUN apt-get update && apt-get install -y --no-install-recommends musl-tools ca-certificates \ && rm -rf /var/lib/apt/lists/* WORKDIR /src COPY rust-toolchain.toml ./ RUN rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl COPY Cargo.toml Cargo.lock ./ +COPY patches/ patches/ COPY crates/ crates/ COPY apps/ apps/ COPY tests/ tests/ ARG TARGETARCH +ARG CARGO_BUILD_JOBS=2 RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/src/target \ + case "$CARGO_BUILD_JOBS" in 1|2|3|4|5|6|7|8) ;; \ + *) echo "CARGO_BUILD_JOBS must be between 1 and 8" >&2; exit 1 ;; esac && \ case "$TARGETARCH" in \ arm64) RUST_TARGET=aarch64-unknown-linux-musl ;; \ amd64) RUST_TARGET=x86_64-unknown-linux-musl ;; \ *) echo "unsupported target architecture" >&2; exit 1 ;; \ esac && \ - cargo build --locked --release --target "$RUST_TARGET" -p rustack-cli --bin rustack && \ + rustc --version && cargo --version && \ + cargo build --locked --release --target "$RUST_TARGET" -p rustack-cli --bin rustack --features stepfunctions && \ cp "/src/target/$RUST_TARGET/release/rustack" /rustack -RUN mkdir -p /runtime/tmp /runtime/data && chmod 1777 /runtime/tmp +RUN mkdir -p /runtime/tmp /runtime/data /runtime/run/rustack \ + && chmod 1777 /runtime/tmp && chmod 0700 /runtime/data \ + && chown 65532:65532 /runtime/tmp /runtime/data \ + && touch /runtime/run/rustack/docker.sock \ + && chmod 0555 /runtime/run /runtime/run/rustack \ + && chmod 0444 /runtime/run/rustack/docker.sock FROM scratch COPY --from=builder /rustack /rustack COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder --chown=65532:65532 /runtime/tmp /tmp -COPY --from=builder --chown=65532:65532 /runtime/data /data +COPY --from=builder /runtime/ / WORKDIR /data USER 65532:65532 # Containers opt in to all-interface binding; bare binaries default to loopback. diff --git a/Makefile b/Makefile index 4dc8ed10..5726f37d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,63 @@ build: @cargo build +# P0 evidence tooling is isolated from the production workspace/runtime. +WORKFLOWS_SPIKE_DIR ?= /tmp/rustack-spikes +WORKFLOWS_DOCKER_SOCKET ?= /var/run/docker.sock +WORKFLOWS_OTHER_DOCKER_SOCKET ?= +WORKFLOWS_ARCH ?= $(if $(filter arm64 aarch64,$(shell uname -m)),arm64,amd64) +WORKFLOWS_IMAGE_BUILDER ?= docker buildx +LAMBDA_RUNTIME_DIR ?= /tmp/rustack-lambda-runtime +LAMBDA_RUNTIME_IMAGE ?= rustack/lambda-runtime:integration-$(WORKFLOWS_ARCH) + +lambda-runtime-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build -p rustack-lambda-runtime --bin rustack-lambda-runtime --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_RUNTIME_DIR)/$(WORKFLOWS_ARCH)" + +lambda-runtime-image: lambda-runtime-build + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file assets/lambda/runtime/Dockerfile --tag "$(LAMBDA_RUNTIME_IMAGE)" "$(LAMBDA_RUNTIME_DIR)/$(WORKFLOWS_ARCH)/rustack-lambda-runtime" + +lambda-runtime-fixture-image: + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Handler.Dockerfile --tag "rustack-p0-handler:$(WORKFLOWS_ARCH)" tools/workflows-spikes + +test-lambda-runtime-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-runtime --test docker -- --ignored --test-threads=1 + +test-lambda-docker-wire: lambda-runtime-image lambda-runtime-fixture-image lambda-custom-ric-images lambda-zip-code-build + @RUSTACK_ZIP_CODE_BOOTSTRAP="$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/zip-code-bootstrap/bootstrap" RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_wire -- --ignored --test-threads=1 + +.PHONY: lambda-runtime-build lambda-runtime-image lambda-runtime-fixture-image test-lambda-runtime-docker test-lambda-docker-wire + +workflows-model-check: + @python3 -c 'import hashlib,json,pathlib; p=pathlib.Path("codegen/smithy-model"); m=json.loads((p/"stepfunctions.provenance.json").read_text()); raw=(p/"stepfunctions.json").read_bytes(); assert len(raw)==m["bytes"] and hashlib.sha256(raw).hexdigest()==m["sha256"], "SFN source hash mismatch"; s=json.loads(raw)["shapes"][m["service"]]; assert s["version"]=="2016-11-23" and "aws.protocols#awsJson1_0" in s["traits"] and s["traits"]["aws.auth#sigv4"]["name"]=="states"; print("Pinned SFN model verified:",len(s["operations"]),"operations")' + +workflows-fixture-check: workflows-model-check + @python3 -c 'import json,pathlib; p=pathlib.Path("tests/workflows"); f=json.loads((p/"semantic-cases.json").read_text()); m=json.loads((p/"provenance.json").read_text()); source=json.loads(pathlib.Path("codegen/smithy-model/stepfunctions.provenance.json").read_text()); ids=[c["id"] for c in f["cases"]]; assert f["schemaVersion"]==1 and len(ids)==len(set(ids)) and all(c["source"] in f["sources"] for c in f["cases"]); assert m["awsModelCommit"]==source["commit"] and m["awsModelSha256"]==source["sha256"]; print("Fixture catalog structure/provenance verified:",len(ids),"cases; interpreter execution is not claimed")' + +workflows-wire-probe: + @PYTHONDONTWRITEBYTECODE=1 uv run --no-project --python 3.13 --with-requirements tests/workflows/requirements.txt python -m unittest discover -s tests/workflows -p 'test_botocore_wire.py' -v + +workflows-codegen-spike: workflows-model-check + @cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output "$(WORKFLOWS_SPIKE_DIR)/stepfunctions-generated/src" + +test-workflows-spikes: + @cargo test --manifest-path tools/workflows-spikes/Cargo.toml + @cargo clippy --manifest-path tools/workflows-spikes/Cargo.toml --all-targets -- -D warnings + +workflows-relay-spike-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --manifest-path tools/workflows-spikes/Cargo.toml --bin relay --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(WORKFLOWS_SPIKE_DIR)/$(WORKFLOWS_ARCH)" + +workflows-relay-spike-images: workflows-relay-spike-build + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Relay.Dockerfile --tag "rustack-p0-relay:$(WORKFLOWS_ARCH)" "$(WORKFLOWS_SPIKE_DIR)/$(WORKFLOWS_ARCH)/relay" + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Handler.Dockerfile --tag "rustack-p0-handler:$(WORKFLOWS_ARCH)" tools/workflows-spikes + +workflows-relay-spike: workflows-relay-spike-images + @cargo run --manifest-path tools/workflows-spikes/Cargo.toml --bin driver -- "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ARCH)" + +.PHONY: workflows-model-check workflows-fixture-check workflows-wire-probe workflows-codegen-spike \ + test-workflows-spikes workflows-relay-spike-build workflows-relay-spike-images workflows-relay-spike + check: @cargo check --all-targets --all-features @@ -10,6 +67,12 @@ test: fmt: @cargo +nightly fmt +fmt-check: + @cargo fmt --all -- --check + @cargo +nightly fmt --all -- --check + +.PHONY: fmt-check + clippy: @cargo clippy --all-targets --all-features -- -D warnings @@ -69,6 +132,285 @@ codegen-lambda: @cd codegen && cargo run -- --config services/lambda.toml --model smithy-model/lambda.json --output ../crates/rustack-lambda-model/src @cargo +nightly fmt -p rustack-lambda-model +codegen-stepfunctions: + @cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output crates/rustack-stepfunctions-model/src + @cargo +nightly fmt -p rustack-stepfunctions-model + +check-stepfunctions-codegen: + @set -eu; generated=$$(mktemp -d /tmp/rustack-sfn-codegen.XXXXXX); \ + trap 'rm -rf "$$generated"' EXIT; \ + cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output "$$generated"; \ + rustup run nightly rustfmt --edition 2024 --config-path rustfmt.toml "$$generated"/*.rs; \ + diff -r "$$generated" crates/rustack-stepfunctions-model/src + +test-stepfunctions-model: check-stepfunctions-codegen + @cargo test --manifest-path codegen/Cargo.toml + @cargo clippy --manifest-path codegen/Cargo.toml --all-targets -- -D warnings + @cargo test -p rustack-stepfunctions-model + @cargo clippy -p rustack-stepfunctions-model --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: codegen-stepfunctions check-stepfunctions-codegen test-stepfunctions-model + +test-stepfunctions-http: + @cargo test -p rustack-stepfunctions-http + @cargo clippy -p rustack-stepfunctions-http --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +test-stepfunctions-http-sdk: test-workflows-manifest + @cargo test -p rustack-stepfunctions-http --test network test_should_exchange_signed_boto3_requests_with_typed_transport -- --ignored --test-threads=1 + +.PHONY: test-stepfunctions-http test-stepfunctions-http-sdk + +test-stepfunctions-core: + @cargo test -p rustack-stepfunctions-core + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order dataflow::callback_tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order payload::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order expression::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::callback + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order compiler::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::fanout::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::scheduler::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::runtime::tests::structured + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::runtime::tests::callback + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order engine::runtime::structured::callback::tests + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order registry::callback + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order registry::tests::nested + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order registry::tests::children + @cargo test -p rustack-stepfunctions-core --features serde_json/preserve_order registry::actor::clock_tests::children + @cargo clippy -p rustack-stepfunctions-core --all-targets -- -D warnings + @cargo clippy -p rustack-stepfunctions-core --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-stepfunctions-core + +test-stepfunctions-bridge: + @cargo test -p rustack-cli --lib --features stepfunctions + @cargo test -p rustack-cli --doc --features stepfunctions + @cargo clippy -p rustack-cli --all-targets --features stepfunctions -- -D warnings + @cargo clippy -p rustack-cli --lib --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +test-stepfunctions-bridge-native: test-workflows-manifest + @cargo build -p rustack-lambda-echo-bootstrap + @RUSTACK_WORKFLOWS_BOOTSTRAP="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/debug/bootstrap" cargo test -p rustack-cli --lib --features stepfunctions native_e2e -- --ignored --test-threads=1 + +# The shared binary-support gate covers real API, pagination and Lambda bridge. +test-stepfunctions-api: test-stepfunctions-bridge + +.PHONY: test-stepfunctions-bridge test-stepfunctions-bridge-native test-stepfunctions-api + +# Actual main composition and both minimal/runtime feature shapes. +test-stepfunctions-runtime: test-stepfunctions-api + @cargo test -p rustack-cli --bin rustack --features stepfunctions + @cargo test -p rustack-cli --bin rustack --no-default-features --features stepfunctions + @cargo clippy -p rustack-cli --all-targets --no-default-features --features stepfunctions -- -D warnings + + @cargo clippy -p rustack-cli --bin rustack --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + @cargo clippy -p rustack-cli --bin rustack --no-default-features --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-stepfunctions-runtime + +# Dedicated build artifact: uv's ephemeral run environment disappears on exit. +workflows-sdk-python: + @uv venv --python 3.13 --allow-existing "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python" + @uv pip install --python "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" --requirement tests/workflows/requirements.txt + +# Actual provided-runtime handler and strict fixture boundary checks. +test-stepfunctions-handler-fixture: + @cargo build -p rustack-lambda-echo-bootstrap --features sdk-callback --bin callback-bootstrap + @cargo clippy -p rustack-lambda-echo-bootstrap --features sdk-callback --bin callback-bootstrap --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + @python3 -c 'import ast,pathlib; ast.parse(pathlib.Path("tests/lambda-fixtures/callback_bootstrap.py").read_text())' + +# Both ordinary SDKs directly own their real client and server processes. +test-stepfunctions-sdk: test-workflows-manifest workflows-sdk-python test-stepfunctions-handler-fixture + @cargo build -p rustack-lambda-echo-bootstrap + @RUSTACK_WORKFLOWS_PYTHON="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" RUSTACK_CALLBACK_BOOTSTRAP="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/debug/callback-bootstrap" RUSTACK_WORKFLOWS_BOOTSTRAP="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/debug/bootstrap" cargo test -p rustack-cli --test workflows_sdk --features stepfunctions -- --ignored --test-threads=1 --nocapture + +.PHONY: workflows-sdk-python test-stepfunctions-handler-fixture test-stepfunctions-sdk + +audit-codegen: + @cargo audit --file codegen/Cargo.lock + +# D69: source-preserving loader overlay and the complete patched codec graph. +verify-workflows-compression: + @python3 -I -c 'import hashlib,json,pathlib; p=pathlib.Path("patches/squib-loader"); v=json.loads((p/"provenance.json").read_text()); names={"src/lib.rs","Cargo.toml.registry","Cargo.toml.orig",".cargo_vcs_info.json","LICENSE.md"}; assert v["schemaVersion"]==1 and v["package"]=="squib-loader" and v["version"]=="0.2.0"; assert set(v["sha256"])==names; assert all(hashlib.sha256((p/n).read_bytes()).hexdigest()==h for n,h in v["sha256"].items()); original=(p/"Cargo.toml.registry").read_text(); before="[dependencies.zstd]\nversion = \"0.13\""; after="[dependencies.zstd]\nversion = \"0.14.3\""; assert original.count(before)==1 and (p/"Cargo.toml").read_text()==original.replace(before,after); f=pathlib.Path("apps/rustack/src/snapshot/fixtures"); a=json.loads((f/"legacy-v2-meta-provenance.json").read_text()); assert hashlib.sha256((f/a["fixture"]).read_bytes()).hexdigest()==a["sha256"]' + @cargo metadata --locked --format-version 1 | python3 -I -c 'import json,pathlib,sys; data=json.load(sys.stdin); minimum={"zstd":(0,14,3),"zstd-safe":(8,2,0),"zstd-sys":(2,1,2)}; codecs=[p for p in data["packages"] if p["name"] in minimum]; assert len(codecs)==3 and {p["name"] for p in codecs}==set(minimum); assert all(tuple(map(int,p["version"].split("+")[0].split(".")))>=minimum[p["name"]] for p in codecs); loader=[p for p in data["packages"] if p["name"]=="squib-loader"]; assert len(loader)==1 and loader[0]["source"] is None and pathlib.Path(loader[0]["manifest_path"]).resolve()==pathlib.Path("patches/squib-loader/Cargo.toml").resolve()' + +test-workflows-compression: verify-workflows-compression + @cargo test --locked -p squib-loader --lib + @cargo test --locked -p rustack-cli --bin rustack --features stepfunctions snapshot:: + +.PHONY: verify-workflows-compression test-workflows-compression + +# D70: installed archive parser/encoder boundaries, including minimal binary builds. +test-workflows-archive: + @cargo test --locked -p rustack-cli --bin rustack --features stepfunctions snapshot:: + @cargo test --locked -p rustack-cli --bin rustack --no-default-features --features stepfunctions snapshot:: + @cargo clippy --locked -p rustack-cli --bin rustack --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + @cargo clippy --locked -p rustack-cli --bin rustack --no-default-features --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-workflows-archive + +# D71: sealed ownership/export, exact history codec and selected runtime consumer. +test-workflows-snapshot-export: + @cargo test --locked -p rustack-stepfunctions-core snapshot + @cargo test --locked -p rustack-stepfunctions-core --features serde_json/preserve_order snapshot + @cargo clippy --locked -p rustack-stepfunctions-core --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-workflows-snapshot-export + +# D72: startup-only atomic restore, original quotas, complete history and no replay. +test-workflows-snapshot-restore: + @cargo test --locked -p rustack-stepfunctions-core registry::snapshot::restore + @cargo test --locked -p rustack-stepfunctions-core --features serde_json/preserve_order registry::snapshot::restore + @cargo clippy --locked -p rustack-stepfunctions-core --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-workflows-snapshot-restore + +# D73: actual service archives and signed Native binary restart without replay. +test-workflows-persistence: + @cargo test --locked -p rustack-stepfunctions-core registry::snapshot::archive + @cargo test --locked -p rustack-cli --bin rustack --features stepfunctions snapshot:: + @cargo test --locked -p rustack-cli --bin rustack --no-default-features --features stepfunctions snapshot:: + @cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions persistence:: + @cargo build --locked -p rustack-lambda-echo-bootstrap --bin bootstrap + @RUSTACK_WORKFLOWS_BOOTSTRAP="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/debug/bootstrap" cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions persistence:: -- --ignored --test-threads=1 --nocapture + +.PHONY: test-workflows-persistence + +# D74: actual pure-workflow latency, bounded loops and one-slot Stop reuse. +test-workflows-performance: + @cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions performance:: + @cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions performance:: -- --ignored --test-threads=1 --nocapture + @cargo clippy --locked -p rustack-cli --test workflows_sdk --features stepfunctions --no-deps -- -D warnings -W clippy::pedantic + +.PHONY: test-workflows-performance + +# D101: direct native arm64 Linux measurements, separate from dedicated-reference acceptance. +WORKFLOWS_LINUX_PROFILE ?= rustack-workflows-performance-arm64 +WORKFLOWS_LINUX_EVIDENCE_DIR ?= $(shell cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-linux-performance + +test-workflows-performance-linux-controls: + @python3 -I -B -m unittest discover -s tests/workflows -p test_linux_performance.py -v + +workflows-performance-linux-provision: test-workflows-performance-linux-controls + @python3 -I -B tests/workflows/linux_performance.py provision "$(WORKFLOWS_LINUX_PROFILE)" "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')" + +workflows-performance-linux-build: test-workflows-server-linker test-workflows-performance-linux-controls + @mkdir -p "$(WORKFLOWS_LINUX_EVIDENCE_DIR)" + @CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$(CURDIR)/tests/workflows/server_linker.py" WORKFLOWS_SERVER_LINKER_TOOL="$$(command -v cargo-zigbuild)" WORKFLOWS_SERVER_LINKER_TARGET=aarch64-linux-gnu CARGO_ZIGBUILD_ZIG_VERSION="$$(zig version)" cargo zigbuild --locked -p rustack-cli --test workflows_sdk --no-default-features --features lambda,stepfunctions,s3,dynamodb --target aarch64-unknown-linux-gnu --message-format=json >"$(WORKFLOWS_LINUX_EVIDENCE_DIR)/build.json" + +test-workflows-performance-linux: workflows-performance-linux-build + @python3 -I -B tests/workflows/linux_performance.py host "$(WORKFLOWS_LINUX_PROFILE)" "$(WORKFLOWS_LINUX_EVIDENCE_DIR)/build.json" + +.PHONY: test-workflows-performance-linux-controls workflows-performance-linux-provision workflows-performance-linux-build test-workflows-performance-linux + +# D102: actual image samples and causal log-flood/timeout cleanup. +test-workflows-image-performance-controls: + @python3 -I -B -m unittest discover -s tests/workflows -p test_image_performance.py -v + @python3 -I -B -m unittest discover -s tests/workflows -p test_image_performance_channel.py -v + @$(MAKE) --no-print-directory test-workflows-performance-linux-controls + +test-workflows-image-performance: test-workflows-image-performance-controls test-lambda-docker-manifest + @set -eu; for performance_case in \ + test_should_measure_one_hundred_warm_image_invocations \ + test_should_measure_twenty_preloaded_cold_image_environments \ + test_should_reuse_one_image_slot_after_one_hundred_log_flood_timeouts; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$performance_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +workflows-image-performance-linux-build: test-workflows-server-linker test-workflows-image-performance-controls + @mkdir -p "$(WORKFLOWS_LINUX_EVIDENCE_DIR)" + @CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$(CURDIR)/tests/workflows/server_linker.py" WORKFLOWS_SERVER_LINKER_TOOL="$$(command -v cargo-zigbuild)" WORKFLOWS_SERVER_LINKER_TARGET=aarch64-linux-gnu CARGO_ZIGBUILD_ZIG_VERSION="$$(zig version)" cargo zigbuild --locked -p rustack-cli --test docker_startup --no-default-features --features lambda,stepfunctions,s3,dynamodb --target aarch64-unknown-linux-gnu --message-format=json >"$(WORKFLOWS_LINUX_EVIDENCE_DIR)/image-build.json" + +workflows-image-performance-linux-images: + @python3 -I -B tests/workflows/image_performance_images.py prepare "$(WORKFLOWS_LINUX_PROFILE)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_LINUX_EVIDENCE_DIR)" v1 + +test-workflows-image-performance-linux: workflows-image-performance-linux-build test-lambda-docker-manifest + @python3 -I -B tests/workflows/linux_image_performance.py host "$(WORKFLOWS_LINUX_PROFILE)" "$(WORKFLOWS_LINUX_EVIDENCE_DIR)/image-build.json" "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/lib/python3.13/site-packages" + +.PHONY: test-workflows-image-performance-controls test-workflows-image-performance workflows-image-performance-linux-build workflows-image-performance-linux-images test-workflows-image-performance-linux + +# D103: isolated rootless infrastructure; full platform qualification stays separate. +WORKFLOWS_ROOTLESS_PROFILE ?= rustack-workflows-rootless-arm64 +WORKFLOWS_ROOTLESS_PREVIOUS_EVIDENCE_DIR ?= +WORKFLOWS_ROOTLESS_EVIDENCE_DIR ?= $(shell cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-rootless-d103 + +test-workflows-rootless-controls: + @python3 -I -B -m unittest discover -s tests/workflows -p test_rootless_platform.py -v + +workflows-rootless-provision: test-workflows-rootless-controls + @python3 -I -B tests/workflows/rootless_platform.py provision "$(WORKFLOWS_ROOTLESS_PROFILE)" "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')" "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" + +workflows-rootless-attest: test-workflows-rootless-controls + @python3 -I -B tests/workflows/rootless_platform.py attest "$(WORKFLOWS_ROOTLESS_PROFILE)" "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" + +workflows-rootless-resume-setup: test-workflows-rootless-controls + @python3 -I -B tests/workflows/rootless_platform.py resume "$(WORKFLOWS_ROOTLESS_PROFILE)" "$(WORKFLOWS_ROOTLESS_PREVIOUS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" + +workflows-rootless-recover-attestation: test-workflows-rootless-controls + @python3 -I -B tests/workflows/rootless_platform.py recover-attest "$(WORKFLOWS_ROOTLESS_PROFILE)" "$(WORKFLOWS_ROOTLESS_PREVIOUS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" + +.PHONY: test-workflows-rootless-controls workflows-rootless-provision workflows-rootless-attest workflows-rootless-resume-setup workflows-rootless-recover-attestation + +# D104: explicit preloaded rootless resource consumers, preserving D103 installation. +WORKFLOWS_ROOTLESS_FORWARD_DIR ?= /tmp/rustack-rootless-forward-d104 +WORKFLOWS_ROOTLESS_RESOURCE_DIR ?= $(shell cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-rootless-resources-d104 + +test-workflows-rootless-resource-controls: + @python3 -I -B -m unittest discover -s tests/workflows -p test_rootless_resources.py -v + +workflows-rootless-forward: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_resources.py forward "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" + +workflows-rootless-kernel-images: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_resources.py prepare "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" + +workflows-rootless-functional-images: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_resources.py prepare-full "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" + +test-workflows-rootless-kernel: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_resources.py kernel "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" "$(WORKFLOWS_DOCKER_TARGET_DIR)" + +test-workflows-rootless-functional: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_functional.py "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" "$(WORKFLOWS_DOCKER_TARGET_DIR)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/arm64" "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" "$(WORKFLOWS_OTHER_DOCKER_SOCKET)" + +test-workflows-rootless-server: test-workflows-rootless-resource-controls + @python3 -I -B tests/workflows/rootless_functional.py --server-focus "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" "$(WORKFLOWS_DOCKER_TARGET_DIR)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/arm64" "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" "$(WORKFLOWS_OTHER_DOCKER_SOCKET)" + +.PHONY: test-workflows-rootless-resource-controls workflows-rootless-forward workflows-rootless-kernel-images workflows-rootless-functional-images test-workflows-rootless-kernel test-workflows-rootless-functional test-workflows-rootless-server + +WORKFLOWS_RUST_EVIDENCE_DIR ?= $(shell cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-rust-verification + +verify-workflows-rust: + @python3 -I -B tests/workflows/rust_gates.py "$(WORKFLOWS_RUST_EVIDENCE_DIR)" + +.PHONY: verify-workflows-rust + +# D75: exact same-feature resource inventories, before any resource execution. +test-workflows-manifest: + @python3 -I -B -m unittest discover -s tests/workflows -p test_required_suites.py -v + @set -eu; \ + workflow_manifest_dir=$$(mktemp -d); \ + trap 'rm -rf "$$workflow_manifest_dir"' EXIT; \ + cargo test --locked -p rustack-cli --lib --features stepfunctions -- --ignored --list --color never >"$$workflow_manifest_dir/native-bridge.txt"; \ + cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions -- --ignored --list --color never >"$$workflow_manifest_dir/registered-sdk.txt"; \ + cargo test --locked -p rustack-stepfunctions-http --test network -- --ignored --list --color never >"$$workflow_manifest_dir/typed-http-sdk.txt"; \ + python3 -I -B tests/workflows/required_suites.py "$$workflow_manifest_dir" + +# This owned pure-service smoke needs no Native bootstrap, uv, boto3 or Docker. +smoke-workflows: test-workflows-manifest + @cargo test --locked -p rustack-cli --test workflows_sdk --features stepfunctions test_should_execute_documented_pure_workflow_in_owned_strict_service -- --ignored --exact --nocapture + +# Explicit current-host workflow scope; the full image/platform release gate remains open. +verify-workflows-native: test-workflows-manifest + @$(MAKE) --no-print-directory -j1 test-stepfunctions-model test-stepfunctions-http test-stepfunctions-http-sdk test-stepfunctions-core test-stepfunctions-runtime test-stepfunctions-bridge-native test-stepfunctions-sdk test-workflows-persistence test-workflows-performance test-workflows-archive test-workflows-compression + @echo 'Verified native-workflow-only suites; image/ZIP/IaC/reference-platform release acceptance remains required.' + +.PHONY: test-workflows-manifest smoke-workflows verify-workflows-native + +deny-codegen: + @cargo deny --manifest-path codegen/Cargo.toml check + +.PHONY: audit-codegen deny-codegen + codegen-kms: @cd codegen && cargo run -- --config services/kms.toml --model smithy-model/kms.json --output ../crates/rustack-kms-model/src @cargo +nightly fmt -p rustack-kms-model @@ -158,6 +500,8 @@ SERVICE_SMOKE_TESTS := \ test_sts::tests::test_should_get_caller_identity_root \ test_lambda::tests::test_should_create_and_get_function \ test_lambda::tests::test_should_invoke_dry_run \ + test_lambda::tests::test_should_enforce_revision_preconditions_through_the_sdk_wire \ + test_lambda::tests::test_should_dry_run_code_without_mutation_and_reject_unsupported_deployment_fields \ test_lambda::tests::test_should_create_function_with_s3_code \ test_lambda::tests::test_should_update_function_code_from_s3_and_read_new_metadata \ test_lambda::tests::test_should_fail_create_function_with_missing_s3_bucket \ @@ -197,6 +541,68 @@ smoke: integration: @cargo test -p rustack-integration -- --ignored +# Shared runtime protocol tests and strict production boundary checks. +test-json-boundaries: + @cargo test -p rustack-core + @cargo clippy -p rustack-core --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + +# Requires a running Lambda HTTP service at SMOKE_ENDPOINT; executes actual AWS SDK IO. +test-lambda-revision-wire: + @S3_ENDPOINT_URL="$(SMOKE_ENDPOINT)" cargo test -p rustack-integration --lib test_lambda::tests::test_should_enforce_revision_preconditions_through_the_sdk_wire -- --ignored --exact --test-threads=1 + +.PHONY: test-lambda-revision-wire + +test-lambda-validation-wire: + @S3_ENDPOINT_URL="$(SMOKE_ENDPOINT)" cargo test -p rustack-integration --lib test_lambda::tests::test_should_dry_run_code_without_mutation_and_reject_unsupported_deployment_fields -- --ignored --exact --test-threads=1 + +.PHONY: test-lambda-validation-wire + +# Deployment state, retained image preparation ownership and provider control-plane tests. +test-lambda-deployments: + @cargo test -p rustack-lambda-core --lib deployment + +.PHONY: test-lambda-deployments + +test-lambda-runtime: + @cargo test -p rustack-lambda-runtime + @cargo clippy -p rustack-lambda-runtime --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-json-boundaries test-lambda-runtime + +# Strict production-library boundary gate; unit, integration and doctest regressions. +test-lambda-boundaries: + @cargo test -p rustack-auth -p rustack-core -p rustack-lambda-model -p rustack-lambda-http -p rustack-lambda-core -p rustack-lambda-runtime + @cargo clippy -p rustack-auth -p rustack-core -p rustack-lambda-model -p rustack-lambda-http -p rustack-lambda-core -p rustack-lambda-runtime --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-boundaries + +# Descriptor lifetime, persisted namespace identity and cleanup ownership. +test-lambda-ownership: + @cargo test -p rustack-lambda-core --lib docker::ownership::tests + @cargo test -p rustack-lambda-core --lib test_should_retain_the_directory_lock_through_abandoned_image_owner_cleanup + +.PHONY: test-lambda-ownership + +# Bounded function output, snapshot/read fairness and joined attachment cleanup. +test-lambda-logs: + @cargo test -p rustack-lambda-core --lib docker::logs::tests + +.PHONY: test-lambda-logs + +test-lambda-environment-plans: + @cargo test -p rustack-lambda-core --lib docker::plan::tests + +.PHONY: test-lambda-environment-plans + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: @@ -756,3 +1162,671 @@ test-iam-integration: test-iam-unit test-iam-integration .PHONY: smoke smoke-services smoke-regressions + +# Explicit real two-container environment lifecycle qualification. +test-lambda-environments-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_environment -- --ignored --test-threads=1 + +.PHONY: test-lambda-environments-docker + +# Pure ownership/failure regressions plus transport failure precedence. +test-lambda-environments: + @cargo test -p rustack-lambda-core --lib docker::environment::tests + @cargo test -p rustack-lambda-runtime --lib test_should_preserve_transport_worker_failure_after_a_prior_scope_error + +.PHONY: test-lambda-environments + +# Warm checkout ownership, bounded FIFO, LRU and result-consumption regressions. +test-lambda-pool: + @cargo test -p rustack-lambda-core --lib docker::pool::tests + +# Includes the real warm-counter/revision-eviction case and owner regressions. +test-lambda-pool-docker: test-lambda-environments-docker + +.PHONY: test-lambda-pool test-lambda-pool-docker + +# Actual Docker Executor/provider/HTTP composition via pinned ordinary boto3. +test-lambda-executor: + @cargo test -p rustack-lambda-core --lib docker::executor::tests + @cargo test -p rustack-lambda-core --lib handler::tests + +test-lambda-executor-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_executor -- --ignored --test-threads=1 + +.PHONY: test-lambda-executor test-lambda-executor-docker + +# Container-origin gateway proof requires an explicit operator bridge/host. +WORKFLOWS_GATEWAY_HOST ?= + +test-lambda-endpoints: + @cargo test --locked -p rustack-lambda-runtime --lib probe::endpoint + @cargo test --locked -p rustack-lambda-runtime --lib diagnostics:: + @cargo test --locked -p rustack-lambda-core --lib docker::probe + @cargo test --locked -p rustack-cli --bin rustack gateway::endpoint_tests + +test-lambda-endpoints-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" RUSTACK_RUNTIME_NETWORK="$(WORKFLOWS_NETWORK)" RUSTACK_RUNTIME_GATEWAY_HOST="$(WORKFLOWS_GATEWAY_HOST)" cargo test --locked -p rustack-lambda-core --test docker_endpoint -- --ignored --nocapture --test-threads=1 + +.PHONY: test-lambda-endpoints test-lambda-endpoints-docker + +# The bridge lane requires an explicit pre-provisioned operator network. +test-lambda-probes: + @cargo test -p rustack-lambda-runtime --lib probe:: + @cargo test -p rustack-lambda-core --lib docker::network::tests + @cargo test -p rustack-lambda-core --lib docker::probe::tests + +test-lambda-probes-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" RUSTACK_RUNTIME_NETWORK="$(WORKFLOWS_NETWORK)" cargo test -p rustack-lambda-core --test docker_probe -- --ignored --test-threads=1 + +.PHONY: test-lambda-probes test-lambda-probes-docker + +# Durable container identities, uncertain absence and actual restart recovery. +test-lambda-recovery: + @cargo test -p rustack-lambda-core --lib docker::ownership::journal::tests + @cargo test -p rustack-lambda-core --lib docker::recovery::tests + @cargo test -p rustack-lambda-core --lib docker::environment::tests::test_should_publish_intent_before_create_and_retain_reservation_until_durable_erase + +test-lambda-recovery-docker: lambda-runtime-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --lib docker::recovery::tests::test_should_reconcile_real_previous_run_containers_from_durable_intents -- --ignored --test-threads=1 + +.PHONY: test-lambda-recovery test-lambda-recovery-docker + +# Durable private tag intents and container-ordered historical alias recovery. +test-lambda-image-recovery: + @cargo test -p rustack-lambda-core --lib docker::ownership::images::tests + @cargo test -p rustack-lambda-core --lib docker::recovery::images::tests + @cargo test -p rustack-lambda-core --lib docker::lease::tests + +test-lambda-image-recovery-docker: lambda-runtime-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --lib docker::recovery::images::tests::test_should_recover_real_private_alias_across_directory_reacquisition -- --ignored --test-threads=1 + +.PHONY: test-lambda-image-recovery test-lambda-image-recovery-docker + +# Persisted engine scope and same-connection identity fences before owned IO. +test-lambda-daemon-binding: + @cargo test -p rustack-lambda-core --lib docker::binding::tests + @cargo test -p rustack-lambda-core --lib docker::ownership::daemon::tests + +test-lambda-daemon-binding-docker: + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_OTHER_DOCKER_SOCKET="$(WORKFLOWS_OTHER_DOCKER_SOCKET)" cargo test -p rustack-lambda-core --lib docker::binding::tests::real::test_should_bind_real_installation_across_reacquisition_and_reject_other_daemon -- --ignored --exact --test-threads=1 + +.PHONY: test-lambda-daemon-binding test-lambda-daemon-binding-docker + +# Recorded retirement plus complete bounded startup namespace inventories. +test-lambda-namespace: + @cargo test -p rustack-lambda-core --lib docker::recovery::namespace::tests + @cargo test -p rustack-lambda-core --lib docker::recovery::inventory::tests + +test-lambda-namespace-docker: lambda-runtime-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --lib docker::recovery::namespace::tests::real::test_should_reconcile_real_namespace_and_preserve_unrecorded_observations -- --ignored --test-threads=1 + +.PHONY: test-lambda-namespace test-lambda-namespace-docker + +# Pure operator settings, private YAML files and retained read-worker limits. +test-lambda-docker-settings: + @cargo test --locked -p rustack-core --lib settings::docker_tests + @cargo test --locked -p rustack-lambda-core --lib docker::config + +.PHONY: test-lambda-docker-settings + +# Exact registry policy, retained pull streams and image-preparation integration. +WORKFLOWS_PULL_IMAGE ?= public.ecr.aws/lambda/nodejs@sha256:92b7cccd985332238af80ac8da42d6965d418552e88955e315da4e3c0c76ca79 + +test-lambda-pull: + @cargo test --locked -p rustack-lambda-core --lib docker::pull + +test-lambda-pull-docker: + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_PULL_IMAGE="$(WORKFLOWS_PULL_IMAGE)" cargo test --locked -p rustack-lambda-core --lib docker::pull::tests::real::test_should_pull_approved_immutable_image_prepare_live_pin_and_join_cleanup -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: test-lambda-pull test-lambda-pull-docker + +# Delivery clocks, protocol-v2 correlation and actual Native process retirement. +test-lambda-timeouts: + @cargo test -p rustack-lambda-runtime --lib + @cargo test -p rustack-lambda-echo-bootstrap --test hardening + +# Captured handler-timeout/retirement/replacement through the real image pool. +test-lambda-timeouts-docker: test-lambda-environments-docker + +.PHONY: test-lambda-timeouts test-lambda-timeouts-docker + +# Invocation log capabilities/receipts and independent diagnostic capture. +test-lambda-log-windows: + @cargo test -p rustack-lambda-core --lib docker::logs::tests + +# Includes requested/silent/uncaptured warm-call log isolation on actual images. +test-lambda-log-windows-docker: test-lambda-environments-docker + +.PHONY: test-lambda-log-windows test-lambda-log-windows-docker + + +# Ordinary production startup consumes actual receipts; no fixture readiness injection. +# Isolate the executable from concurrent plain/minimal workspace builds. +WORKFLOWS_DOCKER_CLI_FEATURES := lambda,stepfunctions,s3,dynamodb +WORKFLOWS_DOCKER_TARGET_DIR ?= $(shell cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-docker +test-lambda-startup: + @cargo test --locked -p rustack-core --lib startup_tests + @cargo test --locked -p rustack-lambda-core --lib docker::startup + @cargo test --locked -p rustack-lambda-core --lib docker::ownership::tests + @cargo test --locked -p rustack-cli --bin rustack --features stepfunctions lambda_docker + +test-lambda-startup-docker: lambda-docker-fixtures + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) -- --ignored --nocapture --test-threads=1 + +.PHONY: test-lambda-startup test-lambda-startup-docker + + +# Independent custom Rust RIC fixtures consumed through ordinary production startup. +LAMBDA_IMAGE_FIXTURE_DIR ?= /tmp/rustack-image-fixture +lambda-zip-code-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --locked -p rustack-lambda-echo-bootstrap --bin zip-code-bootstrap --features zip-code --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)" + +test-lambda-provided-fixture: + @python3 -I -B -m unittest discover -s tests/workflows -p test_provided_zip_fixture.py -v + @cargo test --locked -p rustack-lambda-core --lib docker::zip::tests + @cargo test --locked -p rustack-lambda-core --lib provider::provided_zip_tests + @cargo test --locked -p rustack-lambda-core --lib test_should_acquire_code_profile_with_retained_job + @cargo test --locked -p rustack-lambda-core --lib test_should_distinguish_unconfigured_zip + @cargo clippy --locked -p rustack-lambda-echo-bootstrap --bin bootstrap --features provided-qualification --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + @cargo clippy --locked -p rustack-lambda-runtime-verifier --all-targets --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-provided-fixture + +test-lambda-code-fixture: test-lambda-provided-fixture test-lambda-zip-archive + @cargo test --locked -p rustack-lambda-core --lib docker::code_volume::tests + @cargo test --locked -p rustack-lambda-core --lib docker::ownership::codes::tests + @cargo test --locked -p rustack-lambda-core --lib docker::recovery::code::tests + @cargo test --locked -p rustack-lambda-core --lib docker::code::tests + @cargo test --locked -p rustack-lambda-core --lib docker::volumes::tests + @cargo test --locked -p rustack-lambda-core --lib storage::zip_layout::tests + @cargo clippy --locked -p rustack-lambda-echo-bootstrap --bin zip-code-bootstrap --features zip-code --no-deps -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +WORKFLOWS_ZIP_BASE_IMAGE ?= public.ecr.aws/lambda/provided@sha256:612c98f126c2a95e77a016d56fed50a83055e11d636132ee6ebdd461f01b9617 + +# Same Linux bootstrap source and byte-for-byte Native verifier for provided ZIPs. +lambda-provided-zip-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --locked -p rustack-lambda-echo-bootstrap --bin bootstrap --features provided-qualification --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/provided" + @python3 -I -B tests/workflows/provided_zip_fixture.py "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/provided/bootstrap/bootstrap" "$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/provided" + +lambda-provided-base-image: + @python3 -I -B -c 'import re,subprocess,sys; re.fullmatch(r"public\.ecr\.aws/lambda/provided@sha256:[0-9a-f]{64}", sys.argv[1]) or sys.exit("immutable provided base fixture required"); subprocess.run([sys.argv[2], "--host", "unix://" + sys.argv[3], "image", "pull", "--platform", "linux/" + sys.argv[4], sys.argv[1]], check=True, timeout=120)' "$(WORKFLOWS_ZIP_BASE_IMAGE)" "$(WORKFLOWS_DOCKER_BIN)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ARCH)" + +lambda-zip-native-verifier-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --locked -p rustack-lambda-runtime-verifier --bin rustack-lambda-runtime-verifier --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/native-verifier" + +.PHONY: lambda-provided-zip-build lambda-zip-native-verifier-build lambda-provided-base-image + +.PHONY: lambda-zip-code-build test-lambda-code-fixture + +lambda-custom-ric-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --locked -p rustack-lambda-echo-bootstrap --bin image-bootstrap --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)" + +lambda-custom-ric-images: lambda-custom-ric-build + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/image.Dockerfile --build-arg IMAGE_REVISION=A --tag "rustack-custom-ric:a-$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/image-bootstrap" + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/image.Dockerfile --build-arg IMAGE_REVISION=B --tag "rustack-custom-ric:b-$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/image-bootstrap" + +.PHONY: lambda-custom-ric-build lambda-custom-ric-images + +# Explicit caches: genuine incompatible metadata and an independent protocol-free process. +lambda-negative-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --locked -p rustack-lambda-echo-bootstrap --bin no-runtime-bootstrap --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)" + @python3 -B tests/lambda-fixtures/negative_images.py "$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/negative" + +lambda-negative-images: lambda-negative-build + @$(WORKFLOWS_DOCKER_BIN) --host "unix://$(WORKFLOWS_DOCKER_SOCKET)" image load --input "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/negative/os.tar" + @$(WORKFLOWS_DOCKER_BIN) --host "unix://$(WORKFLOWS_DOCKER_SOCKET)" image load --input "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/negative/architecture.tar" + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/no-runtime.Dockerfile --tag "rustack-no-runtime:$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/no-runtime-bootstrap" + +.PHONY: lambda-negative-build lambda-negative-images + +# Preserved baseline; the interrupted-pull case owns only the extra port443 reference. +lambda-pull-fixture-image: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @python3 -I -B -c 'import re,subprocess,sys; re.fullmatch(r"public\.ecr\.aws/lambda/nodejs@sha256:[0-9a-f]{64}", sys.argv[1]) or sys.exit("immutable public Node fixture required"); subprocess.run([sys.argv[2], "--host", "unix://" + sys.argv[3], "image", "pull", "--platform", "linux/" + sys.argv[4], sys.argv[1]], check=True, timeout=120)' "$(WORKFLOWS_PULL_IMAGE)" "$(WORKFLOWS_DOCKER_BIN)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ARCH)" + +.PHONY: lambda-pull-fixture-image + +WORKFLOWS_DOCKER_BIN ?= $(shell command -v docker) +test-lambda-custom-ric-docker: test-lambda-startup-docker + +.PHONY: test-lambda-custom-ric-docker + +test-lambda-custom-ric-fixture: workflows-sdk-python + @cargo test --locked -p rustack-lambda-echo-bootstrap --bin image-bootstrap + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_custom_ric_collector.py -v + @cargo clippy --locked -p rustack-lambda-echo-bootstrap --bin image-bootstrap -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-custom-ric-fixture + +# Real user-code failures, causal signed client cancellation and Event admission. +test-lambda-faults-docker: test-lambda-startup-docker + +test-lambda-faults: + @cargo test --locked -p rustack-lambda-core --lib impossible_memory + @cargo test --locked -p rustack-lambda-core --lib work::tests + +.PHONY: test-lambda-faults test-lambda-faults-docker + +test-lambda-fault-fixture: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_lambda_faults.py -v + @cargo clippy --locked -p rustack-lambda-echo-bootstrap --bin image-bootstrap -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-fault-fixture + +# Real management IO barriers keep the ordinary server and real daemon in charge. +test-lambda-io-faults-docker: test-lambda-startup-docker + +test-lambda-io-faults: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_docker_io_proxy.py -v + @python3 -I -B -m unittest discover -s tests/workflows -p test_negative_images.py -v + +.PHONY: test-lambda-io-faults test-lambda-io-faults-docker + +test-lambda-deadline-fixture: test-lambda-io-faults + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_lambda_admission.py -v + @cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_validate_pull_completion_before_granting_cache_cleanup + @cargo clippy --locked -p rustack-lambda-echo-bootstrap --bin no-runtime-bootstrap -- -D warnings -W clippy::pedantic -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-deadline-fixture + +# Fixed inventories are compared before any resource test is executed. +test-lambda-docker-manifest: + @python3 -I -B -m unittest discover -s tests/workflows -p test_required_suites.py -v + @python3 -I -B -m unittest discover -s tests/workflows -p test_docker_required_suites.py -v + @set -eu; docker_manifest_dir="$$(mktemp -d)"; \ + trap 'rm -rf "$$docker_manifest_dir"' EXIT; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) -- --ignored --list --color never >"$$docker_manifest_dir/ordinary-startup.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --lib -- --ignored --list --color never >"$$docker_manifest_dir/docker-core.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_environment -- --ignored --list --color never >"$$docker_manifest_dir/docker-environment.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_executor -- --ignored --list --color never >"$$docker_manifest_dir/docker-executor.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_endpoint -- --ignored --list --color never >"$$docker_manifest_dir/docker-endpoint.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_probe -- --ignored --list --color never >"$$docker_manifest_dir/docker-probe.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_wire -- --ignored --list --color never >"$$docker_manifest_dir/docker-wire.txt"; \ + cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-runtime --test docker -- --ignored --list --color never >"$$docker_manifest_dir/relay-runtime.txt"; \ + python3 -B tests/workflows/docker_required_suites.py "$$docker_manifest_dir" + +WORKFLOWS_DOCKER_TEST_ENV = PATH="$(dir $(WORKFLOWS_DOCKER_BIN)):$(PATH)" \ + RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" \ + RUSTACK_OTHER_DOCKER_SOCKET="$(WORKFLOWS_OTHER_DOCKER_SOCKET)" \ + RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" \ + RUSTACK_RUNTIME_NETWORK="$(WORKFLOWS_NETWORK)" RUSTACK_RUNTIME_GATEWAY_HOST="$(WORKFLOWS_GATEWAY_HOST)" \ + RUSTACK_PROVIDED_ZIP_A="$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/provided/provided-a.zip" \ + RUSTACK_PROVIDED_ZIP_B="$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/provided/provided-b.zip" \ + RUSTACK_ZIP_NATIVE_VERIFIER="$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/native-verifier/rustack-lambda-runtime-verifier/bootstrap" \ + RUSTACK_ZIP_BASE_IMAGE="$(WORKFLOWS_ZIP_BASE_IMAGE)" \ + RUSTACK_ZIP_CODE_BOOTSTRAP="$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/zip-code-bootstrap/bootstrap" \ + RUSTACK_CUSTOM_IMAGE_A="rustack-custom-ric:a-$(WORKFLOWS_ARCH)" RUSTACK_CUSTOM_IMAGE_B="rustack-custom-ric:b-$(WORKFLOWS_ARCH)" \ + RUSTACK_NODE_IMAGE="rustack-official-node:$(WORKFLOWS_ARCH)" \ + RUSTACK_HANDLER_SDK_IMAGE="rustack-handler-sdk:$(WORKFLOWS_ARCH)" \ + RUSTACK_CALLBACK_IMAGE="rustack-callback:$(WORKFLOWS_ARCH)" \ + RUSTACK_PULUMI_IMAGE="rustack-pulumi:$(WORKFLOWS_ARCH)" \ + RUSTACK_SERVER_IMAGE="$(WORKFLOWS_SERVER_IMAGE)" \ + RUSTACK_RELEASE_IMAGE="$(WORKFLOWS_RELEASE_IMAGE)" \ + RUSTACK_SERVER_DAEMON_SOCKET="$(WORKFLOWS_DOCKER_DAEMON_SOCKET)" \ + RUSTACK_SERVER_SOCKET_GID="$(WORKFLOWS_DOCKER_DAEMON_SOCKET_GID)" \ + RUSTACK_DOCKER_BIN="$(WORKFLOWS_DOCKER_BIN)" RUSTACK_PULL_IMAGE="$(WORKFLOWS_PULL_IMAGE)" \ + RUSTACK_WORKFLOWS_PYTHON="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" + +# Functional Docker scope only. Keep orchestration sequential even under parent make -j. +test-lambda-docker: + @test -n "$(WORKFLOWS_DOCKER_SOCKET)" && test -n "$(WORKFLOWS_OTHER_DOCKER_SOCKET)" && test "$(WORKFLOWS_DOCKER_SOCKET)" != "$(WORKFLOWS_OTHER_DOCKER_SOCKET)" && test -n "$(WORKFLOWS_NETWORK)" && test -n "$(WORKFLOWS_GATEWAY_HOST)" && test -x "$(WORKFLOWS_DOCKER_BIN)" + @test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET)" && test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET_GID)" + @$(MAKE) --no-print-directory lambda-docker-fixtures + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --lib -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_environment -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_executor -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_endpoint -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_probe -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_wire -- --ignored --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-runtime --test docker -- --ignored --nocapture --test-threads=1 + @echo 'Verified docker-image-functional resources; required arm64/reference/Desktop/rootless and complete release acceptance remain required.' + +.PHONY: test-lambda-docker-manifest test-lambda-docker + +# Durable code namespace recovery only; ordinary runtime qualification is separate. +test-lambda-code-recovery-docker: lambda-runtime-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test --locked -p rustack-lambda-core --lib docker::recovery::code::tests::real::test_should_recover_real_code_volume_and_unstarted_loader_across_reacquisition -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: test-lambda-code-recovery-docker + +# Focused ordinary provided-runtime and byte-identical Linux Native qualification. +test-lambda-provided-docker: test-lambda-docker-manifest + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_execute_ordinary_provided_zip_and_join_owned_retirement -- --ignored --nocapture --test-threads=1 + @$(MAKE) --no-print-directory test-lambda-provided-native-docker + +test-lambda-provided-io-docker: test-lambda-docker-manifest + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) zip_ -- --ignored --nocapture --test-threads=1 + +test-lambda-provided-native-docker: test-lambda-docker-manifest + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-lambda-core --test docker_wire code::test_should_execute_the_same_provided_zip_bytes_through_real_linux_native_and_join_children -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: test-lambda-provided-docker test-lambda-provided-native-docker test-lambda-provided-io-docker + + +# D93: bounded managed archive data/profile/candidate and raw codec controls. +test-lambda-zip-archive: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_provided_zip_archive.py -v + @cargo test --locked -p rustack-lambda-core --lib provider::snapshot + @cargo test --locked -p rustack-lambda-core --lib storage::restore_zip + @cargo test --locked -p rustack-lambda-core --lib deployment::manager::tests::restore + @cargo test --locked -p rustack-cli --bin rustack --no-default-features --features lambda,stepfunctions snapshot::archive::cbor + @cargo test --locked -p rustack-cli --bin rustack --no-default-features --features lambda,stepfunctions snapshot::image_tests + +test-lambda-provided-archive-docker: lambda-runtime-image lambda-runtime-fixture-image lambda-provided-zip-build lambda-provided-base-image workflows-sdk-python test-lambda-docker-manifest + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) provided_zip_archives -- --ignored --nocapture --test-threads=1 + +.PHONY: test-lambda-zip-archive test-lambda-provided-archive-docker + +# D94: official pinned Node24 RIC, explicit selected-platform source and SDK observer. +lambda-node-fixture-image: lambda-pull-fixture-image + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/node.Dockerfile --tag "rustack-official-node:$(WORKFLOWS_ARCH)" tests/lambda-fixtures + +test-lambda-node-fixture: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_node_image.py -v + @cargo test --locked -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) node::tests + @cargo test --locked -p rustack-lambda-runtime --lib http::tests + @cargo test --locked -p rustack-lambda-runtime --lib session::tests::test_should_charge_http_body + +test-lambda-node-docker: + @$(MAKE) --no-print-directory -j1 lambda-runtime-image lambda-runtime-fixture-image lambda-node-fixture-image workflows-sdk-python test-lambda-node-fixture test-lambda-docker-manifest + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_execute_official_node_image_with_signed_sdks_and_asl -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: lambda-node-fixture-image test-lambda-node-fixture test-lambda-node-docker + +# D95: one ordinary CLI cohort and a complete sequential fixture prerequisite set. +workflows-retain-fixture-sources: + @python3 -I -B tests/workflows/image_performance_images.py retain "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ARCH)" v1 + +lambda-docker-fixtures: + @$(MAKE) --no-print-directory -j1 workflows-retain-fixture-sources + @$(MAKE) --no-print-directory -j1 lambda-runtime-image lambda-runtime-fixture-image lambda-custom-ric-images lambda-zip-code-build lambda-provided-zip-build lambda-zip-native-verifier-build lambda-provided-base-image lambda-negative-images lambda-pull-fixture-image lambda-node-fixture-image lambda-handler-sdk-fixture-image lambda-callback-fixture-image lambda-pulumi-fixture-image lambda-workflows-server-image lambda-workflows-release-image workflows-sdk-python test-workflows-iac-fixture test-workflows-server-fixture test-workflows-release-fixture test-lambda-code-fixture test-lambda-deadline-fixture test-lambda-fault-fixture test-lambda-custom-ric-fixture test-lambda-node-fixture test-lambda-handler-sdk-fixture test-lambda-callback-fixture test-lambda-docker-manifest + +.PHONY: workflows-retain-fixture-sources + +lambda-handler-sdk-fixture-image: lambda-pull-fixture-image + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/handler-sdk.Dockerfile --tag "rustack-handler-sdk:$(WORKFLOWS_ARCH)" tests/lambda-fixtures + +test-lambda-handler-sdk-fixture: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_handler_sdk_image.py -v + @cargo test --locked -p rustack-lambda-core --lib sdk + @cargo clippy --locked -p rustack-cli --bin rustack --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --no-deps -- -D warnings -W clippy::pedantic + +test-lambda-handler-sdk-docker: lambda-docker-fixtures + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_execute_local_s3_and_dynamodb_from_an_explicit_image_sdk_context -- --ignored --exact --nocapture --test-threads=1 + @$(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_reject_wrong_image_sdk_credentials_without_ambient_fallback -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: lambda-docker-fixtures lambda-handler-sdk-fixture-image test-lambda-handler-sdk-fixture test-lambda-handler-sdk-docker + +# Requalify the shared image SDK owner against already recorded source images. +test-lambda-image-sdk-consumers-docker: + @$(MAKE) --no-print-directory -j1 test-lambda-node-fixture test-lambda-handler-sdk-fixture test-lambda-docker-manifest + @set -eu; for image_sdk_case in \ + test_should_execute_local_s3_and_dynamodb_from_an_explicit_image_sdk_context \ + test_should_reject_wrong_image_sdk_credentials_without_ambient_fallback \ + test_should_execute_official_node_image_with_signed_sdks_and_asl; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$image_sdk_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +.PHONY: test-lambda-image-sdk-consumers-docker + +lambda-callback-fixture-image: lambda-pull-fixture-image + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --file tests/lambda-fixtures/callback.Dockerfile --tag "rustack-callback:$(WORKFLOWS_ARCH)" tests/lambda-fixtures + +test-lambda-callback-fixture: workflows-sdk-python + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_callback_image.py -v + @cargo test --locked -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) callback::audit::tests + @cargo clippy --locked -p rustack-cli --bin rustack --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --no-deps -- -D warnings -W clippy::pedantic + +test-lambda-callback-docker: lambda-docker-fixtures + @set -eu; for callback_case in \ + test_should_execute_real_image_callbacks_before_and_after_lambda_response \ + test_should_enforce_image_callback_retry_heartbeat_timeouts_and_stop; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$callback_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +.PHONY: lambda-callback-fixture-image test-lambda-callback-fixture test-lambda-callback-docker + +# Private fixture dependencies; no global npm/Pulumi configuration or install scripts. +WORKFLOWS_IAC_NPM_CACHE ?= $(LAMBDA_IMAGE_FIXTURE_DIR)/iac-npm + +workflows-iac-node-dependencies: + @python3 -I -B tests/workflows/pulumi_dependencies.py "$(WORKFLOWS_IAC_NPM_CACHE)" + +test-workflows-iac-dependencies: workflows-iac-node-dependencies + @PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests/workflows -p test_pulumi_dependencies.py -v + @node --test tests/lambda-fixtures/pulumi/dependency-controls.js + @env -i PATH="$(PATH)" NPM_CONFIG_UPDATE_NOTIFIER=false npm --prefix tests/lambda-fixtures/pulumi audit --registry=https://registry.npmjs.org --userconfig="$(WORKFLOWS_IAC_NPM_CACHE)/config/user" --globalconfig="$(WORKFLOWS_IAC_NPM_CACHE)/config/global" --cache="$(WORKFLOWS_IAC_NPM_CACHE)" + +.PHONY: workflows-iac-node-dependencies test-workflows-iac-dependencies + +# Bounded, checksummed Linux executable provisioning; downloads are never executed on the host. +lambda-pulumi-tools: + @python3 -I -B tests/lambda-fixtures/pulumi_tools.py "$(WORKFLOWS_ARCH)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/pulumi-tools" + +test-workflows-iac-tools: + @PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests/workflows -p test_pulumi_tools.py -v + +.PHONY: lambda-pulumi-tools test-workflows-iac-tools + +lambda-pulumi-fixture-image: lambda-pull-fixture-image lambda-pulumi-tools + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --build-context "pulumi-tools=$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/pulumi-tools" --file tests/lambda-fixtures/pulumi.Dockerfile --tag "rustack-pulumi:$(WORKFLOWS_ARCH)" tests/lambda-fixtures + +test-workflows-iac-fixture: test-workflows-iac-dependencies test-workflows-iac-tools workflows-sdk-python + @node --test tests/lambda-fixtures/pulumi/program-controls.js tests/lambda-fixtures/pulumi/child-controls.js tests/lambda-fixtures/pulumi/input-controls.js tests/lambda-fixtures/pulumi/evidence-controls.js + @node --check tests/lambda-fixtures/iac.js + @node --check tests/lambda-fixtures/pulumi/iac_session.js + @node --check tests/lambda-fixtures/pulumi/iac_runtime.js + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_pulumi_archive.py -v + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_pulumi_replay.py -v + +.PHONY: lambda-pulumi-fixture-image test-workflows-iac-fixture + + +test-workflows-iac-docker: lambda-docker-fixtures + @set -eu; for iac_case in \ + test_should_manage_a_real_image_workflow_through_pulumi_lifecycle \ + test_should_retire_the_pulumi_plugin_tree_after_controller_stop; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$iac_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +.PHONY: test-workflows-iac-docker + +# Actual Linux server fixture; credentials arrive only through a private env file. +WORKFLOWS_SERVER_IMAGE ?= rustack/workflows-server:integration-$(WORKFLOWS_ARCH) +WORKFLOWS_DOCKER_DAEMON_SOCKET ?= +WORKFLOWS_DOCKER_DAEMON_SOCKET_GID ?= + +test-workflows-server-linker: + @python3 -I -B -m unittest discover -s tests/workflows -p test_server_linker.py -v + +lambda-workflows-server-build: test-workflows-server-linker + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @CARGO_TARGET_$(if $(filter arm64,$(WORKFLOWS_ARCH)),AARCH64,X86_64)_UNKNOWN_LINUX_GNU_LINKER="$(CURDIR)/tests/workflows/server_linker.py" WORKFLOWS_SERVER_LINKER_TOOL="$$(command -v cargo-zigbuild)" WORKFLOWS_SERVER_LINKER_TARGET="$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-linux-gnu" CARGO_ZIGBUILD_ZIG_VERSION="$$(zig version)" cargo zigbuild --locked -p rustack-cli --bin rustack --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release + @mkdir -p "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/server-image" + @cp "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu/release/rustack" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/server-image/rustack" + +lambda-workflows-server-image: lambda-workflows-server-build lambda-provided-base-image + @cp tests/lambda-fixtures/server-config.yaml "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/server-image/config.yaml" + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --build-arg "BASE_IMAGE=$(WORKFLOWS_ZIP_BASE_IMAGE)" --file tests/lambda-fixtures/server.Dockerfile --tag "$(WORKFLOWS_SERVER_IMAGE)" "$(LAMBDA_IMAGE_FIXTURE_DIR)/$(WORKFLOWS_ARCH)/server-image" + +.PHONY: test-workflows-server-linker lambda-workflows-server-build lambda-workflows-server-image + +test-workflows-server-fixture: workflows-sdk-python test-workflows-server-linker + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_server_container_import.py -v + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_stepfunctions_history.py -v + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_server_container_oracle.py -v + @PYTHONDONTWRITEBYTECODE=1 "$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" -m unittest discover -s tests/workflows -p test_handler_sdk_image.py -v + @cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) container_server:: + @cargo clippy --locked -p rustack-cli --bin rustack --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --no-deps -- -D warnings -W clippy::pedantic + +test-workflows-server-docker: + @test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET)" && test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET_GID)" + @$(MAKE) --no-print-directory lambda-docker-fixtures + @set -eu; for server_container_case in \ + test_should_execute_image_sdk_and_nested_asl_from_a_server_container \ + test_should_refuse_unreachable_server_container_gateway_and_join_startup_cleanup; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$server_container_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +.PHONY: test-workflows-server-fixture test-workflows-server-docker + +# D99: the actual release recipe; source defaults remain unchanged. +WORKFLOWS_RELEASE_IMAGE ?= rustack/workflows-release:integration-$(WORKFLOWS_ARCH) +WORKFLOWS_RELEASE_BUILD_JOBS ?= 2 + +lambda-workflows-release-image: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @case "$(WORKFLOWS_RELEASE_BUILD_JOBS)" in 1|2|3|4|5|6|7|8) ;; *) echo "WORKFLOWS_RELEASE_BUILD_JOBS must be between 1 and 8" >&2; exit 1;; esac + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --provenance=false --platform "linux/$(WORKFLOWS_ARCH)" --build-arg "CARGO_BUILD_JOBS=$(WORKFLOWS_RELEASE_BUILD_JOBS)" --file Dockerfile --tag "$(WORKFLOWS_RELEASE_IMAGE)" . + @$(MAKE) --no-print-directory test-workflows-release-image + +test-workflows-release-image: + @python3 -B tests/workflows/release_image.py "$(WORKFLOWS_ARCH)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_RELEASE_IMAGE)" "$(WORKFLOWS_DOCKER_BIN)" + +.PHONY: lambda-workflows-release-image test-workflows-release-image + +# Independent process-control qualification, separate from Native/Docker resources. +test-workflows-action-manifest: + @PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests/workflows -p test_action_required_suites.py -v + @set -eu; action_inventory_dir=$$(mktemp -d); chmod 700 "$$action_inventory_dir"; \ + cargo test --locked -p rustack-cli --test workflows_action -- --ignored --list --color never >"$$action_inventory_dir/composite-action.txt"; \ + python3 -B tests/workflows/action_required_suites.py "$$action_inventory_dir"; \ + rm -r "$$action_inventory_dir" + +test-workflows-action-fixture: workflows-sdk-python test-workflows-action-manifest + @RUSTACK_WORKFLOWS_PYTHON="$$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')/workflows-sdk-python/bin/python" cargo test --locked -p rustack-cli --test workflows_action test_should_execute_actual_action_process_controls -- --ignored --exact --nocapture + +.PHONY: test-workflows-action-manifest test-workflows-action-fixture + +# Actual release image policy, Action and complete both-topology regression inputs. +test-workflows-release-fixture: test-workflows-action-fixture + @PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests/workflows -p test_release_image.py -v + @cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) container_server:: + @cargo clippy --locked -p rustack-cli --test workflows_action --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --no-deps -- -D warnings -W clippy::pedantic + +test-workflows-release-docker: + @test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET)" && test -n "$(WORKFLOWS_DOCKER_DAEMON_SOCKET_GID)" + @$(MAKE) --no-print-directory lambda-docker-fixtures + @set -eu; for release_image_case in \ + test_should_execute_image_sdk_and_nested_asl_from_the_release_image \ + test_should_refuse_unreachable_release_image_gateway_and_join_startup_cleanup; do \ + $(WORKFLOWS_DOCKER_TEST_ENV) cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test docker_startup --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) "$$release_image_case" -- --ignored --exact --nocapture --test-threads=1; \ + done + +.PHONY: test-workflows-release-fixture test-workflows-release-docker + +# Final M4 evidence gate. Missing platforms/reference receipts are hard failures. +WORKFLOWS_RELEASE_BUNDLE ?= +WORKFLOWS_RELEASE_REPORT ?= +verify-workflows: + @test -n "$(WORKFLOWS_RELEASE_BUNDLE)" && test -n "$(WORKFLOWS_RELEASE_REPORT)" + @python3 -I -B tests/workflows/release_qualification.py "$(WORKFLOWS_RELEASE_BUNDLE)" "$(WORKFLOWS_RELEASE_REPORT)" + +.PHONY: verify-workflows + +WORKFLOWS_IMAGE_REFRESH_BUILD_DIR ?= +WORKFLOWS_ROOTLESS_PREVIOUS_RESOURCE_DIR ?= +workflows-refresh-production-images: + @test -n "$(WORKFLOWS_IMAGE_REFRESH_BUILD_DIR)" + @python3 -I -B tests/workflows/image_refresh_build.py "$(WORKFLOWS_IMAGE_REFRESH_BUILD_DIR)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_IMAGE_BUILDER)" + +workflows-rootless-refresh-images: + @test -n "$(WORKFLOWS_ROOTLESS_PREVIOUS_RESOURCE_DIR)" && test -n "$(WORKFLOWS_IMAGE_REFRESH_BUILD_DIR)" + @python3 -I -B tests/workflows/rootless_resources.py refresh-full "$(WORKFLOWS_ROOTLESS_EVIDENCE_DIR)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ROOTLESS_FORWARD_DIR)" "$(WORKFLOWS_ROOTLESS_RESOURCE_DIR)" "$(WORKFLOWS_ROOTLESS_PREVIOUS_RESOURCE_DIR)" "$(WORKFLOWS_IMAGE_REFRESH_BUILD_DIR)" + +.PHONY: workflows-refresh-production-images workflows-rootless-refresh-images + +workflows-release-seal: + @test -n "$(WORKFLOWS_RELEASE_BUNDLE)" + @python3 -I -B tests/workflows/release_delivery.py seal "$(WORKFLOWS_RELEASE_BUNDLE)" "$(WORKFLOWS_ARCH)" + +workflows-release-preflight: + @test -n "$(WORKFLOWS_RELEASE_BUNDLE)" + @python3 -I -B tests/workflows/release_delivery.py verify "$(WORKFLOWS_RELEASE_BUNDLE)" + +# Explicit publication entry point; never a prerequisite of a local test/build. +workflows-release-promote: + @test -n "$(WORKFLOWS_RELEASE_BUNDLE)" + @python3 -I -B tests/workflows/release_delivery.py promote "$(WORKFLOWS_RELEASE_BUNDLE)" + +.PHONY: workflows-release-seal workflows-release-preflight workflows-release-promote + +workflows-release-input-runs: + @python3 -I -B tests/workflows/release_delivery.py input-runs + +.PHONY: workflows-release-input-runs + +WORKFLOWS_PLATFORM_REPORT ?= +workflows-platform-receipt: + @test -n "$(WORKFLOWS_PLATFORM_REPORT)" + @python3 -I -B tests/workflows/platform_receipt.py "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_PLATFORM_REPORT)" + +.PHONY: workflows-platform-receipt + +WORKFLOWS_EVIDENCE_ROOT ?= +WORKFLOWS_EVIDENCE_OUTPUT ?= +workflows-release-evidence: + @test -n "$(WORKFLOWS_EVIDENCE_ROOT)" && test -n "$(WORKFLOWS_EVIDENCE_OUTPUT)" + @python3 -I -B tests/workflows/release_bundle.py "$(WORKFLOWS_EVIDENCE_ROOT)" "$(WORKFLOWS_EVIDENCE_OUTPUT)" + +.PHONY: workflows-release-evidence + +test-workflows-release-qualification-controls: + @python3 -I -B -m unittest discover -s tests/workflows -p test_release_qualification.py -v + +.PHONY: test-workflows-release-qualification-controls + +WORKFLOWS_FUNCTIONAL_CONFIG ?= +WORKFLOWS_FUNCTIONAL_EVIDENCE ?= +test-workflows-docker-functional: + @test -n "$(WORKFLOWS_FUNCTIONAL_CONFIG)" && test -n "$(WORKFLOWS_FUNCTIONAL_EVIDENCE)" + @python3 -I -B tests/workflows/docker_functional.py "$(WORKFLOWS_FUNCTIONAL_CONFIG)" "$(WORKFLOWS_FUNCTIONAL_EVIDENCE)" + +.PHONY: test-workflows-docker-functional + +WORKFLOWS_NATIVE_EVIDENCE_DIR ?= +verify-workflows-native-recorded: + @test -n "$(WORKFLOWS_NATIVE_EVIDENCE_DIR)" + @python3 -I -B tests/workflows/native_gates.py "$(WORKFLOWS_NATIVE_EVIDENCE_DIR)" + +.PHONY: verify-workflows-native-recorded + +# Original pinned LocalStack pytest tests; no baseline updates or cloud execution. +WORKFLOWS_LOCALSTACK_PYTHON_DIR ?= $(HOME)/.target/workflows-localstack-python +WORKFLOWS_LOCALSTACK_PYTHON ?= $(WORKFLOWS_LOCALSTACK_PYTHON_DIR)/bin/python +WORKFLOWS_LOCALSTACK_EVIDENCE ?= +WORKFLOWS_LOCALSTACK_IMAGES ?= +WORKFLOWS_LOCALSTACK_BUILDER ?= + +workflows-localstack-python: + @test -x "$(WORKFLOWS_LOCALSTACK_PYTHON)" || uv venv --python 3.13 "$(WORKFLOWS_LOCALSTACK_PYTHON_DIR)" + @uv pip sync --python "$(WORKFLOWS_LOCALSTACK_PYTHON)" tests/workflows/localstack-requirements.txt + @uv pip install --python "$(WORKFLOWS_LOCALSTACK_PYTHON)" --no-deps -e vendors/localstack + +workflows-localstack-images: + @test -n "$(WORKFLOWS_LOCALSTACK_IMAGES)" && test -n "$(WORKFLOWS_LOCALSTACK_BUILDER)" + @python3 -I -B tests/workflows/localstack_images.py "$(WORKFLOWS_LOCALSTACK_IMAGES)" "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_LOCALSTACK_BUILDER)" + +test-workflows-localstack-controls: + @"$(WORKFLOWS_LOCALSTACK_PYTHON)" -I -B -m unittest discover -s tests/workflows -p test_localstack_policy.py -v + +check-workflows-localstack: test-workflows-localstack-controls + @cargo clippy --locked -p rustack-cli --lib --test localstack_compat --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) --no-deps -- -D warnings -W clippy::pedantic + +test-workflows-localstack-pure: + @test -n "$(WORKFLOWS_LOCALSTACK_EVIDENCE)" + @RUSTACK_WORKFLOWS_PYTHON="$(WORKFLOWS_LOCALSTACK_PYTHON)" RUSTACK_LOCALSTACK_EVIDENCE="$(WORKFLOWS_LOCALSTACK_EVIDENCE)" cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test localstack_compat --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_execute_original_localstack_pure_cases -- --ignored --exact --nocapture --test-threads=1 + +test-workflows-localstack-docker: + @test -n "$(WORKFLOWS_LOCALSTACK_EVIDENCE)" && test -n "$(WORKFLOWS_LOCALSTACK_IMAGES)" + @RUSTACK_WORKFLOWS_PYTHON="$(WORKFLOWS_LOCALSTACK_PYTHON)" RUSTACK_LOCALSTACK_EVIDENCE="$(WORKFLOWS_LOCALSTACK_EVIDENCE)" RUSTACK_LOCALSTACK_IMAGES="$(WORKFLOWS_LOCALSTACK_IMAGES)/images.json" RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test --locked --target-dir "$(WORKFLOWS_DOCKER_TARGET_DIR)" -p rustack-cli --test localstack_compat --no-default-features --features $(WORKFLOWS_DOCKER_CLI_FEATURES) test_should_execute_original_localstack_image_cases -- --ignored --exact --nocapture --test-threads=1 + +.PHONY: workflows-localstack-python workflows-localstack-images test-workflows-localstack-controls test-workflows-localstack-pure test-workflows-localstack-docker + +.PHONY: check-workflows-localstack diff --git a/README.md b/README.md index bd063bbf..20d6124f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A local AWS service emulator written in Rust, with **18 service adapters**. Routed operation counts describe API surface—not full AWS semantics, authorization enforcement, execution support, or persistence. See the [capability and safety guide](docs/runtime-hardening.md) before choosing a test workload. +An opt-in source build also provides [Step Functions workflows](docs/workflows-guide.md), including real Native and Docker Image Lambda integration and named Native/pure workflow snapshots. Docker requires explicit qualified startup; image archive restoration, ZIP and final platform/release qualification remain in progress. + ## Install ```bash @@ -289,7 +291,7 @@ Set `RUSTACK_CONFIG` to a YAML file; environment values override the correspondi | `GATEWAY_LISTEN` | `127.0.0.1:4566` | Bind address and port; container image explicitly binds `0.0.0.0` | | `RUSTACK_CONFIG` | *(unset)* | YAML configuration file | | `RUSTACK_ADVERTISED_ENDPOINT` | *(derived from bind address)* | Public local HTTP(S) authority for generated URLs | -| `LAMBDA_EXECUTOR` | `disabled` | Explicit `native`/`squib` selection required for execution; Docker unsupported | +| `LAMBDA_EXECUTOR` | `disabled` | Explicit backend selection required; Docker Image needs configured and qualified ownership, daemon, relay, network and kernel startup | | `SERVICES` | *(empty = all)* | Comma-separated list of services to enable | | `LOG_LEVEL` | `info` | Log level (`error`, `warn`, `info`, `debug`, `trace`) | | `RUST_LOG` | | Fine-grained tracing filter (overrides `LOG_LEVEL`) | diff --git a/action.yml b/action.yml index 0fb59b47..c03418a8 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: "Rustack" -description: "Start a high-performance AWS-compatible server (S3 + DynamoDB + SQS + SSM + SNS + Lambda) on localhost:4566 (drop-in replacement for LocalStack)" +description: "Start a high-performance AWS-compatible server (S3, DynamoDB, Lambda and Step Functions) on localhost:4566 (drop-in replacement for LocalStack)" author: "Tyr Chen" branding: @@ -28,11 +28,11 @@ inputs: required: false default: "30" access-key: - description: "AWS access key for authentication (set to empty to disable auth)" + description: "AWS access key (empty pair omits credentials; enabled strict services require a pair)" required: false default: "test" secret-key: - description: "AWS secret key for authentication (set to empty to disable auth)" + description: "AWS secret key (empty pair omits credentials; skip-auth flags still govern)" required: false default: "test" s3-skip-auth: @@ -59,6 +59,10 @@ inputs: description: "Skip Lambda signature validation (true/false)" required: false default: "true" + stepfunctions-skip-auth: + description: "Skip Step Functions signature validation (true/false)" + required: false + default: "true" persistence: description: "Enable durable storage persistence (true/false)" required: false @@ -79,66 +83,25 @@ outputs: runs: using: "composite" steps: - - name: Pull Rustack image - shell: bash - run: | - IMAGE="ghcr.io/tyrchen/rustack:${{ inputs.image-tag }}" - if docker image inspect "$IMAGE" > /dev/null 2>&1; then - echo "Image $IMAGE already available locally" - else - docker pull "$IMAGE" - fi - - - name: Start Rustack + - name: Validate, start and qualify Rustack id: start shell: bash - run: | - CONTAINER_ID=$(docker run -d \ - --name rustack \ - -p "${{ inputs.port }}:4566" \ - -e "DEFAULT_REGION=${{ inputs.default-region }}" \ - -e "LOG_LEVEL=${{ inputs.log-level }}" \ - -e "S3_SKIP_SIGNATURE_VALIDATION=${{ inputs.s3-skip-auth }}" \ - -e "DYNAMODB_SKIP_SIGNATURE_VALIDATION=${{ inputs.dynamodb-skip-auth }}" \ - -e "SQS_SKIP_SIGNATURE_VALIDATION=${{ inputs.sqs-skip-auth }}" \ - -e "SSM_SKIP_SIGNATURE_VALIDATION=${{ inputs.ssm-skip-auth }}" \ - -e "SNS_SKIP_SIGNATURE_VALIDATION=${{ inputs.sns-skip-auth }}" \ - -e "LAMBDA_SKIP_SIGNATURE_VALIDATION=${{ inputs.lambda-skip-auth }}" \ - -e "ACCESS_KEY=${{ inputs.access-key }}" \ - -e "SECRET_KEY=${{ inputs.secret-key }}" \ - -e "PERSISTENCE=${{ inputs.persistence }}" \ - -e "SERVICES=${{ inputs.services }}" \ - "ghcr.io/tyrchen/rustack:${{ inputs.image-tag }}") - - echo "container_id=$CONTAINER_ID" >> "$GITHUB_OUTPUT" - echo "endpoint=http://localhost:${{ inputs.port }}" >> "$GITHUB_OUTPUT" - - - name: Wait for healthy - shell: bash - run: | - ENDPOINT="http://localhost:${{ inputs.port }}" - TIMEOUT=${{ inputs.wait-timeout }} - ELAPSED=0 - - echo "Waiting for Rustack at $ENDPOINT ..." - - until curl -sf "$ENDPOINT/_localstack/health" > /dev/null 2>&1; do - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then - echo "::error::Rustack did not become healthy within ${TIMEOUT}s" - docker logs rustack - exit 1 - fi - sleep 0.5 - ELAPSED=$((ELAPSED + 1)) - done - - echo "Rustack is ready at $ENDPOINT (S3 + DynamoDB + SQS + SSM + SNS + Lambda)" - - - name: Export environment - shell: bash - run: | - echo "AWS_ENDPOINT_URL=http://localhost:${{ inputs.port }}" >> "$GITHUB_ENV" - echo "LOCALSTACK_ENDPOINT=http://localhost:${{ inputs.port }}" >> "$GITHUB_ENV" - echo "AWS_DEFAULT_REGION=${{ inputs.default-region }}" >> "$GITHUB_ENV" - echo "AWS_ACCESS_KEY_ID=${{ inputs.access-key }}" >> "$GITHUB_ENV" - echo "AWS_SECRET_ACCESS_KEY=${{ inputs.secret-key }}" >> "$GITHUB_ENV" + env: + RUSTACK_ACTION_ROOT: ${{ github.action_path }} + RUSTACK_ACTION_IMAGE_TAG: ${{ inputs.image-tag }} + RUSTACK_ACTION_PORT: ${{ inputs.port }} + RUSTACK_ACTION_DEFAULT_REGION: ${{ inputs.default-region }} + RUSTACK_ACTION_LOG_LEVEL: ${{ inputs.log-level }} + RUSTACK_ACTION_WAIT_TIMEOUT: ${{ inputs.wait-timeout }} + RUSTACK_ACTION_ACCESS_KEY: ${{ inputs.access-key }} + RUSTACK_ACTION_SECRET_KEY: ${{ inputs.secret-key }} + RUSTACK_ACTION_PERSISTENCE: ${{ inputs.persistence }} + RUSTACK_ACTION_SERVICES: ${{ inputs.services }} + RUSTACK_ACTION_S3_SKIP_AUTH: ${{ inputs.s3-skip-auth }} + RUSTACK_ACTION_DYNAMODB_SKIP_AUTH: ${{ inputs.dynamodb-skip-auth }} + RUSTACK_ACTION_SQS_SKIP_AUTH: ${{ inputs.sqs-skip-auth }} + RUSTACK_ACTION_SSM_SKIP_AUTH: ${{ inputs.ssm-skip-auth }} + RUSTACK_ACTION_SNS_SKIP_AUTH: ${{ inputs.sns-skip-auth }} + RUSTACK_ACTION_LAMBDA_SKIP_AUTH: ${{ inputs.lambda-skip-auth }} + RUSTACK_ACTION_STEPFUNCTIONS_SKIP_AUTH: ${{ inputs.stepfunctions-skip-auth }} + run: exec python3 -B "$RUSTACK_ACTION_ROOT/tools/github-action/runtime.py" diff --git a/apps/rustack/Cargo.toml b/apps/rustack/Cargo.toml index d3c426cd..dd38a982 100644 --- a/apps/rustack/Cargo.toml +++ b/apps/rustack/Cargo.toml @@ -23,6 +23,7 @@ sqs = ["dep:rustack-sqs-core", "dep:rustack-sqs-http"] ssm = ["dep:rustack-ssm-core", "dep:rustack-ssm-http"] sns = ["dep:rustack-sns-core", "dep:rustack-sns-http", "dep:rustack-sqs-model", "sqs"] lambda = ["dep:rustack-lambda-core", "dep:rustack-lambda-http"] +stepfunctions = ["dep:rustack-stepfunctions-core", "dep:rustack-stepfunctions-http", "dep:rustack-stepfunctions-model", "dep:getrandom", "dep:thiserror"] events = ["dep:rustack-events-core", "dep:rustack-events-http", "dep:rustack-sqs-model", "sqs"] logs = ["dep:rustack-logs-core", "dep:rustack-logs-http"] kms = ["dep:rustack-kms-core", "dep:rustack-kms-http"] @@ -63,6 +64,12 @@ rustack-sns-core = { workspace = true, optional = true } rustack-sns-http = { workspace = true, optional = true } rustack-sqs-model = { workspace = true, optional = true } +# Internal Step Functions composition support (main registration is separate) +rustack-stepfunctions-core = { workspace = true, optional = true } +rustack-stepfunctions-http = { workspace = true, optional = true } +rustack-stepfunctions-model = { workspace = true, optional = true } +getrandom = { workspace = true, optional = true } + # Internal crates - Lambda (optional, gated by "lambda" feature) rustack-lambda-core = { workspace = true, optional = true } rustack-lambda-http = { workspace = true, optional = true } @@ -133,6 +140,7 @@ ciborium = { workspace = true } # Error handling anyhow = { workspace = true } +thiserror = { workspace = true, optional = true } # Logging tracing = { workspace = true } @@ -146,12 +154,28 @@ bytes = { workspace = true } crc32fast = { workspace = true } zstd = { workspace = true } +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, features = ["signal", "process"] } + [dev-dependencies] +sha2 = { workspace = true } +proptest = { workspace = true } +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } tokio-test = { workspace = true } aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } aws-sdk-dynamodb = { workspace = true } aws-sdk-sqs = { workspace = true } +aws-sdk-sfn = { workspace = true } +aws-sdk-lambda = { workspace = true } aws-credential-types = { workspace = true } aws-smithy-runtime-api = { workspace = true } tempfile = { workspace = true } +tar = { workspace = true } +yaml-rust2 = { workspace = true } + +rustack-lambda-model = { workspace = true } +base64 = { workspace = true } +zip = { workspace = true } +uuid = { workspace = true } diff --git a/apps/rustack/src/events_bridge.rs b/apps/rustack/src/events_bridge.rs index cfd4208c..82a20f2b 100644 --- a/apps/rustack/src/events_bridge.rs +++ b/apps/rustack/src/events_bridge.rs @@ -1,4 +1,4 @@ -//! Application-owned EventBridge → SQS delivery bridge. +//! Application-owned `EventBridge` → SQS delivery bridge. use std::sync::Arc; use async_trait::async_trait; diff --git a/apps/rustack/src/gateway.rs b/apps/rustack/src/gateway.rs index 2f87ec84..67a1f869 100644 --- a/apps/rustack/src/gateway.rs +++ b/apps/rustack/src/gateway.rs @@ -16,7 +16,7 @@ use std::{ use bytes::Bytes; use http::{ Method, Response, StatusCode, - header::{CONTENT_TYPE, RETRY_AFTER}, + header::{ALLOW, CACHE_CONTROL, CONTENT_LENGTH, CONTENT_TYPE, RETRY_AFTER}, }; use http_body_util::BodyExt; use hyper::{ @@ -27,6 +27,8 @@ use rustack_core::{ http::{BodyBudget, BudgetedBody}, settings::RuntimeBudgets, }; +#[cfg(feature = "lambda")] +use rustack_lambda_core::docker::{GatewayInstanceId, GatewayReply}; use serde::Serialize; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -35,9 +37,13 @@ use crate::{ service::{GatewayBody, ServiceRouter, gateway_body_from_string}, }; +const RUNTIME_PROBE_PATH: &str = "/_rustack/runtime-probe"; + /// Process state is independent from whether any service supports snapshots. #[derive(Debug)] pub(crate) struct RuntimeStatus { + #[cfg(feature = "lambda")] + instance: Option, ready: AtomicBool, workers: Arc, } @@ -76,6 +82,8 @@ impl GatewayService { admission: Arc::new(Semaphore::new(budgets.requests)), budgets, state: Arc::new(RuntimeStatus { + #[cfg(feature = "lambda")] + instance: None, ready: AtomicBool::new(true), workers: Arc::new(RuntimeWorkers::default()), }), @@ -85,12 +93,62 @@ impl GatewayService { pub(crate) fn with_workers(mut self, workers: Arc) -> Self { self.state = Arc::new(RuntimeStatus { - ready: AtomicBool::new(true), + #[cfg(feature = "lambda")] + instance: self.state.instance, + ready: AtomicBool::new(self.state.ready.load(Ordering::Acquire)), workers, }); self } + #[cfg(feature = "lambda")] + pub(crate) fn with_runtime_identity(mut self, instance: GatewayInstanceId) -> Self { + self.state = Arc::new(RuntimeStatus { + ready: AtomicBool::new(self.state.ready.load(Ordering::Acquire)), + workers: self.state.workers.clone(), + instance: Some(instance), + }); + self + } + fn runtime_probe(method: &Method, installed: Option) -> Response { + let mut response = match installed { + Some(reply) if matches!(*method, Method::GET | Method::HEAD) => { + json_response(StatusCode::OK, &reply) + } + Some(_) => { + let mut reply = json_response( + StatusCode::METHOD_NOT_ALLOWED, + &serde_json::json!({"error":"runtime probe requires GET or HEAD"}), + ); + reply + .headers_mut() + .insert(ALLOW, http::HeaderValue::from_static("GET, HEAD")); + reply + } + None => json_response( + StatusCode::NOT_FOUND, + &serde_json::json!({"error":"runtime probe is not installed"}), + ), + }; + response + .headers_mut() + .insert(CACHE_CONTROL, http::HeaderValue::from_static("no-store")); + if *method == Method::HEAD { + if let Some(size) = response.body().size_hint().exact() { + response + .headers_mut() + .insert(CONTENT_LENGTH, http::HeaderValue::from(size)); + } + *response.body_mut() = gateway_body_from_string(""); + } + response + } + #[cfg(all(test, feature = "stepfunctions"))] + pub(crate) fn with_request_seconds(mut self, seconds: std::num::NonZeroU64) -> Self { + self.budgets.request_seconds = seconds.get(); + self + } + pub(crate) fn service_names(&self) -> Vec<&'static str> { self.services.iter().map(|service| service.name()).collect() } @@ -107,6 +165,13 @@ impl GatewayService { fn diagnostics(&self, req: &http::Request) -> Option> { let path = req.uri().path(); + if path == RUNTIME_PROBE_PATH { + #[cfg(feature = "lambda")] + let reply = self.state.instance.map(GatewayReply::new); + #[cfg(not(feature = "lambda"))] + let reply = None::; + return Some(Self::runtime_probe(req.method(), reply)); + } if !(is_health_check(req.method(), path) || matches!(*req.method(), Method::GET | Method::HEAD) && path == "/_rustack/capabilities") @@ -128,9 +193,13 @@ impl GatewayService { .iter() .map(|service| capability(service.name())) .collect(); + #[cfg(feature = "stepfunctions")] + let workflows = self.state.workers.workflow_capabilities(); + #[cfg(not(feature = "stepfunctions"))] + let workflows = None::; json_response( status, - &serde_json::json!({"version": env!("CARGO_PKG_VERSION"), "ready": ready, "services": capabilities, "workers": workers, "localEndpoints": {"apiGateway": "/_aws/execute-api/{apiId}/{stage}", "lambda": "/lambda-url/{functionName}/", "cloudFront": "/_aws/cloudfront/{distributionId}/"}, "limitations": ["No multi-tenant isolation or production IAM enforcement", "Snapshot resources-only services do not retain messages", "routed does not imply complete operation semantics"]}), + &serde_json::json!({"version": env!("CARGO_PKG_VERSION"), "ready": ready, "services": capabilities, "workers": workers, "workflows": workflows, "localEndpoints": {"apiGateway": "/_aws/execute-api/{apiId}/{stage}", "lambda": "/lambda-url/{functionName}/", "cloudFront": "/_aws/cloudfront/{distributionId}/"}, "limitations": ["No multi-tenant isolation or production IAM enforcement", "Snapshot resources-only services do not retain messages", "routed does not imply complete operation semantics"]}), ) } else { let services: std::collections::BTreeMap<_, _> = self @@ -186,10 +255,15 @@ impl Service> for GatewayService { } _ => self.budgets.request_seconds, }; + let owns_deadline = selected.is_some_and(|service| service.owns_request_deadline()); let service_future = selected.map(|service| service.call(req)); let budget = self.budgets.clone(); Box::pin(async move { let response = match service_future { + Some(future) if owns_deadline => match future.await { + Ok(response) => response, + Err(never) => match never {}, + }, Some(future) => { match tokio::time::timeout(Duration::from_secs(request_seconds), future).await { Ok(Ok(response)) => response, @@ -264,6 +338,9 @@ fn capability(name: &'static str) -> Capability { let snapshot = crate::snapshot::snapshot_coverage(name); let execution = match name { "lambda" => "explicit backend required; native is trusted-code-only", + "stepfunctions" => { + "M1 STANDARD JSONPath; actual integrations and limits in workflows capability" + } "sns" | "events" => "SQS targets only; target availability validated", _ => "partial AWS compatibility; see service capability documentation", }; @@ -310,7 +387,7 @@ fn unavailable(message: &str) -> Response { response } -fn json_response(status: StatusCode, value: &serde_json::Value) -> Response { +fn json_response(status: StatusCode, value: &T) -> Response { let (status, body) = match serde_json::to_string(value) { Ok(body) => (status, body), Err(_) => ( @@ -376,3 +453,6 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod endpoint_tests; diff --git a/apps/rustack/src/gateway/endpoint_tests.rs b/apps/rustack/src/gateway/endpoint_tests.rs new file mode 100644 index 00000000..785e2d96 --- /dev/null +++ b/apps/rustack/src/gateway/endpoint_tests.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "lambda")] +use std::sync::Arc; +use std::time::Duration; + +use http::Method; +use hyper::server::conn::http1; +use hyper_util::rt::TokioIo; +#[cfg(feature = "lambda")] +use rustack_lambda_core::docker::{GatewayInstanceId, GatewayReply}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + time::timeout, +}; + +use super::GatewayService; +#[cfg(feature = "lambda")] +use crate::runtime::RuntimeWorkers; + +async fn exchange(gateway: GatewayService, method: &str, path: &str) -> String { + timeout(Duration::from_secs(10), async { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = async { + let (stream, _) = listener.accept().await.unwrap(); + http1::Builder::new() + .serve_connection(TokioIo::new(stream), gateway) + .await + .unwrap(); + }; + let client = async { + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + format!( + "{method} {path} HTTP/1.1\r\nHost: fixture\r\nConnection: \ + close\r\nContent-Length: 0\r\n\r\n" + ) + .as_bytes(), + ) + .await + .unwrap(); + let mut bytes = Vec::new(); + stream.take(8193).read_to_end(&mut bytes).await.unwrap(); + assert!(bytes.len() <= 8192); + String::from_utf8(bytes).unwrap() + }; + let ((), wire) = tokio::join!(server, client); + wire + }) + .await + .unwrap() +} +#[tokio::test] +async fn test_should_return_disabled_probe_404_without_claiming_an_identity() { + let wire = exchange( + GatewayService::new(Vec::new()), + "GET", + "/_rustack/runtime-probe", + ) + .await; + assert!(wire.starts_with("HTTP/1.1 404")); + assert!(wire.contains("cache-control: no-store")); + assert!(!wire.contains("instanceId")); +} +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_preserve_probe_identity_across_workers_clone_closed_and_saturated_admission() { + let id = GatewayInstanceId::generate().unwrap(); + let gateway = GatewayService::new(Vec::new()) + .with_runtime_identity(id) + .with_workers(Arc::new(RuntimeWorkers::default())) + .health_only(); + let _permits = gateway + .admission + .clone() + .acquire_many_owned(u32::try_from(gateway.budgets.requests).unwrap()) + .await + .unwrap(); + gateway.state().drain(); + let wire = exchange(gateway.clone(), "GET", "/_rustack/runtime-probe").await; + assert!(wire.starts_with("HTTP/1.1 200")); + assert!(wire.contains("cache-control: no-store")); + let (_, body) = wire.split_once("\r\n\r\n").unwrap(); + let reply: GatewayReply = serde_json::from_str(body).unwrap(); + assert_eq!(reply.instance_id(), id); + assert!( + exchange(gateway, "GET", "/bucket") + .await + .starts_with("HTTP/1.1 503") + ); +} +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_bound_head_reply_framing_and_reject_probe_business_methods() { + let gateway = GatewayService::new(Vec::new()) + .with_runtime_identity(GatewayInstanceId::generate().unwrap()); + let wire = exchange(gateway.clone(), "HEAD", "/_rustack/runtime-probe").await; + assert!(wire.starts_with("HTTP/1.1 200")); + assert_eq!(wire.split_once("\r\n\r\n").unwrap().1, ""); + assert!(wire.contains("content-length:")); + let wire = exchange(gateway, "POST", "/_rustack/runtime-probe").await; + assert!(wire.starts_with("HTTP/1.1 405")); + assert!(wire.contains("allow: GET, HEAD")); +} +#[test] +fn test_should_keep_probe_method_outside_normal_health_aliases() { + assert!(!super::is_health_check( + &Method::POST, + "/_rustack/runtime-probe" + )); +} diff --git a/apps/rustack/src/lambda_docker.rs b/apps/rustack/src/lambda_docker.rs new file mode 100644 index 00000000..90821899 --- /dev/null +++ b/apps/rustack/src/lambda_docker.rs @@ -0,0 +1,242 @@ +//! Retained diagnostic-only listener while the exact Docker owner qualifies. + +use std::{future::Future, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use hyper::server::conn::http1; +use hyper_util::rt::{TokioIo, TokioTimer}; +use rustack_lambda_core::docker::{DockerRuntime, GatewayInstanceId}; +use tokio::{ + net::TcpListener, + runtime::Handle, + sync::watch, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; + +pub(crate) async fn qualify( + listener: Arc, + owner: &mut DockerRuntime, + instance: GatewayInstanceId, + deadline: Instant, +) -> Result<()> { + observe( + listener, + instance, + async { + owner + .qualify(deadline) + .await + .context("qualify production Docker runtime") + }, + deadline, + ) + .await +} + +pub(crate) async fn observe( + listener: Arc, + instance: GatewayInstanceId, + work: impl Future>, + deadline: Instant, +) -> Result { + let mut owner = DiagnosticOwner::start(listener, instance, deadline); + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .context("register startup SIGTERM")?; + tokio::pin!(work); + let result = tokio::select! {biased; + result=tokio::signal::ctrl_c()=>result.context("observe startup SIGINT").and_then(|()|Err(anyhow::anyhow!("runtime startup interrupted"))), + _=terminate.recv()=>Err(anyhow::anyhow!("runtime startup interrupted")), + ()=tokio::time::sleep_until(deadline)=>Err(anyhow::anyhow!("runtime startup observation expired")), + result=&mut work=>result, + result=owner.join()=>result.and_then(|()|Err(anyhow::anyhow!("startup diagnostic listener stopped before activation"))), + }; + // The deadline selects a result; it never cancels this retained join. + let cleanup = owner.shutdown().await; + let output = result?; + cleanup?; + ensure!( + Instant::now() < deadline, + "runtime startup observation expired" + ); + Ok(output) +} + +#[derive(Debug)] +struct DiagnosticOwner { + stop: watch::Sender, + task: Option>>, + settled: Option>>, + runtime: Handle, +} +impl DiagnosticOwner { + fn start(listener: Arc, instance: GatewayInstanceId, deadline: Instant) -> Self { + let (stop, stopped) = watch::channel(false); + let runtime = Handle::current(); + Self { + stop, + task: Some(runtime.spawn(serve(listener, instance, stopped, deadline))), + settled: None, + runtime, + } + } + async fn shutdown(&mut self) -> Result<()> { + self.stop.send_replace(true); + self.join().await + } + async fn join(&mut self) -> Result<()> { + if self.settled.is_none() { + let result = match self.task.as_mut() { + Some(task) => task + .await + .context("startup diagnostic supervisor failed") + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settled = Some(result.map_err(Arc::new)); + } + match &self.settled { + Some(Ok(())) => Ok(()), + Some(Err(error)) => Err(anyhow::anyhow!( + "startup diagnostic cleanup failed: {error:#}" + )), + None => bail!("startup diagnostic unsettled"), + } + } +} +impl Drop for DiagnosticOwner { + fn drop(&mut self) { + self.stop.send_replace(true); + if let Some(task) = self.task.take() { + // An abandoned observer cannot abort the actor's child joins. This + // retained reaper observes the exact supervisor on the captured runtime. + self.runtime.spawn(async move { + if !matches!(task.await, Ok(Ok(()))) { + tracing::error!("abandoned startup diagnostic cleanup failed"); + } + }); + } + } +} +async fn serve( + listener: Arc, + instance: GatewayInstanceId, + mut stopped: watch::Receiver, + deadline: Instant, +) -> Result<()> { + let service = crate::gateway::GatewayService::new(Vec::new()) + .with_runtime_identity(instance) + .health_only(); + service.state().drain(); + let mut tasks = JoinSet::new(); + let result=async {loop { + tokio::select! {biased; + _=stopped.wait_for(|stop|*stop)=>break Ok(()), + ()=tokio::time::sleep_until(deadline)=>break Ok(()), + result=tasks.join_next(),if !tasks.is_empty()=>{if let Some(Err(error))=result{bail!("startup diagnostic worker failed: {error}");}}, + connection=listener.accept()=>{ + let (stream,_)=connection.context("accept startup diagnostic connection")?; + if tasks.len()>=8{drop(stream);continue;} + let service=service.clone(); + tasks.spawn(async move { + let mut http=http1::Builder::new(); + http.timer(TokioTimer::new()).header_read_timeout(Duration::from_secs(rustack_core::settings::budgets().header_seconds)).max_headers(32).max_buf_size(16*1024); + let _=timeout_at(deadline,http.serve_connection(TokioIo::new(stream),service)).await; + }); + } + } + }}.await; + tasks.abort_all(); + let mut joined = true; + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + joined &= error.is_cancelled(); + } + } + result?; + ensure!(joined, "startup diagnostic workers failed during join"); + Ok(()) +} +pub(crate) async fn shutdown(owner: &mut Option, deadline: Instant) -> Result<()> { + if let Some(owner) = owner { + owner + .shutdown(deadline) + .await + .context("Docker runtime cleanup incomplete")?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::future::pending; + + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, + time::{sleep, timeout}, + }; + + use super::*; + #[tokio::test] + async fn test_should_join_held_diagnostic_connections_before_deadline_result() { + let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap()); + let address = listener.local_addr().unwrap(); + let observed = listener.clone(); + let job = tokio::spawn(observe( + observed, + GatewayInstanceId::generate().unwrap(), + pending::>(), + Instant::now() + Duration::from_millis(50), + )); + let mut client = TcpStream::connect(address).await.unwrap(); + client + .write_all(b"GET /_rustack/runtime-probe HTTP/1.1\r\n") + .await + .unwrap(); + assert!(job.await.unwrap().is_err()); + assert_eq!(Arc::strong_count(&listener), 1); + assert_eq!( + timeout(Duration::from_secs(1), client.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(TcpListener::bind(address).await.is_err()); + } + #[tokio::test] + async fn test_should_retain_diagnostic_join_after_startup_observer_abandonment() { + let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap()); + let address = listener.local_addr().unwrap(); + let job = tokio::spawn(observe( + listener.clone(), + GatewayInstanceId::generate().unwrap(), + pending::>(), + Instant::now() + Duration::from_secs(5), + )); + let mut client = TcpStream::connect(address).await.unwrap(); + client + .write_all(b"GET /_rustack/runtime-probe HTTP/1.1\r\n") + .await + .unwrap(); + sleep(Duration::from_millis(10)).await; + job.abort(); + assert!(job.await.unwrap_err().is_cancelled()); + timeout(Duration::from_secs(1), async { + while Arc::strong_count(&listener) != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), client.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + } +} diff --git a/apps/rustack/src/lib.rs b/apps/rustack/src/lib.rs new file mode 100644 index 00000000..38d1515b --- /dev/null +++ b/apps/rustack/src/lib.rs @@ -0,0 +1,12 @@ +//! Composition adapters shared by the Rustack binary and integration harnesses. +//! +//! Exporting an adapter does not register or advertise a gateway service. + +#![forbid(unsafe_code)] + +#[cfg(all(feature = "lambda", feature = "stepfunctions"))] +pub mod stepfunctions_bridge; + +/// Real registry-backed Step Functions API and stable pagination ownership. +#[cfg(feature = "stepfunctions")] +pub mod stepfunctions_api; diff --git a/apps/rustack/src/main.rs b/apps/rustack/src/main.rs index c52530a4..7d7198d2 100644 --- a/apps/rustack/src/main.rs +++ b/apps/rustack/src/main.rs @@ -36,6 +36,8 @@ mod events_bridge; mod gateway; #[cfg(feature = "s3")] mod handler; +#[cfg(all(feature = "lambda", unix))] +mod lambda_docker; #[cfg(all(feature = "lambda", feature = "s3"))] mod lambda_s3_bridge; mod runtime; @@ -43,6 +45,8 @@ mod service; mod snapshot; #[cfg(feature = "sns")] mod sns_bridge; +#[cfg(feature = "stepfunctions")] +mod workflows; use std::{net::SocketAddr, sync::Arc, time::Duration}; @@ -137,9 +141,17 @@ use rustack_lambda_core::code::{S3CodeFetcher, UnavailableS3CodeFetcher}; #[cfg(feature = "lambda")] use rustack_lambda_core::config::LambdaConfig; #[cfg(feature = "lambda")] +use rustack_lambda_core::docker::GatewayInstanceId; +#[cfg(feature = "lambda")] use rustack_lambda_core::handler::RustackLambdaHandler; #[cfg(feature = "lambda")] use rustack_lambda_core::provider::RustackLambda; +#[cfg(all(feature = "lambda", unix))] +use rustack_lambda_core::{ + docker::{DockerInstallation, DockerRuntime, DockerRuntimeSettings, DockerStartupConfig}, + executor::ExecutorBackend, + storage::FunctionStore, +}; #[cfg(feature = "lambda")] use rustack_lambda_http::service::{LambdaHttpConfig, LambdaHttpService}; #[cfg(feature = "logs")] @@ -196,6 +208,10 @@ use rustack_ssm_core::handler::RustackSsmHandler; use rustack_ssm_core::provider::RustackSsm; #[cfg(feature = "ssm")] use rustack_ssm_http::service::{SsmHttpConfig, SsmHttpService}; +#[cfg(feature = "stepfunctions")] +use rustack_stepfunctions_core::TenantScope; +#[cfg(feature = "stepfunctions")] +use rustack_stepfunctions_http::{RequestScope, StepFunctionsHttpConfig, StepFunctionsHttpService}; #[cfg(feature = "sts")] use rustack_sts_core::config::StsConfig; #[cfg(feature = "sts")] @@ -212,11 +228,13 @@ use tracing_subscriber::EnvFilter; use crate::events_bridge::LocalTargetDelivery; #[cfg(feature = "sns")] use crate::sns_bridge::RustackSqsPublisher; +#[cfg(feature = "stepfunctions")] +use crate::workflows::{WorkflowRuntime, WorkflowSupervisor}; use crate::{ gateway::GatewayService, runtime::RuntimeWorkers, service::ServiceRouter, - snapshot::{RuntimeProviders, SnapshotConfig}, + snapshot::{RuntimeProviders, SnapshotConfig, SnapshotSession}, }; /// Server version reported in health check responses. @@ -604,7 +622,7 @@ fn build_sts_http_config(config: &StsConfig) -> StsHttpConfig { } /// Build a credential provider from `ACCESS_KEY` / `SECRET_KEY` environment -/// variables (used by MinIO Mint and other test harnesses). +/// variables (used by `MinIO` Mint and other test harnesses). #[cfg(any( feature = "s3", feature = "dynamodb", @@ -612,6 +630,7 @@ fn build_sts_http_config(config: &StsConfig) -> StsHttpConfig { feature = "ssm", feature = "sns", feature = "lambda", + feature = "stepfunctions", feature = "events", feature = "logs", feature = "kms", @@ -643,7 +662,11 @@ fn build_credential_provider() -> Option Result<(Instant, Result<()>)> { +async fn serve( + listener: Arc, + service: GatewayService, + cleanup_reserve: Duration, +) -> Result<(Instant, Result<()>)> { let budgets = settings::budgets(); let graceful = hyper_util::server::graceful::GracefulShutdown::new(); let mut http = HttpConnBuilder::new(TokioExecutor::new()); @@ -700,7 +723,7 @@ async fn serve(listener: TcpListener, service: GatewayService) -> Result<(Instan drop(listener); let deadline = Instant::now() + Duration::from_secs(budgets.shutdown_seconds); info!("stopped ingress; draining connections"); - let drain_deadline = deadline - shutdown_cleanup_reserve(); + let drain_deadline = deadline - cleanup_reserve; let drained = tokio::time::timeout_at(drain_deadline, graceful.shutdown()) .await .context("HTTP drain deadline exceeded"); @@ -725,6 +748,7 @@ fn is_compiled_in(name: &str) -> bool { || (name == "ssm" && cfg!(feature = "ssm")) || (name == "sns" && cfg!(feature = "sns")) || (name == "lambda" && cfg!(feature = "lambda")) + || (name == "stepfunctions" && cfg!(feature = "stepfunctions")) || (name == "events" && cfg!(feature = "events")) || (name == "logs" && cfg!(feature = "logs")) || (name == "kms" && cfg!(feature = "kms")) @@ -770,6 +794,9 @@ fn parse_services_value(raw: &str) -> Vec { if cfg!(feature = "sns") { all.push("sns".to_string()); } + if cfg!(feature = "stepfunctions") { + all.push("stepfunctions".to_owned()); + } if cfg!(feature = "lambda") { all.push("lambda".to_string()); } @@ -890,7 +917,7 @@ async fn run_health_check(addr: &str) -> Result<()> { /// Read the gateway listen address from the environment. /// /// Checks `GATEWAY_LISTEN` (the canonical var) and falls back to the -/// S3Config default when S3 is compiled in. +/// `S3Config` default when S3 is compiled in. fn gateway_listen_addr() -> String { settings::var("GATEWAY_LISTEN").unwrap_or_else(|_| "127.0.0.1:4566".to_string()) } @@ -900,18 +927,79 @@ fn log_level() -> String { settings::var("LOG_LEVEL").unwrap_or_else(|_| "info".to_string()) } +fn enforce_startup_deadline(deadline: Instant) -> Result<()> { + if Instant::now() >= deadline { + anyhow::bail!("runtime startup observation deadline exceeded"); + } + Ok(()) +} + +async fn bind_before_deadline(address: SocketAddr, deadline: Instant) -> Result { + enforce_startup_deadline(deadline)?; + let listener = TcpListener::bind(address) + .await + .with_context(|| format!("failed to bind to {address}"))?; + enforce_startup_deadline(deadline)?; + Ok(listener) +} + /// Built runtime services and provider handles. struct Runtime { services: Vec>, providers: RuntimeProviders, workers: RuntimeWorkers, + #[cfg(feature = "stepfunctions")] + workflows: Option, +} + +impl Runtime { + /// Join constructed owners even when startup never opens the public listener. + async fn abort_startup(&mut self, deadline: Instant) -> Result<()> { + #[cfg(feature = "stepfunctions")] + let workflow = match &mut self.workflows { + Some(workflows) => workflows.shutdown(deadline).await, + None => Ok(()), + }; + let workers = tokio::time::timeout_at(deadline, self.workers.shutdown(deadline)) + .await + .context("startup provider cleanup observation expired")?; + #[cfg(feature = "stepfunctions")] + workflow?; + workers + } +} + +#[cfg(all(test, feature = "stepfunctions"))] +fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { + build_services_with_runtime( + is_enabled, + #[cfg(all(feature = "lambda", unix))] + None, + ) } /// Build all enabled service routers based on environment configuration. #[allow(clippy::too_many_lines)] -fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { +fn build_services_with_runtime( + is_enabled: impl Fn(&str) -> bool, + #[cfg(all(feature = "lambda", unix))] installation: Option, +) -> Result { let mut services: Vec> = Vec::new(); - let mut providers = RuntimeProviders::default(); + let providers = RuntimeProviders::default(); + #[cfg(any( + feature = "stepfunctions", + feature = "s3", + feature = "dynamodb", + feature = "dynamodbstreams", + feature = "sqs", + feature = "ssm", + feature = "iam", + feature = "lambda", + feature = "apigatewayv2", + feature = "cloudfront", + feature = "cloudfront-dataplane" + ))] + let mut providers = providers; let mut workers = RuntimeWorkers::default(); // ----- DynamoDB + DynamoDB Streams (register before S3: S3 is the catch-all) ----- @@ -922,7 +1010,9 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { dynamodb_skip_signature_validation = dynamodb_config.skip_signature_validation, "initializing DynamoDB service", ); - let mut dynamodb_provider = RustackDynamoDB::new(dynamodb_config.clone()); + let dynamodb_provider = RustackDynamoDB::new(dynamodb_config.clone()); + #[cfg(feature = "dynamodbstreams")] + let mut dynamodb_provider = dynamodb_provider; // Wire in DynamoDB Streams emitter and lifecycle manager if enabled. #[cfg(feature = "dynamodbstreams")] @@ -1254,8 +1344,16 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { }; #[cfg(not(feature = "s3"))] let code_fetcher: Arc = Arc::new(UnavailableS3CodeFetcher); - let lambda_provider = - Arc::new(RustackLambda::new(lambda_config.clone()).with_code_fetcher(code_fetcher)); + #[cfg(all(feature = "lambda", unix))] + let lambda_provider = match installation { + Some(installation) => installation.provider(FunctionStore::new( + lambda_data_directory().join("lambda-code"), + ))?, + None => RustackLambda::new(lambda_config.clone()), + }; + #[cfg(not(unix))] + let lambda_provider = RustackLambda::new(lambda_config.clone()); + let lambda_provider = Arc::new(lambda_provider.with_code_fetcher(code_fetcher)); providers.register_lambda(Arc::clone(&lambda_provider)); workers.lambda = Some(Arc::clone(&lambda_provider)); let lambda_handler = RustackLambdaHandler::new(Arc::clone(&lambda_provider)); @@ -1264,6 +1362,34 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { services.push(Box::new(service::LambdaServiceRouter::new(lambda_service))); } + #[cfg(feature = "stepfunctions")] + let workflows = if is_enabled("stepfunctions") { + let region = settings::var("DEFAULT_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); + let account = + settings::var("DEFAULT_ACCOUNT_ID").unwrap_or_else(|_| "000000000000".to_owned()); + let scope = TenantScope::new("aws", ®ion, &account)?; + let skip = settings::var("STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION") + .map_or(true, |value| value == "true"); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".to_owned(), region, account)?, + skip, + build_credential_provider(), + )?; + let workflow = WorkflowRuntime::new( + scope, + settings::budgets(), + #[cfg(feature = "lambda")] + workers.lambda.clone(), + )?; + providers.register_workflows(workflow.archive); + workers.workflows = Some(workflow.status); + let http = StepFunctionsHttpService::new(workflow.api, config); + services.insert(0, Box::new(service::StepFunctionsServiceRouter::new(http))); + Some(workflow.owner) + } else { + None + }; + // ----- CloudFront (management + data plane, register before S3 catch-all) ----- #[cfg(feature = "cloudfront")] if is_enabled("cloudfront") { @@ -1312,11 +1438,22 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { services, providers, workers, + #[cfg(feature = "stepfunctions")] + workflows, }) } -fn shutdown_cleanup_reserve() -> Duration { - (Duration::from_secs(settings::budgets().shutdown_seconds) / 10).min(Duration::from_secs(1)) +#[cfg(all(feature = "lambda", unix))] +fn lambda_data_directory() -> std::path::PathBuf { + settings::var("DATA_DIR").map_or_else( + |_| rustack_core::RustackConfig::default().data_dir.into(), + Into::into, + ) +} + +fn shutdown_cleanup_reserve(shutdown: Duration, docker_cleanup: Option) -> Duration { + let floor = (shutdown / 10).min(Duration::from_secs(1)); + docker_cleanup.map_or(floor, |cleanup| cleanup.min(shutdown / 2).max(floor)) } fn main() -> Result<()> { @@ -1392,7 +1529,95 @@ async fn run() -> Result<()> { } config.validate_credentials(&enabled)?; info!(settings = ?config, "validated runtime configuration"); - let runtime = build_services(|name| enabled.iter().any(|s| s == name))?; + let startup_deadline = + Instant::now() + Duration::from_secs(settings::budgets().startup_seconds); + #[cfg(feature = "lambda")] + let gateway_instance = if enabled.iter().any(|name| name == "lambda") { + Some( + GatewayInstanceId::generate_owned( + startup_deadline.min(Instant::now() + Duration::from_secs(60)), + ) + .await + .context("generate runtime gateway identity")?, + ) + } else { + None + }; + #[cfg(all(feature = "lambda", unix))] + let mut docker: Option = None; + #[cfg(all(feature = "lambda", unix))] + let mut startup_listener = None; + #[cfg(all(feature = "lambda", unix))] + let installation = if enabled.iter().any(|name| name == "lambda") { + let lambda = LambdaConfig::from_env()?; + if lambda.executor == ExecutorBackend::Docker { + let settings = DockerRuntimeSettings::from_lookup( + |key| settings::var(key).ok(), + &settings::budgets(), + lambda.idle_timeout, + )?; + let instance = gateway_instance.context("Lambda gateway identity absent")?; + let startup = + DockerStartupConfig::new(settings, lambda_data_directory(), instance, lambda)?; + let mut owner = DockerRuntime::new(startup)?; + let qualification = async { + let addr: SocketAddr = listen_addr.parse().context("invalid bind address")?; + let listener = Arc::new(bind_before_deadline(addr, startup_deadline).await?); + lambda_docker::qualify(listener.clone(), &mut owner, instance, startup_deadline) + .await?; + let installed = owner.installation()?; + Result::<_>::Ok((listener, installed)) + } + .await; + let (listener, installed) = match qualification { + Ok(qualified) => qualified, + Err(error) => { + let cleanup = owner + .shutdown( + Instant::now() + + Duration::from_secs(settings::budgets().shutdown_seconds), + ) + .await; + if cleanup.is_err() { + tracing::error!("startup Docker cleanup remains owned"); + } + return Err(error); + } + }; + startup_listener = Some(listener); + docker = Some(owner); + Some(installed) + } else { + None + } + } else { + None + }; + let construction = build_services_with_runtime( + |name| enabled.iter().any(|s| s == name), + #[cfg(all(feature = "lambda", unix))] + installation, + ); + let mut runtime = match construction { + Ok(runtime) => runtime, + Err(error) => { + #[cfg(all(feature = "lambda", unix))] + if lambda_docker::shutdown( + &mut docker, + Instant::now() + Duration::from_secs(settings::budgets().shutdown_seconds), + ) + .await + .is_err() + { + tracing::error!("startup Docker cleanup remains owned"); + } + return Err(error); + } + }; + #[cfg(all(feature = "lambda", unix))] + if let Some(owner) = &docker { + runtime.workers.docker = Some(owner.status()); + } if runtime.services.is_empty() { anyhow::bail!( @@ -1401,32 +1626,103 @@ async fn run() -> Result<()> { ); } - let _snapshot_lease = match snapshot_config.as_ref() { - Some(config) => Some(config.acquire_lease().await?), - None => None, + let mut snapshot = SnapshotSession::new(snapshot_config); + #[cfg(all(feature = "lambda", unix))] + let diagnostic_startup = startup_listener.is_some(); + let startup_work = async { + let work = async { + snapshot.load(&runtime.providers, startup_deadline).await?; + #[cfg(feature = "lambda")] + if let Some(lambda) = &runtime.workers.lambda { + lambda.validate_runtime_restore()?; + } + #[cfg(all(feature = "lambda", unix))] + if let Some(owner) = &docker { + owner.installation()?; + } + Result::<()>::Ok(()) + }; + #[cfg(all(feature = "lambda", unix))] + match startup_listener.as_ref() { + Some(listener) => { + lambda_docker::observe( + listener.clone(), + gateway_instance.context("Lambda gateway identity absent")?, + work, + startup_deadline, + ) + .await?; + } + None => work.await?, + } + #[cfg(not(all(feature = "lambda", unix)))] + work.await?; + let addr: SocketAddr = listen_addr + .parse() + .with_context(|| format!("invalid bind address: {listen_addr}"))?; + #[cfg(all(feature = "lambda", unix))] + let listener = match startup_listener.take() { + Some(listener) => listener, + None => Arc::new(bind_before_deadline(addr, startup_deadline).await?), + }; + #[cfg(not(all(feature = "lambda", unix)))] + let listener = Arc::new(bind_before_deadline(addr, startup_deadline).await?); + Result::<_>::Ok((addr, listener)) + }; + #[cfg(all(feature = "lambda", unix))] + let startup = if diagnostic_startup { + startup_work.await + } else { + tokio::time::timeout_at(startup_deadline, startup_work) + .await + .context("runtime startup observation deadline exceeded") + .and_then(|result| result) + }; + #[cfg(not(all(feature = "lambda", unix)))] + let startup = tokio::time::timeout_at(startup_deadline, startup_work) + .await + .context("runtime startup observation deadline exceeded") + .and_then(|result| result); + let (addr, listener) = match startup { + Ok(started) => started, + Err(error) => { + let cleanup_deadline = + Instant::now() + Duration::from_secs(settings::budgets().shutdown_seconds); + if let Err(cleanup) = snapshot.finish(cleanup_deadline).await { + tracing::error!(error = %cleanup, "startup snapshot cleanup incomplete"); + } + let cleanup = runtime.abort_startup(cleanup_deadline).await; + #[cfg(all(feature = "lambda", unix))] + if lambda_docker::shutdown(&mut docker, cleanup_deadline) + .await + .is_err() + { + tracing::error!("startup Docker cleanup remains owned"); + } + snapshot.quarantine(); + if let Err(cleanup) = cleanup { + tracing::error!(error = %cleanup, "startup cleanup incomplete"); + } + return Err(error); + } }; - if let Some(config) = snapshot_config.as_ref() { - info!(coverage = ?runtime.providers.coverage(), "snapshot includes only declared state; other services and SQS messages are not persisted"); - config.load(&runtime.providers).await?; - } let Runtime { services, providers, workers, + #[cfg(feature = "stepfunctions")] + mut workflows, } = runtime; let workers = Arc::new(workers); let gateway = GatewayService::new(services).with_workers(Arc::clone(&workers)); + #[cfg(feature = "lambda")] + let gateway = match gateway_instance { + Some(instance) => gateway.with_runtime_identity(instance), + None => gateway, + }; let service_names = gateway.service_names(); - let addr: SocketAddr = listen_addr - .parse() - .with_context(|| format!("invalid bind address: {listen_addr}"))?; - - let listener = TcpListener::bind(addr) - .await - .with_context(|| format!("failed to bind to {addr}"))?; - info!( %addr, services = ?service_names, @@ -1434,38 +1730,66 @@ async fn run() -> Result<()> { "starting Rustack Server", ); - let (deadline, serve_result) = match serve(listener, gateway).await { + #[cfg(all(feature = "lambda", unix))] + let docker_cleanup = workers + .docker + .as_ref() + .map(|_| Duration::from_secs(settings::budgets().lambda_docker_cleanup_seconds)); + #[cfg(not(all(feature = "lambda", unix)))] + let docker_cleanup = None; + let cleanup_reserve = shutdown_cleanup_reserve( + Duration::from_secs(settings::budgets().shutdown_seconds), + docker_cleanup, + ); + let (deadline, serve_result) = match serve(listener, gateway, cleanup_reserve).await { Ok(outcome) => outcome, Err(error) => ( Instant::now() + Duration::from_secs(settings::budgets().shutdown_seconds), Err(error), ), }; - let work_deadline = deadline - shutdown_cleanup_reserve(); + let work_deadline = deadline - cleanup_reserve; let save_result = async { serve_result?; - workers - .quiesce(work_deadline.saturating_duration_since(Instant::now())) - .await?; - if let Some(config) = snapshot_config.as_ref() { - tokio::time::timeout_at(work_deadline, config.save(&providers, VERSION)) - .await - .context("snapshot save deadline exceeded")??; + workers.quiesce_producers(work_deadline).await?; + #[cfg(feature = "stepfunctions")] + if let Some(workflows) = &mut workflows { + workflows.quiesce(work_deadline).await?; } + workers.quiesce_consumers(work_deadline).await?; + snapshot.save(&providers, VERSION, work_deadline).await?; Result::<()>::Ok(()) } .await; let stop_result = async { - match tokio::time::timeout_at(deadline, workers.shutdown()).await { - Ok(result) => result.context("runtime shutdown failed")?, - Err(_) => anyhow::bail!( - "resource cleanup incomplete: shutdown deadline exceeded; dropping providers now \ - (kill_on_drop reaps remaining children)" - ), - } + let snapshot_result = snapshot.finish(deadline).await; + #[cfg(feature = "stepfunctions")] + let workflow_result = match &mut workflows { + Some(workflows) => workflows.shutdown(deadline).await, + None => Ok(()), + }; + let worker_result = + match tokio::time::timeout_at(deadline, workers.shutdown(deadline)).await { + Ok(result) => result.context("runtime shutdown failed"), + Err(_) => Err(anyhow::anyhow!( + "resource cleanup incomplete: shutdown deadline exceeded; dropping providers \ + now (kill_on_drop reaps remaining children)" + )), + }; + #[cfg(all(feature = "lambda", unix))] + let docker_result = lambda_docker::shutdown(&mut docker, deadline).await; + snapshot_result?; + #[cfg(feature = "stepfunctions")] + workflow_result?; + worker_result?; + #[cfg(all(feature = "lambda", unix))] + docker_result?; Result::<()>::Ok(()) } .await; + if save_result.is_err() || stop_result.is_err() { + snapshot.quarantine(); + } save_result?; stop_result?; Ok(()) @@ -1473,7 +1797,48 @@ async fn run() -> Result<()> { #[cfg(test)] mod tests { + #[test] + fn test_should_reserve_docker_cleanup_within_the_deadline_and_preserve_native_grace() { + for (seconds, cleanup, expected_millis) in [ + (30, None, 1000), + (60, None, 1000), + (1, None, 100), + (30, Some(10), 10000), + (60, Some(60), 30000), + (1, Some(60), 500), + ] { + assert_eq!( + shutdown_cleanup_reserve( + Duration::from_secs(seconds), + cleanup.map(Duration::from_secs) + ), + Duration::from_millis(expected_millis) + ); + } + } + + #[tokio::test(start_paused = true)] + async fn test_should_refuse_expired_startup_before_opening_listener() -> Result<()> { + let reservation = TcpListener::bind("127.0.0.1:0").await?; + let address = reservation.local_addr()?; + drop(reservation); + let deadline = Instant::now() + Duration::from_secs(1); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(bind_before_deadline(address, deadline).await.is_err()); + assert!(tokio::net::TcpStream::connect(address).await.is_err()); + Ok(()) + } + + #[cfg(feature = "stepfunctions")] + use std::num::NonZeroU64; + + #[cfg(feature = "stepfunctions")] + use hyper::server::conn::http1; + #[cfg(feature = "stepfunctions")] + use rustack_core::http::BodyBudget; use tokio::io::AsyncWriteExt; + #[cfg(feature = "stepfunctions")] + use tokio::{io::AsyncReadExt, net::TcpStream}; use super::*; @@ -1497,6 +1862,246 @@ mod tests { Ok(result) } + #[cfg(feature = "stepfunctions")] + async fn gateway_exchange(gateway: GatewayService, wire: String) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let deadline = Instant::now() + Duration::from_secs(10); + let server = async { + let (stream, _) = listener.accept().await?; + http1::Builder::new() + .serve_connection(TokioIo::new(stream), gateway) + .await?; + Result::<()>::Ok(()) + }; + let client = async { + let mut stream = TcpStream::connect(address).await?; + stream.write_all(wire.as_bytes()).await?; + let mut response = Vec::new(); + stream.take(32_769).read_to_end(&mut response).await?; + anyhow::ensure!(response.len() <= 32_768, "fixture response exceeded cap"); + Result::::Ok(String::from_utf8(response)?) + }; + let (server, client) = tokio::join!( + tokio::time::timeout_at(deadline, server), + tokio::time::timeout_at(deadline, client) + ); + server??; + client? + } + + #[cfg(feature = "stepfunctions")] + #[tokio::test] + async fn test_should_prioritize_workflow_target_over_conflicting_path_and_signing_scope() + -> Result<()> { + let runtime = build_services(|name| { + ["stepfunctions", "lambda", "sns", "sqs", "s3", "dynamodb"].contains(&name) + })?; + let Runtime { + services, + workers, + mut workflows, + .. + } = runtime; + assert_eq!( + services.first().map(|service| service.name()), + Some("stepfunctions") + ); + let workers = Arc::new(workers); + let gateway = GatewayService::new(services).with_workers(workers.clone()); + let result = async { + for (path, headers) in [ + ( + "/2015-03-31/functions/Fn/invocations", + "X-Amz-Target: AWSStepFunctions.ListStateMachines\r\nContent-Type: \ + application/x-amz-json-1.0\r\n", + ), + ( + "/", + "X-Amz-Target: AWSStepFunctions.ListStateMachines\r\nContent-Type: \ + application/x-www-form-urlencoded\r\nAuthorization: AWS4-HMAC-SHA256 \ + Credential=local/date/us-east-1/sns/aws4_request\r\n", + ), + ( + "/", + "X-Amz-Target: DynamoDB_20120810.ListTables\r\nX-Amz-Target: \ + AWSStepFunctions.ListStateMachines\r\nContent-Type: \ + application/x-amz-json-1.0\r\n", + ), + ] { + let wire = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\n{headers}Content-Length: \ + 2\r\nConnection: close\r\n\r\n{{}}" + ); + let response = gateway_exchange(gateway.clone(), wire).await?; + anyhow::ensure!( + response.starts_with("HTTP/1.1 400"), + "wrong route response: {response}" + ); + let (_, body) = response.split_once("\r\n\r\n").context("error body")?; + let error: serde_json::Value = serde_json::from_str(body)?; + assert_eq!(error["__type"], "ValidationException"); + assert_eq!(error["message"], "Invalid HTTP request envelope"); + } + Result::<()>::Ok(()) + } + .await; + workflows + .as_mut() + .context("workflow owner")? + .shutdown(Instant::now() + Duration::from_secs(5)) + .await?; + workers + .shutdown(Instant::now() + Duration::from_secs(30)) + .await?; + result + } + + #[cfg(feature = "stepfunctions")] + #[tokio::test] + async fn test_should_preserve_modeled_timeout_envelope_through_actual_gateway() -> Result<()> { + let runtime = WorkflowRuntime::new( + TenantScope::default(), + settings::RuntimeBudgets::default(), + #[cfg(feature = "lambda")] + None, + )?; + let scope = RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into())?; + let total = Duration::from_secs(1); + let body = BodyBudget::new(2 * 1024 * 1024, total, total)?; + let config = StepFunctionsHttpConfig::new(scope, true, None)?.with_budgets(body, total)?; + let router = service::StepFunctionsServiceRouter::new(StepFunctionsHttpService::new( + runtime.api, + config, + )); + let gateway = GatewayService::new(vec![Box::new(router)]) + .with_request_seconds(NonZeroU64::new(1).context("request duration")?); + let mut owner = runtime.owner; + let response = gateway_exchange( + gateway, + "POST / HTTP/1.1\r\nHost: localhost\r\nX-Amz-Target: \ + AWSStepFunctions.ListStateMachines\r\nContent-Type: \ + application/x-amz-json-1.0\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{" + .to_owned(), + ) + .await; + owner + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + let response = response?; + assert!(response.starts_with("HTTP/1.1 408"), "{response}"); + assert!( + response.to_ascii_lowercase().contains("x-amzn-requestid:"), + "{response}" + ); + let (_, body) = response.split_once("\r\n\r\n").context("timeout body")?; + let error: serde_json::Value = serde_json::from_str(body)?; + assert_eq!(error["__type"], "RequestTimeoutException"); + Ok(()) + } + + #[cfg(feature = "stepfunctions")] + #[tokio::test] + async fn test_should_route_actual_socket_requests_to_real_workflow_and_capabilities() + -> Result<()> { + let runtime = build_services(|name| name == "stepfunctions")?; + let Runtime { + services, + workers, + mut workflows, + .. + } = runtime; + let gateway = GatewayService::new(services).with_workers(Arc::new(workers)); + let result = async { + let response = gateway_exchange( + gateway.clone(), + "POST / HTTP/1.1\r\nHost: localhost\r\nX-Amz-Target: \ + AWSStepFunctions.ListStateMachines\r\nContent-Type: \ + application/x-amz-json-1.0\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}" + .to_owned(), + ) + .await?; + anyhow::ensure!( + response.starts_with("HTTP/1.1 200"), + "workflow route did not succeed" + ); + anyhow::ensure!( + response.contains("\"stateMachines\":[]"), + "real empty registry response missing" + ); + let response = gateway_exchange( + gateway, + "GET /_rustack/capabilities HTTP/1.1\r\nHost: localhost\r\nConnection: \ + close\r\n\r\n" + .to_owned(), + ) + .await?; + let (_, body) = response.split_once("\r\n\r\n").context("capability body")?; + let capabilities: serde_json::Value = serde_json::from_str(body)?; + anyhow::ensure!(capabilities["ready"] == true, "workflow gateway not ready"); + anyhow::ensure!( + capabilities["workflows"]["stateTypes"] + .as_array() + .is_some_and(|states| states.len() == 8 + && states.contains(&serde_json::json!("Parallel")) + && states.contains(&serde_json::json!("Map"))) + && capabilities["workflows"]["limits"]["frames"] == 4096, + "installed structured profile or frame limit not reported" + ); + anyhow::ensure!( + capabilities["workflows"]["lambdaRequestResponse"] == false, + "pure service advertised Lambda" + ); + anyhow::ensure!( + capabilities["workflows"]["snapshot"] == "resources-and-history", + "snapshot coverage was overstated" + ); + Result::<()>::Ok(()) + } + .await; + let cleanup = workflows + .as_mut() + .context("workflow owner")? + .quiesce(Instant::now() + Duration::from_secs(5)) + .await; + cleanup?; + result + } + + #[cfg(feature = "stepfunctions")] + #[tokio::test] + async fn test_should_register_real_workflow_service_and_shared_health() -> Result<()> { + let mut runtime = build_services(|name| name == "stepfunctions" || name == "s3")?; + let workflow = runtime + .services + .iter() + .position(|service| service.name() == "stepfunctions") + .context("workflow router")?; + if let Some(s3) = runtime + .services + .iter() + .position(|service| service.name() == "s3") + { + assert!(workflow < s3); + } + assert!(is_compiled_in("stepfunctions")); + assert!( + parse_services_value("") + .iter() + .any(|name| name == "stepfunctions") + ); + assert!(runtime.workers.diagnostics().ready); + assert!(runtime.workers.workflow_capabilities().is_some()); + runtime + .workflows + .as_mut() + .context("workflow owner")? + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + assert!(!runtime.workers.diagnostics().ready); + Ok(()) + } + #[tokio::test] async fn test_should_parse_health_status_not_substrings() -> Result<()> { let valid = r#"{"ready":true,"services":{"s3":"running"}}"#; diff --git a/apps/rustack/src/runtime.rs b/apps/rustack/src/runtime.rs index 1b251c9f..45370c84 100644 --- a/apps/rustack/src/runtime.rs +++ b/apps/rustack/src/runtime.rs @@ -1,13 +1,38 @@ //! Runtime worker ownership independent of snapshot participation. -use std::{collections::BTreeMap, fmt, sync::Arc, time::Duration}; +#[cfg(not(all( + any(feature = "events", feature = "sns", feature = "dynamodb"), + any(feature = "lambda", feature = "sqs") +)))] +use std::future::{Ready, ready}; +#[cfg(any( + feature = "dynamodb", + feature = "events", + feature = "sns", + feature = "lambda", + feature = "sqs" +))] +use std::sync::Arc; +use std::{collections::BTreeMap, fmt}; -use anyhow::{Context, Result}; +#[cfg(any( + feature = "events", + feature = "sns", + feature = "dynamodb", + feature = "sqs" +))] +use anyhow::Context; +use anyhow::Result; +#[cfg(feature = "lambda")] +use rustack_lambda_core::diagnostics::InvocationWorkStatus; use serde::Serialize; +use tokio::runtime::Handle; /// Handles retained until quiescence, snapshot publication, and final shutdown. #[derive(Default)] pub(crate) struct RuntimeWorkers { + #[cfg(feature = "stepfunctions")] + pub(crate) workflows: Option, #[cfg(feature = "dynamodb")] pub(crate) dynamodb: Option>, #[cfg(feature = "events")] @@ -16,6 +41,8 @@ pub(crate) struct RuntimeWorkers { pub(crate) sns: Option>, #[cfg(feature = "lambda")] pub(crate) lambda: Option>, + #[cfg(all(feature = "lambda", unix))] + pub(crate) docker: Option, #[cfg(feature = "sqs")] pub(crate) sqs: Option>, } @@ -40,6 +67,14 @@ pub(crate) struct DeliverySummary { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct WorkerDiagnostics { + #[serde(skip_serializing_if = "Option::is_none")] + runtime_tasks: Option, + #[cfg(feature = "lambda")] + #[serde(skip_serializing_if = "Option::is_none")] + lambda_admission: Option, + #[cfg(all(feature = "lambda", unix))] + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) docker: Option, pub(crate) ready: bool, pub(crate) services: BTreeMap<&'static str, &'static str>, delivery: BTreeMap<&'static str, DeliverySummary>, @@ -49,6 +84,14 @@ impl RuntimeWorkers { /// Report live supervisor state, independently of historical delivery failures. pub(crate) fn diagnostics(&self) -> WorkerDiagnostics { let readiness: &[Option<(&'static str, bool)>] = &[ + #[cfg(feature = "lambda")] + self.lambda + .as_ref() + .map(|provider| ("lambda", provider.is_ready())), + #[cfg(feature = "stepfunctions")] + self.workflows + .as_ref() + .map(|workflow| ("stepfunctions", workflow.is_ready())), #[cfg(feature = "events")] self.events .as_ref() @@ -95,6 +138,16 @@ impl RuntimeWorkers { }), ]; WorkerDiagnostics { + runtime_tasks: Handle::try_current() + .ok() + .map(|handle| handle.metrics().num_alive_tasks()), + #[cfg(feature = "lambda")] + lambda_admission: self.lambda.as_ref().map(|provider| provider.work_status()), + #[cfg(all(feature = "lambda", unix))] + docker: self + .docker + .as_ref() + .map(rustack_lambda_core::docker::DockerRuntimeStatus::capabilities), ready: readiness.iter().flatten().all(|(_, ready)| *ready), services: readiness .iter() @@ -105,9 +158,17 @@ impl RuntimeWorkers { } } - /// Stop new cross-service work before collecting the supported snapshot state. - pub(crate) async fn quiesce(&self, remaining: Duration) -> Result<()> { - let deadline = tokio::time::Instant::now() + remaining; + #[cfg(feature = "stepfunctions")] + pub(crate) fn workflow_capabilities(&self) -> Option { + self.workflows + .as_ref() + .map(crate::workflows::WorkflowStatus::capabilities) + } + + /// Quiesce producers before the unique workflow supervisor drains. + #[cfg(any(feature = "events", feature = "sns", feature = "dynamodb"))] + pub(crate) async fn quiesce_producers(&self, deadline: tokio::time::Instant) -> Result<()> { + check_deadline(deadline)?; #[cfg(feature = "events")] if let Some(events) = &self.events { tokio::time::timeout_at(deadline, events.quiesce()) @@ -126,6 +187,13 @@ impl RuntimeWorkers { .await .context("DynamoDB quiesce deadline exceeded")??; } + Ok(()) + } + + /// Downstream consumers stay available until workflow cleanup joins. + #[cfg(any(feature = "lambda", feature = "sqs"))] + pub(crate) async fn quiesce_consumers(&self, deadline: tokio::time::Instant) -> Result<()> { + check_deadline(deadline)?; #[cfg(feature = "lambda")] if let Some(lambda) = &self.lambda { lambda @@ -141,11 +209,22 @@ impl RuntimeWorkers { Ok(()) } + #[cfg(not(any(feature = "events", feature = "sns", feature = "dynamodb")))] + pub(crate) fn quiesce_producers(&self, deadline: tokio::time::Instant) -> Ready> { + tracing::debug!(workers = ?self.diagnostics(), "No producer services require quiescence"); + ready(check_deadline(deadline)) + } + #[cfg(not(any(feature = "lambda", feature = "sqs")))] + pub(crate) fn quiesce_consumers(&self, deadline: tokio::time::Instant) -> Ready> { + tracing::debug!(workers = ?self.diagnostics(), "No consumer services require quiescence"); + ready(check_deadline(deadline)) + } + /// Final resource destruction. Call only after the snapshot decision. - pub(crate) async fn shutdown(&self) -> Result<()> { + pub(crate) async fn shutdown(&self, deadline: tokio::time::Instant) -> Result<()> { // After the snapshot decision, stop independent resource owners together: // a failed queue worker must not delay killing Lambda child processes. - let (events_result, (), (), ()) = tokio::join!( + let (events_result, (), lambda_result, ()) = tokio::join!( async { #[cfg(feature = "events")] if let Some(events) = &self.events { @@ -162,8 +241,9 @@ impl RuntimeWorkers { async { #[cfg(feature = "lambda")] if let Some(lambda) = &self.lambda { - lambda.shutdown().await; + lambda.shutdown_before(deadline).await?; } + Result::<()>::Ok(()) }, async { #[cfg(feature = "sqs")] @@ -172,6 +252,48 @@ impl RuntimeWorkers { } }, ); - events_result + events_result?; + lambda_result?; + check_deadline(deadline) + } +} + +fn check_deadline(deadline: tokio::time::Instant) -> Result<()> { + anyhow::ensure!( + tokio::time::Instant::now() < deadline, + "runtime quiescence deadline exceeded" + ); + Ok(()) +} + +#[cfg(test)] +mod diagnostic_tests { + #[cfg(feature = "lambda")] + use rustack_lambda_core::{config::LambdaConfig, provider::RustackLambda}; + + use super::*; + + #[test] + fn test_should_omit_admission_counts_without_an_installed_lambda_provider() { + let value = serde_json::to_value(RuntimeWorkers::default().diagnostics()).unwrap(); + assert!(value.get("lambdaAdmission").is_none()); + } + + #[cfg(feature = "lambda")] + #[tokio::test] + async fn test_should_project_the_actual_installed_lambda_admission_without_starting_work() { + let provider = Arc::new(RustackLambda::new(LambdaConfig::default())); + let workers = RuntimeWorkers { + lambda: Some(provider.clone()), + ..RuntimeWorkers::default() + }; + let value = serde_json::to_value(workers.diagnostics()).unwrap(); + assert_eq!( + value.get("lambdaAdmission"), + Some(&serde_json::json!({"acceptedInvocations":0, + "activeExecutions":0,"accepting":true,"limits":{"acceptedInvocations":128, + "activeExecutions":32,"defaultFunctionExecutions":8}})) + ); + assert_eq!(provider.work_status().active_executions, 0); } } diff --git a/apps/rustack/src/service.rs b/apps/rustack/src/service.rs index ed7e62ed..23f40727 100644 --- a/apps/rustack/src/service.rs +++ b/apps/rustack/src/service.rs @@ -1,6 +1,6 @@ //! Service router abstraction for the gateway. //! -//! Each AWS service (S3, DynamoDB, etc.) implements [`ServiceRouter`] to +//! Each AWS service (S3, `DynamoDB`, etc.) implements [`ServiceRouter`] to //! declare how it matches incoming requests and handles them. The gateway //! holds a list of routers and dispatches to the first match. //! @@ -9,6 +9,15 @@ use std::{convert::Infallible, future::Future, io, pin::Pin}; use bytes::Bytes; +#[cfg(any( + feature = "sns", + feature = "ses", + feature = "cloudwatch", + feature = "iam", + feature = "sts", + feature = "cloudfront" +))] +use http::HeaderMap; use http_body_util::{BodyExt, Full, combinators::BoxBody}; use hyper::body::Incoming; @@ -66,6 +75,13 @@ pub trait ServiceRouter: Send + Sync { /// Returns `true` if this router should handle the given request. fn matches(&self, req: &http::Request) -> bool; + /// Whether this bounded transport owns its complete modeled request timeout. + /// + /// Other services keep the gateway's generic deadline. + fn owns_request_deadline(&self) -> bool { + false + } + /// Handle the request, producing a response with a type-erased body. fn call( &self, @@ -73,6 +89,20 @@ pub trait ServiceRouter: Send + Sync { ) -> Pin, Infallible>> + Send>>; } +#[cfg(any( + feature = "sns", + feature = "ses", + feature = "cloudwatch", + feature = "iam", + feature = "sts", + feature = "cloudfront" +))] +fn extract_sigv4_service(headers: &HeaderMap) -> Option<&str> { + let auth = headers.get("authorization")?.to_str().ok()?; + let (_, rest) = auth.split_once("Credential=")?; + rest.split(',').next()?.split('/').nth(3) +} + // --------------------------------------------------------------------------- // S3 // --------------------------------------------------------------------------- @@ -143,7 +173,7 @@ mod dynamodb_router { use super::{GatewayBody, ServiceRouter}; - /// Routes requests to the DynamoDB service. + /// Routes requests to the `DynamoDB` service. /// /// Matches requests whose `X-Amz-Target` header starts with `DynamoDB_`. pub struct DynamoDBServiceRouter { @@ -202,7 +232,7 @@ mod dynamodbstreams_router { use super::{GatewayBody, ServiceRouter}; - /// Routes requests to the DynamoDB Streams service. + /// Routes requests to the `DynamoDB` Streams service. /// /// Matches requests whose `X-Amz-Target` header starts with `DynamoDBStreams_`. pub struct DynamoDBStreamsServiceRouter { @@ -371,29 +401,12 @@ mod sns_router { use hyper::{body::Incoming, service::Service}; use rustack_sns_http::{dispatch::SnsHandler, service::SnsHttpService}; - use super::{GatewayBody, ServiceRouter}; - - /// Extract the SigV4 service name from the Authorization header. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let credential = &auth[credential_start..credential_end]; - // Format: AKID/date/region/service/aws4_request - let parts: Vec<&str> = credential.split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; /// Routes requests to the SNS service. /// /// Matches `POST /` requests with `Content-Type: application/x-www-form-urlencoded` - /// where the SigV4 signing service is `sns`. SNS uses the `awsQuery` protocol + /// where the `SigV4` signing service is `sns`. SNS uses the `awsQuery` protocol /// where the operation is determined by the `Action=` parameter in the form body. pub struct SnsServiceRouter { inner: SnsHttpService, @@ -411,8 +424,8 @@ mod sns_router { "sns" } - /// SNS matches form-urlencoded POST requests signed with `sns` SigV4 - /// service name. All SNS operations require SigV4 authentication. + /// SNS matches form-urlencoded POST requests signed with `sns` `SigV4` + /// service name. All SNS operations require `SigV4` authentication. fn matches(&self, req: &http::Request) -> bool { if *req.method() != http::Method::POST { return false; @@ -463,57 +476,18 @@ mod lambda_router { /// Lambda API uses multiple date-versioned path prefixes. /// The SDK may use different dates than the spec documents. fn is_lambda_path(path: &str) -> bool { - // Function CRUD and invoke paths. - if path.contains("/functions") { - // Match any date prefix for /YYYY-MM-DD/functions - if let Some(rest) = path.strip_prefix('/') { - let parts: Vec<&str> = rest.splitn(2, '/').collect(); - if parts.len() == 2 && parts[0].len() == 10 && parts[1].starts_with("functions") { - return true; - } - } - } - // Tags paths (e.g., /2017-03-31/tags/{arn}). - if path.contains("/tags/") { - if let Some(rest) = path.strip_prefix('/') { - let parts: Vec<&str> = rest.splitn(2, '/').collect(); - if parts.len() == 2 && parts[0].len() == 10 && parts[1].starts_with("tags/") { - return true; - } - } - } - // Account settings paths. - if path.contains("/account-settings") { - if let Some(rest) = path.strip_prefix('/') { - let parts: Vec<&str> = rest.splitn(2, '/').collect(); - if parts.len() == 2 && parts[0].len() == 10 && parts[1] == "account-settings" { - return true; - } - } - } - // Layer paths (e.g., /2018-10-31/layers/{name}/versions). - if path.contains("/layers") { - if let Some(rest) = path.strip_prefix('/') { - let parts: Vec<&str> = rest.splitn(2, '/').collect(); - if parts.len() == 2 && parts[0].len() == 10 && parts[1].starts_with("layers") { - return true; - } - } - } - // Event source mapping paths (e.g., /2015-03-31/event-source-mappings/). - if path.contains("/event-source-mappings") { - if let Some(rest) = path.strip_prefix('/') { - let parts: Vec<&str> = rest.splitn(2, '/').collect(); - if parts.len() == 2 - && parts[0].len() == 10 - && parts[1].starts_with("event-source-mappings") - { - return true; - } - } - } - // Function URL invocation paths. path.starts_with("/lambda-url/") + || path + .strip_prefix('/') + .and_then(|rest| rest.split_once('/')) + .is_some_and(|(date, resource)| { + date.len() == 10 + && (resource.starts_with("functions") + || resource.starts_with("tags/") + || resource == "account-settings" + || resource.starts_with("layers") + || resource.starts_with("event-source-mappings")) + }) } /// Routes requests to the Lambda service. @@ -553,6 +527,36 @@ mod lambda_router { }) } } + #[cfg(test)] + mod tests { + use super::is_lambda_path; + + #[test] + fn test_should_preserve_lambda_prefix_classification_without_indexing() { + for path in [ + "/2015-03-31/functions/Fn/invocations", + "/2017-03-31/tags/arn", + "/2016-08-19/account-settings", + "/2018-10-31/layers/L/versions", + "/2015-03-31/event-source-mappings/", + "/lambda-url/Fn/", + ] { + assert!(is_lambda_path(path), "{path}"); + } + for path in [ + "", + "/", + "/functions", + "/2015-03-3/functions", + "/2015-03-311/functions", + "/2015-03-31/tags", + "/2015-03-31/account-settings/other", + "/bucket/key", + ] { + assert!(!is_lambda_path(path), "{path}"); + } + } + } } #[cfg(feature = "lambda")] @@ -572,7 +576,7 @@ mod events_router { use super::{GatewayBody, ServiceRouter}; - /// Routes requests to the EventBridge service. + /// Routes requests to the `EventBridge` service. /// /// Matches requests whose `X-Amz-Target` header starts with `AWSEvents.`. pub struct EventsServiceRouter { @@ -629,7 +633,7 @@ mod logs_router { use super::{GatewayBody, ServiceRouter}; - /// Routes requests to the CloudWatch Logs service. + /// Routes requests to the `CloudWatch` Logs service. /// /// Matches requests whose `X-Amz-Target` header starts with `Logs_20140328.`. pub struct LogsServiceRouter { @@ -857,30 +861,11 @@ mod ses_router { use hyper::{body::Incoming, service::Service}; use rustack_ses_http::{dispatch::SesHandler, service::SesHttpService, v2::SesV2HttpService}; - use super::{GatewayBody, ServiceRouter}; - - /// Extract the SigV4 service name from the Authorization header. - /// - /// Parses `Credential=AKID/date/region/SERVICE/aws4_request` and returns SERVICE. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let credential = &auth[credential_start..credential_end]; - // Format: AKID/date/region/service/aws4_request - let parts: Vec<&str> = credential.split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; /// Routes requests to the SES service. /// - /// Matches SES v1 (`awsQuery` via SigV4 service=`email`) and SES v2 + /// Matches SES v1 (`awsQuery` via `SigV4` service=`email`) and SES v2 /// (`restJson1` via `/v2/email/` path prefix). Also handles the /// `/_aws/ses` retrospection endpoint. pub struct SesServiceRouter { @@ -906,7 +891,7 @@ mod ses_router { /// SES matches in three ways: /// 1. SES v2: path starts with `/v2/email/` /// 2. SES retrospection: path starts with `/_aws/ses` - /// 3. SES v1: form-urlencoded POST with SigV4 service=`email` + /// 3. SES v1: form-urlencoded POST with `SigV4` service=`email` fn matches(&self, req: &http::Request) -> bool { let path = req.uri().path(); @@ -1218,29 +1203,12 @@ mod cloudwatch_router { use hyper::{body::Incoming, service::Service}; use rustack_cloudwatch_http::{dispatch::CloudWatchHandler, service::CloudWatchHttpService}; - use super::{GatewayBody, ServiceRouter}; - - /// Extract the SigV4 service name from the Authorization header. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let credential = &auth[credential_start..credential_end]; - // Format: AKID/date/region/service/aws4_request - let parts: Vec<&str> = credential.split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; - /// Routes requests to the CloudWatch Metrics service. + /// Routes requests to the `CloudWatch` Metrics service. /// /// Matches awsQuery requests (`POST /` with form-urlencoded body signed - /// with `monitoring` SigV4 service) and rpcv2Cbor requests (POST to + /// with `monitoring` `SigV4` service) and rpcv2Cbor requests (POST to /// `/service/GraniteServiceVersion20100801/operation/{Op}`). pub struct CloudWatchServiceRouter { inner: CloudWatchHttpService, @@ -1258,11 +1226,11 @@ mod cloudwatch_router { "cloudwatch" } - /// CloudWatch Metrics matches in three ways: - /// 1. awsQuery: form-urlencoded POST signed with `monitoring` SigV4 service. + /// `CloudWatch` Metrics matches in three ways: + /// 1. awsQuery: form-urlencoded POST signed with `monitoring` `SigV4` service. /// 2. rpcv2Cbor path: POST to `/service/GraniteServiceVersion20100801/...`. /// 3. rpcv2Cbor header: POST with `smithy-protocol: rpc-v2-cbor` signed with `monitoring` - /// SigV4 service (AWS SDK v1.108+). + /// `SigV4` service (AWS SDK v1.108+). fn matches(&self, req: &http::Request) -> bool { if *req.method() != http::Method::POST { return false; @@ -1338,29 +1306,12 @@ mod iam_router { use hyper::{body::Incoming, service::Service}; use rustack_iam_http::{dispatch::IamHandler, service::IamHttpService}; - use super::{GatewayBody, ServiceRouter}; - - /// Extract the SigV4 service name from the Authorization header. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let credential = &auth[credential_start..credential_end]; - // Format: AKID/date/region/service/aws4_request - let parts: Vec<&str> = credential.split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; /// Routes requests to the IAM service. /// /// Matches `POST /` requests with `Content-Type: application/x-www-form-urlencoded` - /// where the SigV4 signing service is `iam`. + /// where the `SigV4` signing service is `iam`. pub struct IamServiceRouter { inner: IamHttpService, } @@ -1378,7 +1329,7 @@ mod iam_router { } /// IAM matches form-urlencoded POST requests signed - /// with the `iam` SigV4 service name. + /// with the `iam` `SigV4` service name. fn matches(&self, req: &http::Request) -> bool { if *req.method() != http::Method::POST { return false; @@ -1424,29 +1375,12 @@ mod sts_router { use hyper::{body::Incoming, service::Service}; use rustack_sts_http::{dispatch::StsHandler, service::StsHttpService}; - use super::{GatewayBody, ServiceRouter}; - - /// Extract the SigV4 service name from the Authorization header. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let credential = &auth[credential_start..credential_end]; - // Format: AKID/date/region/service/aws4_request - let parts: Vec<&str> = credential.split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; /// Routes requests to the STS service. /// /// Matches `POST /` requests with `Content-Type: application/x-www-form-urlencoded` - /// where the SigV4 signing service is `sts`. + /// where the `SigV4` signing service is `sts`. pub struct StsServiceRouter { inner: StsHttpService, } @@ -1463,10 +1397,10 @@ mod sts_router { "sts" } - /// STS matches form-urlencoded POST requests signed with `sts` SigV4 + /// STS matches form-urlencoded POST requests signed with `sts` `SigV4` /// service name, or unsigned form-urlencoded requests (for federation /// operations like `AssumeRoleWithWebIdentity` and `AssumeRoleWithSAML` - /// which do not require SigV4 signing). + /// which do not require `SigV4` signing). fn matches(&self, req: &http::Request) -> bool { if *req.method() != http::Method::POST { return false; @@ -1521,26 +1455,11 @@ mod cloudfront_router { use hyper::{body::Incoming, service::Service}; use rustack_cloudfront_http::{dispatch::CloudFrontHandler, service::CloudFrontHttpService}; - use super::{GatewayBody, ServiceRouter}; + use super::{GatewayBody, ServiceRouter, extract_sigv4_service}; - /// Extract SigV4 service name. - fn extract_sigv4_service(headers: &http::HeaderMap) -> Option<&str> { - let auth = headers.get("authorization")?.to_str().ok()?; - let credential_start = auth.find("Credential=")? + "Credential=".len(); - let credential_end = auth[credential_start..] - .find(',') - .map_or(auth.len(), |i| credential_start + i); - let parts: Vec<&str> = auth[credential_start..credential_end].split('/').collect(); - if parts.len() >= 4 { - Some(parts[3]) - } else { - None - } - } - - /// Routes requests to the CloudFront management service. + /// Routes requests to the `CloudFront` management service. /// - /// Matches `/2020-05-31/` path prefix OR SigV4 signing service `cloudfront`. + /// Matches `/2020-05-31/` path prefix OR `SigV4` signing service `cloudfront`. pub struct CloudFrontServiceRouter { inner: CloudFrontHttpService, } @@ -1598,7 +1517,7 @@ mod cloudfront_dataplane_router { use super::{GatewayBody, ServiceRouter}; - /// Routes requests to the CloudFront pass-through data plane. + /// Routes requests to the `CloudFront` pass-through data plane. pub struct CloudFrontDataPlaneRouter { plane: DataPlane, } @@ -1666,3 +1585,94 @@ mod cloudfront_dataplane_router { #[cfg(feature = "cloudfront-dataplane")] pub use cloudfront_dataplane_router::CloudFrontDataPlaneRouter; + +#[cfg(feature = "stepfunctions")] +mod stepfunctions_router { + use std::{convert::Infallible, future::Future, pin::Pin}; + + use http_body_util::BodyExt; + use hyper::body::Incoming; + use rustack_stepfunctions_http::{StepFunctionsHttpService, claims_request}; + + use super::{GatewayBody, ServiceRouter}; + + /// Claims the exact target family, including unsupported operations. + pub struct StepFunctionsServiceRouter { + inner: StepFunctionsHttpService, + } + impl StepFunctionsServiceRouter { + pub fn new(inner: StepFunctionsHttpService) -> Self { + Self { inner } + } + } + impl ServiceRouter for StepFunctionsServiceRouter { + fn name(&self) -> &'static str { + "stepfunctions" + } + fn matches(&self, request: &http::Request) -> bool { + claims_request(request.headers()) + } + fn owns_request_deadline(&self) -> bool { + true + } + fn call( + &self, + request: http::Request, + ) -> Pin, Infallible>> + Send>> + { + let service = self.inner.clone(); + Box::pin(async move { + Ok(service + .handle(request) + .await + .map(|body| body.map_err(|never: Infallible| match never {}).boxed())) + }) + } + } +} +#[cfg(feature = "stepfunctions")] +pub use stepfunctions_router::StepFunctionsServiceRouter; + +#[cfg(all( + test, + any( + feature = "sns", + feature = "ses", + feature = "cloudwatch", + feature = "iam", + feature = "sts", + feature = "cloudfront" + ) +))] +mod routing_boundary_tests { + use http::{HeaderMap, HeaderValue}; + + use super::extract_sigv4_service; + + #[test] + fn test_should_classify_credential_service_without_slicing_or_allocating() { + for (header, expected) in [ + ( + "AWS4-HMAC-SHA256 Credential=key/date/region/ses/aws4_request, SignedHeaders=host", + Some("ses"), + ), + ( + "Credential=key/date/region/states/aws4_request", + Some("states"), + ), + ("Credential=key/date/region/sts", Some("sts")), + ("Credential=key/date/region", None), + ( + "Credential=, Credential=key/date/region/iam/aws4_request", + None, + ), + ("Credential", None), + ("", None), + ] { + let mut headers = HeaderMap::new(); + headers.insert("authorization", HeaderValue::from_static(header)); + assert_eq!(extract_sigv4_service(&headers), expected); + } + assert_eq!(extract_sigv4_service(&HeaderMap::new()), None); + } +} diff --git a/apps/rustack/src/snapshot.rs b/apps/rustack/src/snapshot.rs index 02494284..acb3df7e 100644 --- a/apps/rustack/src/snapshot.rs +++ b/apps/rustack/src/snapshot.rs @@ -1,8 +1,10 @@ //! Runtime snapshot loading and saving for the Rustack gateway. +//! `SnapshotService` uses async-trait because the registry stores object-safe dyn services. use std::{ collections::BTreeMap, io::ErrorKind, + mem, path::{Path, PathBuf}, sync::Arc, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, @@ -10,9 +12,9 @@ use std::{ use anyhow::{Context, Result, bail}; use archive::{ - ArchiveKind, ArchiveSection, ArchiveStats, SECTION_MANIFEST_CBOR, SECTION_STATE_CBOR, - from_cbor, get_required_section, pack_data_archive, read_archive, to_cbor, unpack_data_archive, - validated_relative_path, write_archive, + ArchiveKind, ArchiveSection, ArchiveStats, BudgetLimit, SECTION_MANIFEST_CBOR, + SECTION_STATE_CBOR, from_cbor, get_required_section, pack_data_archive, read_archive, + read_archive_limited, to_cbor, unpack_data_archive, validated_relative_path, write_archive, }; use async_trait::async_trait; #[cfg(feature = "apigatewayv2")] @@ -35,11 +37,36 @@ use rustack_s3_core::{RustackS3, snapshot::S3Snapshot}; use rustack_sqs_core::provider::{RustackSqs, SqsSnapshot}; #[cfg(feature = "ssm")] use rustack_ssm_core::{provider::RustackSsm, storage::ParameterStoreSnapshot}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use tokio::{fs, io::AsyncWriteExt as _, sync::Semaphore, task::JoinSet}; +#[cfg(any( + feature = "stepfunctions", + feature = "s3", + feature = "dynamodb", + feature = "dynamodbstreams", + feature = "sqs", + feature = "ssm", + feature = "iam", + feature = "lambda", + feature = "apigatewayv2", + feature = "cloudfront", + feature = "cloudfront-dataplane" +))] +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::{ + fs, io::AsyncWriteExt as _, sync::Semaphore, task::JoinSet, time::Instant as Deadline, +}; use tracing::{info, warn}; mod archive; +mod lease; +mod session; +mod task; +pub(crate) use session::SnapshotSession; +mod worker; +pub(crate) use task::SnapshotTask; +use worker::WorkerFailure; +#[cfg(feature = "stepfunctions")] +pub(crate) mod workflows; const SNAPSHOT_SCHEMA_VERSION: u32 = 2; const SNAPSHOT_ROOT_ENV: &str = "RUSTACK_SNAPSHOT_DIR"; @@ -52,14 +79,20 @@ const SERVICES_DIR: &str = "services"; const STAGING_DIR: &str = ".staging"; const LOAD_STAGING_PREFIX: &str = ".load"; const SNAPSHOT_SERVICE_PARALLELISM: usize = 4; +type ArchiveLimits = BTreeMap<&'static str, usize>; /// Registry of services that support runtime snapshots. -#[derive(Default)] +#[derive(Default, Clone)] pub(crate) struct RuntimeProviders { services: Vec>, } impl RuntimeProviders { + #[cfg(feature = "stepfunctions")] + pub(crate) fn register_workflows(&mut self, archive: workflows::WorkflowArchive) { + self.register(archive); + } + #[cfg(feature = "s3")] pub(crate) fn register_s3(&mut self, provider: Arc) { self.register(S3SnapshotService { provider }); @@ -110,6 +143,19 @@ impl RuntimeProviders { self.register(CloudFrontCacheSnapshotService { plane }); } + #[cfg(any( + feature = "stepfunctions", + feature = "s3", + feature = "dynamodb", + feature = "dynamodbstreams", + feature = "sqs", + feature = "ssm", + feature = "iam", + feature = "lambda", + feature = "apigatewayv2", + feature = "cloudfront", + feature = "cloudfront-dataplane" + ))] fn register(&mut self, service: T) where T: SnapshotService + 'static, @@ -130,6 +176,16 @@ impl RuntimeProviders { .collect() } + fn limits(&self) -> ArchiveLimits { + self.services + .iter() + .filter_map(|service| { + service + .payload_limit() + .map(|limit| (service.service_name(), limit)) + }) + .collect() + } fn services(&self) -> &[Arc] { &self.services } @@ -141,6 +197,7 @@ pub(crate) fn snapshot_coverage(service: &str) -> &'static str { "s3" | "dynamodb" => "resources-and-data", "dynamodbstreams" => "resources-and-records", "lambda" => "resources-and-code", + "stepfunctions" => "resources-and-history", "cloudfront-cache" | "cloudfront-dataplane" => "cache-data", "sqs" | "ssm" | "iam" | "apigatewayv2" | "cloudfront" => "resources-only", _ => "unsupported", @@ -155,9 +212,27 @@ trait SnapshotService: Send + Sync { SnapshotKind::Resource } + fn payload_limit(&self) -> Option { + None + } + fn data_file(&self) -> Option<&'static str> { + None + } + async fn save_meta(&self, data_staging_dir: &Path) -> Result>; - async fn load_meta(&self, state_cbor: &[u8], data_staging_dir: &Path) -> Result<()>; + async fn load_meta(&self, state_cbor: Vec, data_staging_dir: &Path) -> Result<()>; + + async fn load_meta_before( + &self, + state_cbor: Vec, + data_staging_dir: &Path, + deadline: Deadline, + ) -> Result<()> { + check_load_deadline(deadline)?; + self.load_meta(state_cbor, data_staging_dir).await?; + check_load_deadline(deadline) + } } #[cfg(feature = "s3")] @@ -181,8 +256,8 @@ impl SnapshotService for S3SnapshotService { encode_state(&snapshot) } - async fn load_meta(&self, state_cbor: &[u8], data_staging_dir: &Path) -> Result<()> { - let snapshot: S3Snapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, data_staging_dir: &Path) -> Result<()> { + let snapshot: S3Snapshot = decode_state(&state_cbor)?; self.provider .import_snapshot(snapshot, data_staging_dir) .await?; @@ -210,8 +285,8 @@ impl SnapshotService for DynamoDBSnapshotService { encode_state(&self.provider.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: DynamoDBSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: DynamoDBSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot)?; Ok(()) } @@ -233,8 +308,8 @@ impl SnapshotService for DynamoDBStreamsSnapshotService { encode_state(&self.store.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: StreamStoreSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: StreamStoreSnapshot = decode_state(&state_cbor)?; self.store.import_snapshot(snapshot); Ok(()) } @@ -256,8 +331,8 @@ impl SnapshotService for SqsSnapshotService { encode_state(&self.provider.export_snapshot().await?) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: SqsSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: SqsSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot).await?; Ok(()) } @@ -279,8 +354,8 @@ impl SnapshotService for SsmSnapshotService { encode_state(&self.provider.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: ParameterStoreSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: ParameterStoreSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot); Ok(()) } @@ -302,8 +377,8 @@ impl SnapshotService for IamSnapshotService { encode_state(&self.provider.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: IamStoreSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: IamStoreSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot); Ok(()) } @@ -321,12 +396,43 @@ impl SnapshotService for LambdaSnapshotService { "lambda" } + fn payload_limit(&self) -> Option { + self.provider.archive_payload_limit() + } + + async fn load_meta_before( + &self, + state_cbor: Vec, + _data_staging_dir: &Path, + deadline: Deadline, + ) -> Result<()> { + check_load_deadline(deadline)?; + let snapshot: LambdaSnapshot = match self.provider.docker_archive_profile() { + Some(profile) => archive::from_cbor_lambda(&state_cbor, profile)?, + None => decode_state(&state_cbor)?, + }; + self.provider + .import_snapshot_before(snapshot, deadline) + .await?; + check_load_deadline(deadline) + } + async fn save_meta(&self, _data_staging_dir: &Path) -> Result> { - encode_state(&self.provider.export_snapshot()) + let snapshot = self + .provider + .export_snapshot_before(Deadline::now() + Duration::from_secs(300)) + .await?; + match self.provider.docker_archive_profile() { + Some(profile) => archive::to_cbor_lambda(&snapshot, profile), + None => encode_state(&snapshot), + } } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: LambdaSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: LambdaSnapshot = match self.provider.docker_archive_profile() { + Some(profile) => archive::from_cbor_lambda(&state_cbor, profile)?, + None => decode_state(&state_cbor)?, + }; self.provider.import_snapshot(snapshot).await?; Ok(()) } @@ -348,8 +454,8 @@ impl SnapshotService for ApiGatewayV2SnapshotService { encode_state(&self.provider.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: ApiStoreSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: ApiStoreSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot); Ok(()) } @@ -371,8 +477,8 @@ impl SnapshotService for CloudFrontSnapshotService { encode_state(&self.provider.export_snapshot()) } - async fn load_meta(&self, state_cbor: &[u8], _data_staging_dir: &Path) -> Result<()> { - let snapshot: CloudFrontStoreSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let snapshot: CloudFrontStoreSnapshot = decode_state(&state_cbor)?; self.provider.import_snapshot(snapshot); Ok(()) } @@ -403,8 +509,8 @@ impl SnapshotService for CloudFrontCacheSnapshotService { encode_state(&snapshot) } - async fn load_meta(&self, state_cbor: &[u8], data_staging_dir: &Path) -> Result<()> { - let snapshot: CloudFrontCacheSnapshot = decode_state(state_cbor)?; + async fn load_meta(&self, state_cbor: Vec, data_staging_dir: &Path) -> Result<()> { + let snapshot: CloudFrontCacheSnapshot = decode_state(&state_cbor)?; self.plane .import_cache_snapshot(snapshot, data_staging_dir) .await @@ -427,38 +533,15 @@ pub(crate) struct SnapshotLease { _file: std::fs::File, } -impl SnapshotConfig { - /// Acquire an OS-released exclusive lease before loading or serving a named snapshot. - /// - /// The blocking std file API is required for advisory locking and is isolated - /// inside a blocking worker; tokio offers no equivalent lock surface. - #[allow(clippy::disallowed_types, clippy::disallowed_methods)] - pub(crate) async fn acquire_lease(&self) -> Result { - let root = self.root.clone(); - let name = self.name.as_str().to_owned(); - tokio::task::spawn_blocking(move || { - std::fs::create_dir_all(&root).context("create snapshot root for lease")?; - let lock_path = root.join(format!(".{name}.lock")); - if std::fs::symlink_metadata(&lock_path) - .is_ok_and(|metadata| metadata.file_type().is_symlink()) - { - bail!("snapshot lock path must not be a symbolic link"); - } - let file = std::fs::OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path) - .context("open snapshot lease")?; - file.try_lock() - .context("snapshot is already owned by another runtime")?; - Ok(SnapshotLease { _file: file }) - }) - .await - .context("snapshot lease task failed")? +impl SnapshotLease { + /// Fail closed until process exit when async storage retirement is unproven. + /// There is at most one named lease in a runtime; OS exit releases the lock. + pub(crate) fn quarantine(self: Arc) { + mem::forget(self); } +} +impl SnapshotConfig { /// Build snapshot configuration from a CLI-provided name. /// /// # Errors @@ -479,11 +562,22 @@ impl SnapshotConfig { /// # Errors /// /// Returns an error if the snapshot exists but cannot be parsed or applied. + #[cfg(test)] pub(crate) async fn load(&self, providers: &RuntimeProviders) -> Result<()> { + self.load_before(providers, Deadline::now() + Duration::from_secs(3600)) + .await + } + + pub(crate) async fn load_before( + &self, + providers: &RuntimeProviders, + deadline: Deadline, + ) -> Result<()> { + check_load_deadline(deadline)?; let started = Instant::now(); let dir = self.snapshot_dir(); cleanup_stale_staging(&self.root, self.name.as_str()).await?; - recover_snapshot(&self.root, &dir, self.name.as_str()).await?; + recover_snapshot(&self.root, &dir, self.name.as_str(), &providers.limits()).await?; if !path_exists(&dir).await? { info!(snapshot = %self.name.as_str(), path = %dir.display(), "snapshot not found, starting empty"); return Ok(()); @@ -536,20 +630,31 @@ impl SnapshotConfig { .acquire_owned() .await .context("snapshot load semaphore closed")?; - load_service_snapshot(service, snapshot_dir, staging_root, entry).await + load_service_snapshot(service, snapshot_dir, staging_root, entry, deadline) + .await }); } } let load_result = async { + let mut failure = None; while let Some(result) = join_set.join_next().await { - result.context("snapshot load task failed")??; + if let Err(error) = result + .map_err(WorkerFailure::new) + .context("snapshot load task failed") + .and_then(|value| value) + { + if failure.is_none() { + failure = Some(error); + } + } } - Result::<()>::Ok(()) + failure.map_or(Ok(()), Err) } .await; let cleanup_result = remove_dir_if_exists(&load_staging).await; load_result?; cleanup_result?; + retire_previous_checked(&self.root, self.name.as_str(), Some(deadline)).await?; record_snapshot_timing("load_ms", started.elapsed()).await; info!(snapshot = %self.name.as_str(), path = %dir.display(), "loaded runtime snapshot"); @@ -560,7 +665,7 @@ impl SnapshotConfig { /// /// # Errors /// - /// Returns an error if state export, JSON serialization, or atomic directory + /// Returns an error if state export, CBOR serialization, or atomic directory /// replacement fails. pub(crate) async fn save(&self, providers: &RuntimeProviders, version: &str) -> Result<()> { let started = Instant::now(); @@ -597,14 +702,35 @@ impl SnapshotConfig { save_service_snapshot(service, snapshot_dir).await }); } + let mut failure = None; while let Some(result) = join_set.join_next().await { - let service = result.context("snapshot save task failed")??; - manifest.add_service(service.name, service.manifest); + match result + .map_err(WorkerFailure::new) + .context("snapshot save task failed") + .and_then(|value| value) + { + Ok(service) => manifest.add_service(service.name, service.manifest), + Err(error) => { + if failure.is_none() { + failure = Some(error); + } + } + } + } + if let Some(error) = failure { + return Err(error); } remove_dir_if_exists(&temp.join(STAGING_DIR)).await?; write_manifest(&temp.join(MANIFEST_FILE), &manifest).await?; - replace_directory(&temp, &target, &self.root, self.name.as_str()).await?; + replace_directory( + &temp, + &target, + &self.root, + self.name.as_str(), + &providers.limits(), + ) + .await?; record_snapshot_timing("save_ms", started.elapsed()).await; info!(snapshot = %self.name.as_str(), path = %target.display(), "saved runtime snapshot"); @@ -765,7 +891,9 @@ async fn load_service_snapshot( snapshot_dir: PathBuf, staging_root: PathBuf, manifest: SnapshotServiceManifest, + deadline: Deadline, ) -> Result<()> { + check_load_deadline(deadline)?; let service_name = service.service_name(); let started = Instant::now(); let staging_dir = staging_root.join(service_name); @@ -779,18 +907,39 @@ async fn load_service_snapshot( if let Some(data_file) = manifest.data_file.as_ref() { let data_path = snapshot_child(&snapshot_dir, data_file)?; - unpack_data_archive(&data_path, &staging_dir) - .await - .with_context(|| format!("failed to load snapshot data archive for {service_name}"))?; + unpack_data_archive( + &data_path, + &staging_dir, + service.payload_limit(), + service.data_file(), + ) + .await + .with_context(|| format!("failed to load snapshot data archive for {service_name}"))?; } let meta_path = snapshot_child(&snapshot_dir, &manifest.meta_file)?; - let sections = read_archive(&meta_path, ArchiveKind::ServiceMeta) - .await - .with_context(|| format!("failed to read snapshot metadata archive for {service_name}"))?; - let state_cbor = get_required_section(§ions, SECTION_STATE_CBOR)?; + let sections = read_archive_limited( + &meta_path, + ArchiveKind::ServiceMeta, + service.payload_limit(), + ) + .await + .with_context(|| format!("failed to read snapshot metadata archive for {service_name}"))?; + if service.payload_limit().is_some() + && (sections.len() != 1 + || sections + .first() + .is_none_or(|section| section.kind != SECTION_STATE_CBOR || section.row_count != 1)) + { + bail!("invalid service metadata section layout"); + } + let state_cbor = sections + .into_iter() + .find(|section| section.kind == SECTION_STATE_CBOR) + .ok_or_else(|| anyhow::anyhow!("missing snapshot state section"))? + .bytes; service - .load_meta(state_cbor, &staging_dir) + .load_meta_before(state_cbor, &staging_dir, deadline) .await .with_context(|| format!("failed to load snapshot metadata for {service_name}"))?; @@ -805,6 +954,18 @@ async fn load_service_snapshot( Ok(()) } +#[cfg(any( + feature = "s3", + feature = "dynamodb", + feature = "dynamodbstreams", + feature = "sqs", + feature = "ssm", + feature = "iam", + feature = "lambda", + feature = "apigatewayv2", + feature = "cloudfront", + feature = "cloudfront-dataplane" +))] fn encode_state(value: &T) -> Result> where T: Serialize, @@ -812,6 +973,19 @@ where to_cbor(value) } +#[cfg(any( + feature = "stepfunctions", + feature = "s3", + feature = "dynamodb", + feature = "dynamodbstreams", + feature = "sqs", + feature = "ssm", + feature = "iam", + feature = "lambda", + feature = "apigatewayv2", + feature = "cloudfront", + feature = "cloudfront-dataplane" +))] fn decode_state(bytes: &[u8]) -> Result where T: DeserializeOwned, @@ -886,12 +1060,19 @@ async fn remove_dir_if_exists(path: &Path) -> Result<()> { } } -async fn replace_directory(temp: &Path, target: &Path, root: &Path, name: &str) -> Result<()> { - recover_snapshot(root, target, name).await?; - validate_generation(temp, name) +async fn replace_directory( + temp: &Path, + target: &Path, + root: &Path, + name: &str, + limits: &ArchiveLimits, +) -> Result<()> { + recover_snapshot(root, target, name, limits).await?; + validate_generation(temp, name, limits) .await .context("validate prepared snapshot")?; sync_tree(temp).await?; + retire_previous(root, name).await?; let backup = root.join(format!(".{name}.previous")); let had_target = path_exists(target).await?; if had_target { @@ -921,8 +1102,44 @@ async fn replace_directory(temp: &Path, target: &Path, root: &Path, name: &str) Ok(()) } +/// Call only after successful domain import or fully validated/synced prepare. +fn check_load_deadline(deadline: Deadline) -> Result<()> { + if Deadline::now() >= deadline { + bail!("snapshot import commit deadline exceeded"); + } + Ok(()) +} + +async fn retire_previous(root: &Path, name: &str) -> Result<()> { + retire_previous_checked(root, name, None).await +} + +async fn retire_previous_checked( + root: &Path, + name: &str, + deadline: Option, +) -> Result<()> { + if let Some(deadline) = deadline { + check_load_deadline(deadline)?; + } + let previous = root.join(format!(".{name}.previous")); + if path_exists(&previous).await? { + if let Some(deadline) = deadline { + check_load_deadline(deadline)?; + } + remove_dir_if_exists(&previous).await?; + sync_directory(root).await?; + } + Ok(()) +} + /// Recover an interrupted publish, including the legacy randomly named backups. -async fn recover_snapshot(root: &Path, target: &Path, name: &str) -> Result<()> { +async fn recover_snapshot( + root: &Path, + target: &Path, + name: &str, + limits: &ArchiveLimits, +) -> Result<()> { if !path_exists(root).await? { return Ok(()); } @@ -953,12 +1170,16 @@ async fn recover_snapshot(root: &Path, target: &Path, name: &str) -> Result<()> let Some(backup) = backup else { return Ok(()); }; - if path_exists(target).await? && validate_generation(target, name).await.is_ok() { - remove_dir_if_exists(&backup).await?; - sync_directory(root).await?; - return Ok(()); + if path_exists(target).await? { + match validate_generation(target, name, limits).await { + // Only physical selection is proven here. Domain import can still + // reject this generation; retain its previous completed copy. + Ok(()) => return Ok(()), + Err(error) if error.is::() => return Err(error), + Err(_) => {} + } } - validate_generation(&backup, name) + validate_generation(&backup, name, limits) .await .context("previous snapshot is not a valid recovery generation")?; if path_exists(target).await? { @@ -997,20 +1218,27 @@ async fn cleanup_stale_staging(root: &Path, name: &str) -> Result<()> { Ok(()) } -async fn validate_generation(directory: &Path, name: &str) -> Result<()> { +async fn validate_generation(directory: &Path, name: &str, limits: &ArchiveLimits) -> Result<()> { let manifest = read_manifest(&directory.join(MANIFEST_FILE)).await?; if manifest.schema_version != SNAPSHOT_SCHEMA_VERSION || manifest.snapshot_name != name { bail!("snapshot generation name/schema mismatch"); } - for entry in manifest.services.values() { - let meta = read_archive( + for (service, entry) in &manifest.services { + let limit = limits.get(service.as_str()).copied(); + let meta = read_archive_limited( &snapshot_child(directory, &entry.meta_file)?, ArchiveKind::ServiceMeta, + limit, ) .await?; get_required_section(&meta, SECTION_STATE_CBOR)?; if let Some(data) = &entry.data_file { - read_archive(&snapshot_child(directory, data)?, ArchiveKind::ServiceData).await?; + read_archive_limited( + &snapshot_child(directory, data)?, + ArchiveKind::ServiceData, + limit, + ) + .await?; } } Ok(()) @@ -1069,8 +1297,126 @@ fn now_unix_millis() -> Result { #[cfg(test)] mod tests { + use ciborium::Value; + #[cfg(feature = "dynamodb")] + use rustack_dynamodb_core::config::DynamoDBConfig; + #[cfg(feature = "dynamodb")] + use serde_json::json; + use super::*; + #[cfg(feature = "dynamodb")] + #[tokio::test] + async fn test_should_preserve_long_dynamodb_names_through_actual_archive_import() -> Result<()> + { + let source = Arc::new(RustackDynamoDB::new(DynamoDBConfig::default())); + source.handle_create_table(serde_json::from_value(json!({ + "TableName": "ArchiveNames", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + }))?)?; + // Existing PutItem accepts these ordinary attribute names within400KiB. + // They are data keys, unrelated to a workflow identifier's allowance. + let names = ["a".repeat(4097), "b".repeat(65535), "c".repeat(300_000)]; + for (index, name) in names.iter().enumerate() { + source.handle_put_item(serde_json::from_value(json!({ + "TableName": "ArchiveNames", + "Item": {"pk": {"S": index.to_string()}, name: {"S": "preserved"}} + }))?)?; + } + let directory = tempfile::tempdir()?; + let service = DynamoDBSnapshotService { provider: source }; + let cbor = service.save_meta(directory.path()).await?; + let archive = directory.path().join(META_FILE); + write_archive( + &archive, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new(SECTION_STATE_CBOR, cbor, 1)], + ) + .await?; + let restored = Arc::new(RustackDynamoDB::new(DynamoDBConfig::default())); + let service = DynamoDBSnapshotService { + provider: restored.clone(), + }; + let sections = read_archive(&archive, ArchiveKind::ServiceMeta).await?; + service + .load_meta( + get_required_section(§ions, SECTION_STATE_CBOR)?.to_vec(), + directory.path(), + ) + .await?; + for (index, name) in names.iter().enumerate() { + let result = restored.handle_get_item(serde_json::from_value(json!({ + "TableName": "ArchiveNames", "Key": {"pk": {"S": index.to_string()}} + }))?)?; + let item = serde_json::to_value(result)?; + assert_eq!( + item.get("Item") + .and_then(|item| item.get(name)) + .and_then(|value| value.get("S")), + Some(&json!("preserved")) + ); + } + Ok(()) + } + + struct CborService { + value: Value, + } + + #[async_trait] + impl SnapshotService for CborService { + fn service_name(&self) -> &'static str { + "cbor-test" + } + + async fn save_meta(&self, _data_staging_dir: &Path) -> Result> { + to_cbor(&self.value) + } + + async fn load_meta(&self, state_cbor: Vec, _data_staging_dir: &Path) -> Result<()> { + let value: Value = from_cbor(&state_cbor)?; + if value != self.value { + bail!("unexpected test service state"); + } + Ok(()) + } + } + + #[tokio::test] + async fn test_should_preserve_committed_snapshot_when_cbor_profile_validation_fails() + -> Result<()> { + let root = tempfile::tempdir()?; + let config = SnapshotConfig { + root: root.path().to_owned(), + name: SnapshotName::try_from("dev".to_owned())?, + }; + let valid = RuntimeProviders { + services: vec![Arc::new(CborService { value: Value::Null })], + }; + config.save(&valid, "committed").await?; + let invalid = RuntimeProviders { + services: vec![Arc::new(CborService { + value: Value::Map(vec![ + (Value::Text("private-key".into()), Value::Null), + (Value::Text("private-key".into()), Value::Null), + ]), + })], + }; + let result = config.save(&invalid, "rejected").await; + assert!(result.is_err()); + if let Err(error) = result { + let message = format!("{error:#}"); + assert!(message.contains("duplicate snapshot CBOR key")); + assert!(!message.contains("private-key")); + } + let manifest = read_manifest(&config.snapshot_dir().join(MANIFEST_FILE)).await?; + assert_eq!(manifest.rustack_version, "committed"); + config.load(&valid).await?; + Ok(()) + } + #[test] fn test_should_accept_valid_snapshot_name() { let name = SnapshotName::try_from("dev.snapshot-1".to_owned()); @@ -1117,7 +1463,7 @@ mod tests { let root = tempfile::tempdir()?; let target = generation(root.path(), "dev", "old").await?; fs::rename(&target, root.path().join(".dev.previous")).await?; - recover_snapshot(root.path(), &target, "dev").await?; + recover_snapshot(root.path(), &target, "dev", &BTreeMap::new()).await?; assert_eq!( read_manifest(&target.join(MANIFEST_FILE)) .await? @@ -1132,13 +1478,20 @@ mod tests { let root = tempfile::tempdir()?; let target = generation(root.path(), "dev", "new").await?; let previous = generation(root.path(), ".dev.previous", "old").await?; - recover_snapshot(root.path(), &target, "dev").await?; + recover_snapshot(root.path(), &target, "dev", &BTreeMap::new()).await?; assert_eq!( read_manifest(&target.join(MANIFEST_FILE)) .await? .rustack_version, "new" ); + assert!(path_exists(&previous).await?); + SnapshotConfig { + root: root.path().to_owned(), + name: SnapshotName::try_from("dev".to_owned())?, + } + .load(&RuntimeProviders::default()) + .await?; assert!(!path_exists(&previous).await?); Ok(()) } @@ -1149,7 +1502,7 @@ mod tests { let target = generation(root.path(), "dev", "new").await?; generation(root.path(), ".dev.previous", "old").await?; fs::write(target.join(MANIFEST_FILE), b"corrupt").await?; - recover_snapshot(root.path(), &target, "dev").await?; + recover_snapshot(root.path(), &target, "dev", &BTreeMap::new()).await?; assert_eq!( read_manifest(&target.join(MANIFEST_FILE)) .await? @@ -1176,10 +1529,14 @@ mod tests { let root = tempfile::tempdir()?; let target = root.path().join("dev"); generation(root.path(), ".dev.bak.legacy", "old").await?; - recover_snapshot(root.path(), &target, "dev").await?; + recover_snapshot(root.path(), &target, "dev", &BTreeMap::new()).await?; fs::rename(&target, root.path().join(".dev.bak.one")).await?; generation(root.path(), ".dev.bak.two", "other").await?; - assert!(recover_snapshot(root.path(), &target, "dev").await.is_err()); + assert!( + recover_snapshot(root.path(), &target, "dev", &BTreeMap::new()) + .await + .is_err() + ); Ok(()) } @@ -1203,7 +1560,7 @@ mod tests { let root = tempfile::tempdir()?; let target = generation(root.path(), "dev", "old").await?; let staged = generation(root.path(), ".dev.tmp.new", "new").await?; - replace_directory(&staged, &target, root.path(), "dev").await?; + replace_directory(&staged, &target, root.path(), "dev", &BTreeMap::new()).await?; assert_eq!( read_manifest(&target.join(MANIFEST_FILE)) .await? @@ -1213,7 +1570,7 @@ mod tests { let invalid = root.path().join(".dev.tmp.invalid"); fs::create_dir(&invalid).await?; assert!( - replace_directory(&invalid, &target, root.path(), "dev") + replace_directory(&invalid, &target, root.path(), "dev", &BTreeMap::new()) .await .is_err() ); @@ -1255,3 +1612,9 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod policy_tests; + +#[cfg(all(test, feature = "lambda"))] +mod image_tests; diff --git a/apps/rustack/src/snapshot/archive.rs b/apps/rustack/src/snapshot/archive.rs index 2d8dd855..6816539d 100644 --- a/apps/rustack/src/snapshot/archive.rs +++ b/apps/rustack/src/snapshot/archive.rs @@ -2,14 +2,23 @@ use std::{ collections::BTreeSet, + error::Error as StdError, + fmt, io::{Cursor, Read as _}, path::{Component, Path, PathBuf}, }; use anyhow::{Context, Result, bail}; use crc32fast::Hasher; +#[cfg(unix)] +use nix::fcntl::OFlag; +#[cfg(feature = "lambda")] +use rustack_lambda_core::provider::DockerArchiveProfile; use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use tokio::fs; +use tokio::{fs, io::AsyncReadExt}; + +mod cbor; +use super::worker::WorkerFailure; const MAGIC: [u8; 8] = *b"RSSNAP\0\x02"; const HEADER_LEN: usize = 36; @@ -99,16 +108,35 @@ pub(super) fn to_cbor(value: &T) -> Result> where T: Serialize, { - let mut bytes = Vec::new(); - ciborium::into_writer(value, &mut bytes).context("failed to encode snapshot CBOR")?; - Ok(bytes) + Ok(cbor::encode(value)?) } pub(super) fn from_cbor(bytes: &[u8]) -> Result where T: DeserializeOwned, { - ciborium::from_reader(bytes).context("failed to decode snapshot CBOR") + Ok(cbor::decode(bytes)?) +} + +#[cfg(feature = "lambda")] +pub(super) fn from_cbor_lambda( + bytes: &[u8], + profile: DockerArchiveProfile, +) -> Result { + Ok(cbor::decode_lambda(bytes, profile)?) +} + +#[cfg(feature = "lambda")] +pub(super) fn to_cbor_lambda( + value: &T, + profile: DockerArchiveProfile, +) -> Result> { + Ok(cbor::encode_lambda(value, profile)?) +} + +#[cfg(feature = "stepfunctions")] +pub(super) fn to_cbor_bounded(value: &T, bytes: usize) -> Result> { + Ok(cbor::encode_bounded(value, bytes)?) } pub(super) async fn write_archive( @@ -134,25 +162,126 @@ pub(super) async fn write_archive( }) } -pub(super) async fn read_archive( +/// Policy exhaustion is not evidence that a committed generation is corrupt. +#[derive(Debug)] +pub(super) struct BudgetLimit; +impl fmt::Display for BudgetLimit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("snapshot configured archive byte limit exceeded") + } +} +impl StdError for BudgetLimit {} +#[derive(Debug)] +struct ExpansionLimit; +impl fmt::Display for ExpansionLimit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("decompressed snapshot archive exceeds maximum size") + } +} +impl StdError for ExpansionLimit {} + +/// Bounded regular-file streaming read, including a single detection byte. +pub(super) async fn read_file_bounded(path: &Path, limit: u64) -> Result> { + read_file_bounded_owned( + path, + limit, + #[cfg(test)] + None, + ) + .await +} +async fn read_file_bounded_owned( path: &Path, - expected_kind: ArchiveKind, -) -> Result> { - let metadata = fs::metadata(path) + limit: u64, + #[cfg(test)] gate: Option<&bounded_tests::OpenGate>, +) -> Result> { + if !fs::symlink_metadata(path) .await - .with_context(|| format!("failed to inspect snapshot archive {}", path.display()))?; + .context("inspect snapshot input path")? + .file_type() + .is_file() + { + bail!("snapshot input must be a regular file"); + } + #[cfg(test)] + if let Some(gate) = gate { + gate.wait().await; + } + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags((OFlag::O_NONBLOCK | OFlag::O_NOFOLLOW).bits()); + let file = options + .open(path) + .await + .context("open bounded snapshot input")?; + let metadata = file + .metadata() + .await + .context("inspect bounded snapshot input")?; if !metadata.is_file() { - bail!("snapshot archive is not a regular file: {}", path.display()); + bail!("snapshot input must be a regular file"); } - if metadata.len() > MAX_ARCHIVE_BYTES { - bail!( - "snapshot archive {} exceeds maximum size of {MAX_ARCHIVE_BYTES} bytes", - path.display() - ); + if metadata.len() > limit { + return Err(BudgetLimit.into()); } - let compressed = fs::read(path) + let detection = limit.checked_add(1).ok_or(BudgetLimit)?; + let mut reader = file.take(detection); + let mut bytes = Vec::new(); + reader + .read_to_end(&mut bytes) + .await + .context("read bounded snapshot input")?; + if usize_to_u64(bytes.len())? > limit { + return Err(BudgetLimit.into()); + } + Ok(bytes) +} + +pub(super) async fn read_archive_limited( + path: &Path, + kind: ArchiveKind, + payload: Option, +) -> Result> { + let Some(payload) = payload else { + return read_archive(path, kind).await; + }; + let framed = usize_to_u64(payload)? + .checked_add(1024) + .ok_or(BudgetLimit)?; + let compressed = framed + .checked_mul(2) + .and_then(|n| n.checked_add(1024 * 1024)) + .ok_or(BudgetLimit)? + .min(MAX_ARCHIVE_BYTES); + let bytes = read_file_bounded(path, compressed).await?; + let inner = tokio::task::spawn_blocking(move || decode_zstd_with_limit(bytes, framed)) + .await + .map_err(WorkerFailure::new) + .context("bounded snapshot decompression worker failed")? + .map_err(|error| { + if error.is::() { + BudgetLimit.into() + } else { + error + } + })?; + parse_archive_limited(&inner, kind, Some(payload)) +} + +pub(super) async fn read_archive( + path: &Path, + expected_kind: ArchiveKind, +) -> Result> { + let compressed = read_file_bounded(path, MAX_ARCHIVE_BYTES) .await - .with_context(|| format!("failed to read snapshot archive {}", path.display()))?; + .map_err(|error| { + if error.is::() { + anyhow::anyhow!("snapshot archive exceeds maximum size") + } else { + error + } + })?; let inner = decompress(compressed).await?; parse_archive(&inner, expected_kind) } @@ -214,11 +343,27 @@ pub(super) async fn pack_data_archive( Ok(Some(stats)) } -pub(super) async fn unpack_data_archive(archive_path: &Path, target_dir: &Path) -> Result<()> { - let sections = read_archive(archive_path, ArchiveKind::ServiceData).await?; +pub(super) async fn unpack_data_archive( + archive_path: &Path, + target_dir: &Path, + payload: Option, + expected_file: Option<&str>, +) -> Result<()> { + let sections = read_archive_limited(archive_path, ArchiveKind::ServiceData, payload).await?; let directory_bytes = get_required_section(§ions, SECTION_DATA_DIRECTORY_CBOR)?; let data = get_required_section(§ions, SECTION_DATA_BYTES)?; - let entries: Vec = from_cbor(directory_bytes)?; + let entries: Vec = if let Some(expected) = expected_file { + if sections.len() != 2 || directory_bytes.len() > 1024 { + bail!("invalid workflow data section layout"); + } + let [entry]: [DataDirectoryEntry; 1] = from_cbor(directory_bytes)?; + if entry.relative_path != expected { + bail!("invalid workflow data payload path"); + } + vec![entry] + } else { + from_cbor(directory_bytes)? + }; fs::create_dir_all(target_dir).await.with_context(|| { format!( @@ -346,6 +491,13 @@ fn encode_archive(kind: ArchiveKind, sections: Vec) -> Result Result> { + parse_archive_limited(bytes, expected_kind, None) +} +fn parse_archive_limited( + bytes: &[u8], + expected_kind: ArchiveKind, + payload: Option, +) -> Result> { let min_len = checked_add_usize(HEADER_LEN, FOOTER_LEN)?; if bytes.len() < min_len { bail!("snapshot archive is too short"); @@ -425,6 +577,9 @@ fn parse_archive(bytes: &[u8], expected_kind: ArchiveKind) -> Result limit) { + return Err(BudgetLimit.into()); + } let end = checked_add_usize(start, len)?; let section_bytes = bytes .get(start..end) @@ -516,7 +671,11 @@ async fn decompress(bytes: Vec) -> Result> { } fn decode_zstd_bounded(bytes: Vec) -> Result> { - let read_limit = MAX_ARCHIVE_BYTES + decode_zstd_with_limit(bytes, MAX_ARCHIVE_BYTES) +} + +fn decode_zstd_with_limit(bytes: Vec, limit: u64) -> Result> { + let read_limit = limit .checked_add(1) .ok_or_else(|| anyhow::anyhow!("snapshot archive size overflow"))?; let decoder = zstd::stream::read::Decoder::new(Cursor::new(bytes)) @@ -526,8 +685,8 @@ fn decode_zstd_bounded(bytes: Vec) -> Result> { limited .read_to_end(&mut output) .context("failed to decompress snapshot archive")?; - if usize_to_u64(output.len())? > MAX_ARCHIVE_BYTES { - bail!("decompressed snapshot archive exceeds maximum size"); + if usize_to_u64(output.len())? > limit { + return Err(ExpansionLimit.into()); } Ok(output) } @@ -678,10 +837,115 @@ fn u64_to_usize(value: u64) -> Result { #[cfg(test)] mod tests { + use std::io::{Read, Write}; + use tempfile::tempdir; + use zstd::{ + stream::{read::Decoder as ZstdReader, write::Encoder as ZstdWriter}, + zstd_safe::{CCtx, DCtx}, + }; use super::*; + const LEGACY_V2: &[u8] = include_bytes!("fixtures/legacy-v2-meta.ss.zst"); + + #[test] + fn test_should_preserve_original_zstd_0133_snapshot_envelope() -> Result<()> { + let plain = decode_zstd_bounded(LEGACY_V2.to_vec())?; + assert_eq!(plain.len(), 88); + let sections = parse_archive(&plain, ArchiveKind::ServiceMeta)?; + let section = sections.first().context("legacy section")?; + assert_eq!(sections.len(), 1); + assert_eq!( + (section.kind, section.flags, section.row_count), + (SECTION_STATE_CBOR, 0, 7) + ); + assert_eq!(section.bytes, b"legacy-state"); + Ok(()) + } + + #[test] + fn test_should_reject_every_truncated_legacy_frame_and_reencoded_corrupt_envelope() -> Result<()> + { + for end in 0..LEGACY_V2.len() { + let prefix = LEGACY_V2.get(..end).context("bounded prefix")?; + assert!( + decode_zstd_bounded(prefix.to_vec()) + .and_then(|plain| parse_archive(&plain, ArchiveKind::ServiceMeta)) + .is_err(), + "truncated prefix {end}" + ); + } + let mut plain = decode_zstd_bounded(LEGACY_V2.to_vec())?; + let index = plain.len().checked_sub(FOOTER_LEN).context("CRC offset")?; + let crc = plain.get_mut(index).context("CRC byte")?; + *crc = crc.wrapping_add(1); + let frame = zstd::stream::encode_all(Cursor::new(plain), ZSTD_LEVEL)?; + let plain = decode_zstd_bounded(frame)?; + assert!(parse_archive(&plain, ArchiveKind::ServiceMeta).is_err()); + Ok(()) + } + + #[rstest::rstest] + #[case(0)] + #[case(1)] + #[case(4096)] + #[test] + fn test_should_enforce_exact_decompressed_byte_boundary(#[case] limit: usize) -> Result<()> { + let exact = zstd::stream::encode_all(Cursor::new(vec![b'x'; limit]), ZSTD_LEVEL)?; + assert_eq!( + decode_zstd_with_limit(exact, u64::try_from(limit)?)?.len(), + limit + ); + let excess = zstd::stream::encode_all(Cursor::new(vec![b'x'; limit + 1]), ZSTD_LEVEL)?; + assert!(decode_zstd_with_limit(excess, u64::try_from(limit)?).is_err()); + assert!(decode_zstd_with_limit(Vec::new(), u64::MAX).is_err()); + Ok(()) + } + + #[rstest::rstest] + #[case(1)] + #[case(4096)] + #[case(131_071)] + #[test] + fn test_should_discard_abandoned_encoder_input_before_reusing_context( + #[case] size: usize, + ) -> Result<()> { + let mut context = CCtx::create(); + { + let mut abandoned = ZstdWriter::with_context(Vec::new(), &mut context); + abandoned.write_all(&vec![b'X'; size])?; + } + let mut current = ZstdWriter::with_context(Vec::new(), &mut context); + current.write_all(b"current-stream")?; + let frame = current.finish()?; + assert_eq!(decode_zstd_with_limit(frame, 64)?, b"current-stream"); + Ok(()) + } + + #[rstest::rstest] + #[case(4096)] + #[case(131_072)] + #[test] + fn test_should_discard_abandoned_decoder_output_before_reusing_context( + #[case] size: usize, + ) -> Result<()> { + let mut context = DCtx::create(); + let previous = zstd::stream::encode_all(Cursor::new(vec![b'X'; size]), ZSTD_LEVEL)?; + { + let mut abandoned = ZstdReader::with_context(Cursor::new(previous), &mut context); + let mut byte = [0_u8; 1]; + abandoned.read_exact(&mut byte)?; + assert_eq!(byte, [b'X']); + } + let frame = zstd::stream::encode_all(Cursor::new(b"current-stream"), ZSTD_LEVEL)?; + let mut current = ZstdReader::with_context(Cursor::new(frame), &mut context).take(64); + let mut output = Vec::new(); + current.read_to_end(&mut output)?; + assert_eq!(output, b"current-stream"); + Ok(()) + } + #[tokio::test] async fn test_should_round_trip_archive_sections() -> Result<()> { let dir = tempdir()?; @@ -770,7 +1034,7 @@ mod tests { assert!(stats.compressed_bytes > 0); let target = dir.path().join("target"); - unpack_data_archive(&archive, &target).await?; + unpack_data_archive(&archive, &target, None, None).await?; let restored = fs::read(target.join("objects").join("0.bin")) .await .context("restored file should be readable")?; @@ -809,7 +1073,7 @@ mod tests { ) .await?; - let result = unpack_data_archive(&archive, &dir.path().join("target")).await; + let result = unpack_data_archive(&archive, &dir.path().join("target"), None, None).await; assert!(result.is_err()); Ok(()) } @@ -820,3 +1084,6 @@ mod tests { assert!(result.is_err()); } } + +#[cfg(test)] +mod bounded_tests; diff --git a/apps/rustack/src/snapshot/archive/bounded_tests.rs b/apps/rustack/src/snapshot/archive/bounded_tests.rs new file mode 100644 index 00000000..31e26f8b --- /dev/null +++ b/apps/rustack/src/snapshot/archive/bounded_tests.rs @@ -0,0 +1,168 @@ +//! Workflow payload limits allow framing and reject before section allocation. + +use serde::{Serialize, Serializer}; + +use super::*; + +#[derive(Debug)] +struct Blob(Vec); +impl Serialize for Blob { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.0) + } +} +fn noise() -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + (0..4096) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let [byte, ..] = state.to_le_bytes(); + byte + }) + .collect() +} +#[tokio::test] +async fn test_should_allow_incompressible_payload_at_exact_limit_with_frame_overhead() -> Result<()> +{ + let directory = tempfile::tempdir()?; + let path = directory.path().join("meta.ss.zst"); + let raw = noise(); + let payload = to_cbor(&Blob(raw.clone()))?; + let limit = payload.len(); + let stats = write_archive( + &path, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new(SECTION_STATE_CBOR, payload.clone(), 1)], + ) + .await?; + assert!( + stats.compressed_bytes > usize_to_u64(limit)?, + "fixture failed to exercise compressed overhead" + ); + let read = read_archive_limited(&path, ArchiveKind::ServiceMeta, Some(limit)).await?; + assert_eq!(get_required_section(&read, SECTION_STATE_CBOR)?, payload); + let decoded: ciborium::Value = from_cbor(get_required_section(&read, SECTION_STATE_CBOR)?)?; + assert_eq!(decoded, ciborium::Value::Bytes(raw)); + let error = read_archive_limited(&path, ArchiveKind::ServiceMeta, Some(limit - 1)) + .await + .err() + .ok_or_else(|| anyhow::anyhow!("accepted oversized section"))?; + assert!(error.is::()); + Ok(()) +} +#[tokio::test] +async fn test_should_reject_compressed_and_expansion_limits_as_policy_errors() -> Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("meta.ss.zst"); + fs::write(&path, vec![0; 1024 * 1024 + 4096]).await?; + assert!( + read_archive_limited(&path, ArchiveKind::ServiceMeta, Some(1)) + .await + .err() + .is_some_and(|error| error.is::()) + ); + write_archive( + &path, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new(SECTION_STATE_CBOR, vec![0; 4096], 1)], + ) + .await?; + assert!( + read_archive_limited(&path, ArchiveKind::ServiceMeta, Some(1)) + .await + .err() + .is_some_and(|error| error.is::()) + ); + Ok(()) +} +#[tokio::test] +async fn test_should_reject_wrong_or_extra_workflow_data_entries_before_extraction() -> Result<()> { + let directory = tempfile::tempdir()?; + for extra in [false, true] { + let staging = directory.path().join(if extra { "two" } else { "wrong" }); + fs::create_dir(&staging).await?; + fs::write(staging.join("unexpected"), b"private payload").await?; + if extra { + fs::write(staging.join("history.cbor"), b"history").await?; + } + let path = directory.path().join("data.ss.zst"); + pack_data_archive(&staging, &path).await?; + let target = directory.path().join("not-created"); + assert!( + unpack_data_archive(&path, &target, Some(4096), Some("history.cbor")) + .await + .is_err() + ); + assert!(!fs::try_exists(target).await?); + } + Ok(()) +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_reject_fifo_without_a_writer_and_join_the_exact_read_task() -> Result<()> { + use nix::{sys::stat::Mode, unistd::mkfifo}; + use tokio::time::{Duration, timeout}; + + let directory = tempfile::tempdir()?; + let path = directory.path().join("meta.ss.zst"); + let create = path.clone(); + tokio::task::spawn_blocking(move || mkfifo(&create, Mode::S_IRUSR | Mode::S_IWUSR)).await??; + let mut task = tokio::spawn(async move { + read_archive_limited(&path, ArchiveKind::ServiceMeta, Some(4096)).await + }); + let result = timeout(Duration::from_secs(2), &mut task).await??; + assert!(result.is_err()); + assert!(task.is_finished()); + Ok(()) +} + +#[derive(Debug, Default)] +pub(super) struct OpenGate { + entered: tokio::sync::Notify, + released: tokio::sync::Notify, +} +impl OpenGate { + pub(super) async fn wait(&self) { + let release = self.released.notified(); + self.entered.notify_one(); + release.await; + } +} +#[cfg(unix)] +#[tokio::test] +async fn test_should_reject_fifo_and_symlink_swap_after_regular_file_precheck() -> Result<()> { + use std::sync::Arc; + + use nix::{sys::stat::Mode, unistd::mkfifo}; + use tokio::time::{Duration, timeout}; + + for symlink in [false, true] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("meta.ss.zst"); + let victim = directory.path().join("private"); + fs::write(&path, b"regular input").await?; + fs::write(&victim, b"private file").await?; + let gate = Arc::new(OpenGate::default()); + let delayed = gate.clone(); + let input = path.clone(); + let mut task = + tokio::spawn( + async move { read_file_bounded_owned(&input, 4096, Some(&delayed)).await }, + ); + timeout(Duration::from_secs(2), gate.entered.notified()).await?; + fs::remove_file(&path).await?; + if symlink { + fs::symlink(&victim, &path).await?; + } else { + tokio::task::spawn_blocking(move || mkfifo(&path, Mode::S_IRUSR | Mode::S_IWUSR)) + .await??; + } + gate.released.notify_one(); + assert!(timeout(Duration::from_secs(2), &mut task).await??.is_err()); + assert!(task.is_finished()); + } + Ok(()) +} diff --git a/apps/rustack/src/snapshot/archive/cbor.rs b/apps/rustack/src/snapshot/archive/cbor.rs new file mode 100644 index 00000000..da22cdc2 --- /dev/null +++ b/apps/rustack/src/snapshot/archive/cbor.rs @@ -0,0 +1,408 @@ +//! Borrowed structural validation before typed snapshot CBOR allocation. +//! +//! The v2 profile intentionally excludes tags, indefinite strings and compound +//! map keys. All generated snapshot DTOs use scalar keys and definite strings. + +#![forbid(unsafe_code)] + +use std::{ + collections::BTreeSet, + error::Error as StdError, + fmt, + io::{self, Cursor, Write}, + str, +}; + +#[cfg(feature = "lambda")] +use rustack_lambda_core::provider::DockerArchiveProfile; +use serde::{Serialize, de::DeserializeOwned}; + +mod lambda; +use lambda::{Budget as LambdaBudget, Path as LambdaPath}; + +const TYPED_RECURSION_LIMIT: usize = 256; + +const LIMITS: Limits = Limits { + bytes: 2 * 1024 * 1024 * 1024, + depth: 128, + nodes: 8_000_000, + collection: 1_000_000, + scalar: 128 * 1024 * 1024, + key: 128 * 1024 * 1024, +}; + +#[derive(Debug, Clone, Copy)] +struct Limits { + bytes: usize, + depth: usize, + nodes: usize, + collection: usize, + scalar: usize, + key: usize, +} + +/// Fixed classes: parser/serde diagnostics must never echo snapshot data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Failure { + Bytes, + Depth, + Nodes, + Collection, + Scalar, + Key, + Duplicate, + Syntax, + Profile, + Trailing, + Schema, + Encoding, + Allocation, +} +impl fmt::Display for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Bytes => "snapshot CBOR byte limit exceeded", + Self::Depth => "snapshot CBOR depth limit exceeded", + Self::Nodes => "snapshot CBOR value limit exceeded", + Self::Collection => "snapshot CBOR collection limit exceeded", + Self::Scalar => "snapshot CBOR scalar limit exceeded", + Self::Key => "snapshot CBOR key limit exceeded", + Self::Duplicate => "duplicate snapshot CBOR key", + Self::Syntax => "invalid snapshot CBOR syntax", + Self::Profile => "unsupported snapshot CBOR profile", + Self::Trailing => "trailing snapshot CBOR data", + Self::Schema => "invalid snapshot CBOR schema", + Self::Encoding => "failed to encode snapshot CBOR", + Self::Allocation => "snapshot CBOR allocation failed", + }) + } +} +impl StdError for Failure {} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Key<'a> { + Unsigned(u64), + Negative(u64), + Text(&'a str), +} + +struct Scan<'a> { + input: &'a [u8], + offset: usize, + remaining: usize, + limits: Limits, + lambda: Option, +} +impl<'a> Scan<'a> { + fn validate(input: &'a [u8], limits: Limits) -> Result<(), Failure> { + if input.len() > limits.bytes { + return Err(Failure::Bytes); + } + let mut scan = Self { + input, + offset: 0, + remaining: limits.nodes, + limits, + lambda: None, + }; + scan.item(0, LambdaPath::Root)?; + if scan.offset != input.len() { + return Err(Failure::Trailing); + } + Ok(()) + } + fn read(&mut self, len: usize) -> Result<&'a [u8], Failure> { + if let Some(budget) = &mut self.lambda { + budget.metadata = budget.metadata.checked_sub(len).ok_or(Failure::Bytes)?; + } + self.read_unaccounted(len) + } + fn read_unaccounted(&mut self, len: usize) -> Result<&'a [u8], Failure> { + let end = self.offset.checked_add(len).ok_or(Failure::Syntax)?; + let bytes = self.input.get(self.offset..end).ok_or(Failure::Syntax)?; + self.offset = end; + Ok(bytes) + } + fn byte(&mut self) -> Result { + self.read(1)?.first().copied().ok_or(Failure::Syntax) + } + fn header(&mut self) -> Result<(u8, Option), Failure> { + self.remaining = self.remaining.checked_sub(1).ok_or(Failure::Nodes)?; + let initial = self.byte()?; + let major = initial >> 5; + let value = match initial & 31 { + value @ 0..=23 => Some(u64::from(value)), + 24 => { + let value = self.byte()?; + if major == 7 && value < 32 { + return Err(Failure::Syntax); + } + Some(u64::from(value)) + } + 25 => Some(u64::from(u16::from_be_bytes( + self.read(2)?.try_into().map_err(|_| Failure::Syntax)?, + ))), + 26 => Some(u64::from(u32::from_be_bytes( + self.read(4)?.try_into().map_err(|_| Failure::Syntax)?, + ))), + 27 => Some(u64::from_be_bytes( + self.read(8)?.try_into().map_err(|_| Failure::Syntax)?, + )), + 31 => None, + _ => return Err(Failure::Syntax), + }; + Ok((major, value)) + } + fn scalar(&mut self, len: Option, key: bool) -> Result<&'a [u8], Failure> { + let len = len.ok_or(Failure::Profile)?; + let len = usize::try_from(len).map_err(|_| Failure::Scalar)?; + if key && len > self.limits.key { + return Err(Failure::Key); + } + if len > self.limits.scalar { + return Err(Failure::Scalar); + } + self.read(len) + } + fn key(&mut self) -> Result, Failure> { + let (major, value) = self.header()?; + match major { + 0 => Ok(Key::Unsigned(value.ok_or(Failure::Syntax)?)), + 1 => Ok(Key::Negative(value.ok_or(Failure::Syntax)?)), + 3 => Ok(Key::Text( + str::from_utf8(self.scalar(value, true)?).map_err(|_| Failure::Syntax)?, + )), + _ => Err(Failure::Profile), + } + } + fn item(&mut self, depth: usize, path: LambdaPath) -> Result<(), Failure> { + let (major, value) = self.header()?; + let path = path.shape(major); + match major { + 0 | 1 | 7 => { + value.ok_or(Failure::Syntax)?; + } + 2 => { + self.scalar(value, false)?; + } + 3 => { + if path == LambdaPath::Zip && self.lambda.as_ref().is_some_and(|budget| budget.zip) + { + let len = usize::try_from(value.ok_or(Failure::Profile)?) + .map_err(|_| Failure::Scalar)?; + let bytes = self.read_unaccounted(len)?; + self.lambda.as_mut().ok_or(Failure::Profile)?.blob(bytes)?; + str::from_utf8(bytes).map_err(|_| Failure::Syntax)?; + } else { + str::from_utf8(self.scalar(value, false)?).map_err(|_| Failure::Syntax)?; + } + } + 4 | 5 => { + let depth = depth.checked_add(1).ok_or(Failure::Depth)?; + if depth > self.limits.depth { + return Err(Failure::Depth); + } + self.collection(major == 5, value, depth, path)?; + } + _ => return Err(Failure::Profile), + } + Ok(()) + } + fn collection( + &mut self, + map: bool, + count: Option, + depth: usize, + path: LambdaPath, + ) -> Result<(), Failure> { + let count = count + .map(|count| usize::try_from(count).map_err(|_| Failure::Collection)) + .transpose()?; + if let Some(count) = count { + if let Some(budget) = &self.lambda { + budget.collection(path, count)?; + } + if count > self.limits.collection { + return Err(Failure::Collection); + } + let minimum = count + .checked_mul(if map { 2 } else { 1 }) + .ok_or(Failure::Collection)?; + if minimum > self.remaining { + return Err(Failure::Nodes); + } + if minimum > self.input.len().saturating_sub(self.offset) { + return Err(Failure::Syntax); + } + } + let mut keys = BTreeSet::new(); + let mut visited = 0_usize; + loop { + if count == Some(visited) { + return Ok(()); + } + if count.is_none() && self.input.get(self.offset) == Some(&0xff) { + self.byte()?; + return Ok(()); + } + visited = visited.checked_add(1).ok_or(Failure::Collection)?; + if visited > self.limits.collection { + return Err(Failure::Collection); + } + let key = if map { Some(self.key()?) } else { None }; + if let Some(budget) = &mut self.lambda { + budget.member(path, key.as_ref())?; + } + let child = path.child(key.as_ref()); + if let Some(key) = key { + if !keys.insert(key) { + return Err(Failure::Duplicate); + } + } + self.item(depth, child)?; + } + } +} + +struct CappedWriter { + bytes: Vec, + limit: usize, + failure: Option, +} +impl Write for CappedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let result = self.append(bytes); + if let Err(failure) = result { + self.failure = Some(failure); + } + result.map_err(io::Error::other) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +impl CappedWriter { + fn append(&mut self, bytes: &[u8]) -> Result { + let next = self + .bytes + .len() + .checked_add(bytes.len()) + .filter(|next| *next <= self.limit) + .ok_or(Failure::Bytes)?; + if next > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(next) + .min(self.limit); + self.bytes + .try_reserve_exact(capacity.saturating_sub(self.bytes.len())) + .map_err(|_| Failure::Allocation)?; + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } +} + +pub(super) fn encode(value: &T) -> Result, Failure> { + encode_with_limits(value, LIMITS) +} +#[cfg(feature = "stepfunctions")] +pub(super) fn encode_bounded(value: &T, bytes: usize) -> Result, Failure> { + encode_with_limits( + value, + Limits { + bytes: bytes.min(LIMITS.bytes), + ..LIMITS + }, + ) +} +fn encode_with_limits(value: &T, limits: Limits) -> Result, Failure> { + let mut writer = CappedWriter { + bytes: Vec::new(), + limit: limits.bytes, + failure: None, + }; + let result = ciborium::into_writer(value, &mut writer); + if let Some(failure) = writer.failure { + return Err(failure); + } + result.map_err(|_| Failure::Encoding)?; + Scan::validate(&writer.bytes, limits)?; + Ok(writer.bytes) +} + +pub(super) fn decode(bytes: &[u8]) -> Result { + decode_with_limits(bytes, LIMITS) +} +fn decode_with_limits(bytes: &[u8], limits: Limits) -> Result { + Scan::validate(bytes, limits)?; + decode_typed(bytes) +} +fn decode_typed(bytes: &[u8]) -> Result { + let mut reader = Cursor::new(bytes); + let value = ciborium::de::from_reader_with_recursion_limit(&mut reader, TYPED_RECURSION_LIMIT) + .map_err(|_| Failure::Schema)?; + if usize::try_from(reader.position()).map_err(|_| Failure::Trailing)? != bytes.len() { + return Err(Failure::Trailing); + } + Ok(value) +} + +#[cfg(feature = "lambda")] +fn validate_lambda(bytes: &[u8], profile: DockerArchiveProfile) -> Result<(), Failure> { + if bytes.len() > profile.payload_limit() { + return Err(Failure::Bytes); + } + let mut scan = Scan { + input: bytes, + offset: 0, + remaining: 131_073, + limits: Limits { + bytes: profile.payload_limit(), + depth: 64, + nodes: 131_073, + collection: 65_536, + scalar: 128 * 1024, + key: 1024, + }, + lambda: Some(LambdaBudget::new(profile)), + }; + scan.item(0, LambdaPath::Root)?; + if scan.offset != bytes.len() { + return Err(Failure::Trailing); + } + Ok(()) +} + +#[cfg(feature = "lambda")] +pub(super) fn decode_lambda( + bytes: &[u8], + profile: DockerArchiveProfile, +) -> Result { + validate_lambda(bytes, profile)?; + decode_typed(bytes) +} + +#[cfg(feature = "lambda")] +pub(super) fn encode_lambda( + value: &T, + profile: DockerArchiveProfile, +) -> Result, Failure> { + let bytes = encode_with_limits( + value, + Limits { + bytes: profile.payload_limit(), + ..LIMITS + }, + )?; + validate_lambda(&bytes, profile)?; + Ok(bytes) +} + +#[cfg(test)] +mod tests; + +#[cfg(all(test, feature = "lambda"))] +mod lambda_tests; diff --git a/apps/rustack/src/snapshot/archive/cbor/lambda.rs b/apps/rustack/src/snapshot/archive/cbor/lambda.rs new file mode 100644 index 00000000..e7f07343 --- /dev/null +++ b/apps/rustack/src/snapshot/archive/cbor/lambda.rs @@ -0,0 +1,133 @@ +//! Exact-path metadata accounting, shared by the existing CBOR scanner. + +#[cfg(feature = "lambda")] +use rustack_lambda_core::provider::DockerArchiveProfile; + +use super::{Failure, Key}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Path { + Root, + Functions, + Rows, + Function, + Versions, + Version, + Zip, + Other, +} +impl Path { + pub(super) fn shape(self, major: u8) -> Self { + match (self, major) { + (Self::Root | Self::Functions | Self::Function | Self::Versions | Self::Version, 5) + | (Self::Rows, 4) + | (Self::Zip, 3) => self, + _ => Self::Other, + } + } + pub(super) fn child(self, key: Option<&Key<'_>>) -> Self { + match (self, key) { + (Self::Root, Some(Key::Text("functions"))) => Self::Functions, + (Self::Functions, Some(Key::Text("functions"))) => Self::Rows, + (Self::Rows, None) => Self::Function, + (Self::Function, Some(Key::Text("latest"))) + | (Self::Versions, Some(Key::Unsigned(_))) => Self::Version, + (Self::Function, Some(Key::Text("versions"))) => Self::Versions, + (Self::Version, Some(Key::Text("zipBytesBase64"))) => Self::Zip, + _ => Self::Other, + } + } +} + +#[derive(Debug)] +pub(super) struct Budget { + pub(super) metadata: usize, + members: usize, + functions: usize, + revisions: usize, + zip_remaining: usize, + zip_maximum: usize, + pub(super) zip: bool, +} +impl Budget { + #[cfg(feature = "lambda")] + pub(super) fn new(profile: DockerArchiveProfile) -> Self { + Self { + metadata: DockerArchiveProfile::METADATA_BYTES, + members: 65_536, + functions: DockerArchiveProfile::MAX_FUNCTIONS, + revisions: DockerArchiveProfile::MAX_REVISIONS, + zip_remaining: DockerArchiveProfile::MAX_ZIP_DATA_BYTES, + zip_maximum: DockerArchiveProfile::MAX_ZIP_BYTES, + zip: profile.zip_architecture().is_some(), + } + } + pub(super) fn collection(&self, path: Path, count: usize) -> Result<(), Failure> { + if count > self.members + || (path == Path::Rows && count > self.functions) + || (path == Path::Versions && count > self.revisions) + { + return Err(Failure::Collection); + } + Ok(()) + } + pub(super) fn member(&mut self, path: Path, key: Option<&Key<'_>>) -> Result<(), Failure> { + self.members = self.members.checked_sub(1).ok_or(Failure::Nodes)?; + if path == Path::Rows { + self.functions = self.functions.checked_sub(1).ok_or(Failure::Collection)?; + } + if path == Path::Versions + || (path == Path::Function && matches!(key, Some(Key::Text("latest")))) + { + self.revisions = self.revisions.checked_sub(1).ok_or(Failure::Collection)?; + } + Ok(()) + } + pub(super) fn blob(&mut self, bytes: &[u8]) -> Result<(), Failure> { + let maximum = self + .zip_maximum + .div_ceil(3) + .checked_mul(4) + .ok_or(Failure::Scalar)?; + if bytes.len() > maximum || !bytes.len().is_multiple_of(4) { + return Err(Failure::Scalar); + } + let padding = if bytes.ends_with(b"==") { + 2 + } else { + usize::from(bytes.ends_with(b"=")) + }; + let decoded = bytes + .len() + .checked_div(4) + .and_then(|n| n.checked_mul(3)) + .and_then(|n| n.checked_sub(padding)) + .ok_or(Failure::Scalar)?; + if decoded > self.zip_maximum { + return Err(Failure::Scalar); + } + self.zip_remaining = self + .zip_remaining + .checked_sub(decoded) + .ok_or(Failure::Bytes)?; + Ok(()) + } +} + +#[cfg(all(test, feature = "lambda"))] +mod tests { + use rustack_lambda_core::{artifact::Architecture, provider::DockerArchiveProfile}; + + use super::*; + #[test] + fn test_should_bound_each_blob_and_aggregate_before_typed_allocation() { + let mut budget = Budget::new(DockerArchiveProfile::ProvidedZip(Architecture::Arm64)); + budget.zip_maximum = 3; + budget.zip_remaining = 5; + assert_eq!(budget.blob(b"AAAA"), Ok(())); + assert_eq!(budget.blob(b"AAA="), Ok(())); + assert_eq!(budget.blob(b"AA=="), Err(Failure::Bytes)); + assert_eq!(budget.blob(b"AAAAAAAA"), Err(Failure::Scalar)); + assert_eq!(budget.blob(b"AAA"), Err(Failure::Scalar)); + } +} diff --git a/apps/rustack/src/snapshot/archive/cbor/lambda_tests.rs b/apps/rustack/src/snapshot/archive/cbor/lambda_tests.rs new file mode 100644 index 00000000..4e454ea3 --- /dev/null +++ b/apps/rustack/src/snapshot/archive/cbor/lambda_tests.rs @@ -0,0 +1,219 @@ +//! Raw Lambda path and metadata barriers must precede typed allocation. + +use std::cell::Cell; + +use ciborium::Value; +use rstest::rstest; +use rustack_lambda_core::{artifact::Architecture, provider::DockerArchiveProfile}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::IgnoredAny, + ser::{SerializeMap, SerializeSeq}, +}; + +use super::*; + +thread_local! { static VISITS: Cell = const { Cell::new(0) }; } +#[derive(Debug)] +struct Sentinel; +impl<'de> Deserialize<'de> for Sentinel { + fn deserialize>(decoder: D) -> Result { + VISITS.set(VISITS.get().saturating_add(1)); + IgnoredAny::deserialize(decoder)?; + Ok(Self) + } +} +fn managed() -> DockerArchiveProfile { + DockerArchiveProfile::ProvidedZip(Architecture::Arm64) +} +fn map(values: Vec<(&str, Value)>) -> Value { + Value::Map( + values + .into_iter() + .map(|(key, value)| (Value::Text(key.into()), value)) + .collect(), + ) +} +fn body() -> Value { + Value::Text("A".repeat(128 * 1024 + 4)) +} +fn version() -> Value { + map(vec![("zipBytesBase64", body())]) +} +fn row(published: bool) -> Value { + if published { + map(vec![( + "versions", + Value::Map(vec![(Value::Integer(1_u64.into()), version())]), + )]) + } else { + map(vec![("latest", version())]) + } +} +fn rows(value: Value) -> Value { + map(vec![( + "functions", + map(vec![("functions", Value::Array(vec![value]))]), + )]) +} +fn rejected(value: &Value, profile: DockerArchiveProfile, expected: Failure) { + let mut bytes = Vec::new(); + ciborium::into_writer(value, &mut bytes).unwrap(); + VISITS.set(0); + assert_eq!( + decode_lambda::(&bytes, profile).err(), + Some(expected) + ); + assert_eq!(VISITS.get(), 0); +} + +struct Indefinite<'a>(&'a Value); +impl Serialize for Indefinite<'_> { + fn serialize(&self, serializer: S) -> Result { + match self.0 { + Value::Map(values) => { + let mut map = serializer.serialize_map(None)?; + for (key, value) in values { + map.serialize_entry(key, &Self(value))?; + } + map.end() + } + Value::Array(values) => { + let mut sequence = serializer.serialize_seq(None)?; + for value in values { + sequence.serialize_element(&Self(value))?; + } + sequence.end() + } + value => value.serialize(serializer), + } + } +} +#[rstest] +#[case(false, false)] +#[case(false, true)] +#[case(true, false)] +#[case(true, true)] +fn test_should_allow_only_known_latest_and_unsigned_version_body_paths( + #[case] published: bool, + #[case] indefinite: bool, +) { + let value = rows(row(published)); + let bytes = if indefinite { + encode(&Indefinite(&value)).unwrap() + } else { + encode(&value).unwrap() + }; + let decoded: Value = decode_lambda(&bytes, managed()).unwrap(); + assert_eq!(decoded, value); + assert!(matches!( + decode_lambda::(&bytes, DockerArchiveProfile::ImageOnly), + Err(Failure::Scalar) + )); + assert_eq!( + encode_lambda(&value, managed()).unwrap(), + encode(&value).unwrap() + ); +} + +#[rstest] +#[case(0)] +#[case(1)] +#[case(2)] +#[case(3)] +#[case(4)] +#[case(5)] +#[case(6)] +#[case(7)] +fn test_should_reject_misplaced_and_unknown_lookalikes_before_deserialization( + #[case] location: usize, +) { + let value = match location { + 0 => map(vec![("unknown", rows(row(false)))]), + 1 => map(vec![("functions", map(vec![("unknown", version())]))]), + 2 => rows(map(vec![("policy", map(vec![("latest", version())]))])), + 3 => rows(map(vec![("latest", map(vec![("unknown", version())]))])), + 4 => map(vec![("layers", map(vec![("latest", version())]))]), + 5 => rows(map(vec![( + "versions", + Value::Map(vec![(Value::Text("1".into()), version())]), + )])), + 6 => rows(map(vec![( + "versions", + Value::Map(vec![(Value::Integer((-1_i64).into()), version())]), + )])), + _ => map(vec![( + "functions", + map(vec![("functions", map(vec![("latest", version())]))]), + )]), + }; + rejected(&value, managed(), Failure::Scalar); +} +#[test] +fn test_should_reject_duplicate_keys_even_when_both_values_are_authorized_zip_slots() { + rejected( + &rows(map(vec![("latest", version()), ("latest", version())])), + managed(), + Failure::Duplicate, + ); +} +#[rstest] +#[case(false)] +#[case(true)] +fn test_should_bound_function_and_revision_counts_before_typed_allocation(#[case] revisions: bool) { + let value = if revisions { + let versions = (1_u64..=1024) + .map(|n| (Value::Integer(n.into()), Value::Null)) + .collect(); + rows(map(vec![ + ("latest", Value::Null), + ("versions", Value::Map(versions)), + ])) + } else { + map(vec![( + "functions", + map(vec![("functions", Value::Array(vec![Value::Null; 257]))]), + )]) + }; + for bytes in [ + encode(&value).unwrap(), + encode(&Indefinite(&value)).unwrap(), + ] { + VISITS.set(0); + assert!(matches!( + decode_lambda::(&bytes, managed()), + Err(Failure::Collection) + )); + assert_eq!(VISITS.get(), 0); + } +} +#[test] +fn test_should_charge_headers_and_keys_but_exempt_only_exact_body_contents() { + let bytes = encode(&rows(row(false))).unwrap(); + let metadata = bytes.len().checked_sub(128 * 1024 + 4).unwrap(); + for (allowed, expected) in [ + (metadata, Ok(())), + (metadata.checked_sub(1).unwrap(), Err(Failure::Bytes)), + ] { + let mut budget = LambdaBudget::new(managed()); + budget.metadata = allowed; + let mut scan = Scan { + input: &bytes, + offset: 0, + remaining: LIMITS.nodes, + limits: LIMITS, + lambda: Some(budget), + }; + assert_eq!(scan.item(0, LambdaPath::Root), expected); + } +} +#[test] +fn test_should_preserve_pair_and_array_member_budget_semantics() { + let exact = encode(&vec![(); 65_536]).unwrap(); + assert!(decode_lambda::(&exact, managed()).is_ok()); + rejected( + &Value::Array(vec![Value::Null; 65_537]), + managed(), + Failure::Collection, + ); +} diff --git a/apps/rustack/src/snapshot/archive/cbor/tests.rs b/apps/rustack/src/snapshot/archive/cbor/tests.rs new file mode 100644 index 00000000..f9d126d6 --- /dev/null +++ b/apps/rustack/src/snapshot/archive/cbor/tests.rs @@ -0,0 +1,505 @@ +//! Profile, resource and before-deserialization barriers for snapshot imports. + +use std::{cell::Cell, collections::BTreeMap, error::Error}; + +use ciborium::Value; +use proptest::{collection, prelude::*}; +use rstest::rstest; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::IgnoredAny, + ser::{Error as _, SerializeSeq}, +}; + +use super::{ + CappedWriter, Failure, LIMITS, Limits, Scan, decode, decode_with_limits, encode, + encode_with_limits, +}; + +type TestResult = Result<(), Box>; + +thread_local! { + static VISITS: Cell = const { Cell::new(0) }; +} + +#[derive(Debug)] +struct Sentinel; +impl<'de> Deserialize<'de> for Sentinel { + fn deserialize>(decoder: D) -> Result { + VISITS.set(VISITS.get().saturating_add(1)); + IgnoredAny::deserialize(decoder)?; + Ok(Self) + } +} + +fn rejected(bytes: &[u8], limits: Limits, expected: Failure) { + VISITS.set(0); + assert_eq!( + decode_with_limits::(bytes, limits).err(), + Some(expected) + ); + assert_eq!(VISITS.get(), 0, "typed allocation must follow preflight"); +} + +#[rstest] +#[case(&[], Failure::Syntax)] +#[case(&[0xff], Failure::Syntax)] +#[case(&[0x1f], Failure::Syntax)] +#[case(&[0x3f], Failure::Syntax)] +#[case(&[0x1c], Failure::Syntax)] +#[case(&[0x1d], Failure::Syntax)] +#[case(&[0x1e], Failure::Syntax)] +#[case(&[0x18], Failure::Syntax)] +#[case(&[0x19, 0x01], Failure::Syntax)] +#[case(&[0x1a, 0x00, 0x00, 0x00], Failure::Syntax)] +#[case(&[0x1b, 0x00, 0x00, 0x00, 0x00], Failure::Syntax)] +#[case(&[0x61], Failure::Syntax)] +#[case(&[0x81], Failure::Syntax)] +#[case(&[0xa1, 0x00], Failure::Syntax)] +#[case(&[0x9f], Failure::Syntax)] +#[case(&[0xbf, 0x61, b'a', 0xff], Failure::Syntax)] +#[case(&[0x81, 0xff], Failure::Syntax)] +#[case(&[0xa1, 0xff, 0x00], Failure::Profile)] +#[case(&[0x62, 0xc0, 0xaf], Failure::Syntax)] +#[case(&[0xa1, 0x61, 0xff, 0x00], Failure::Syntax)] +#[case(&[0xf8, 0x00], Failure::Syntax)] +#[case(&[0xf8, 0x1f], Failure::Syntax)] +#[case(&[0xfc], Failure::Syntax)] +#[case(&[0xfd], Failure::Syntax)] +#[case(&[0xfe], Failure::Syntax)] +#[case(&[0xc0, 0x60], Failure::Profile)] +#[case(&[0x5f, 0xff], Failure::Profile)] +#[case(&[0x7f, 0xff], Failure::Profile)] +#[case(&[0xa1, 0x80, 0x00], Failure::Profile)] +#[case(&[0xa1, 0x40, 0x00], Failure::Profile)] +#[case(&[0xa1, 0xf5, 0x00], Failure::Profile)] +#[case(&[0xa1, 0xf9, 0x00, 0x00, 0x00], Failure::Profile)] +#[case(&[0x00, 0x00], Failure::Trailing)] +#[case(&[0x00, 0xff], Failure::Trailing)] +#[case(&[0x80, 0x61, b'a'], Failure::Trailing)] +fn test_should_reject_hostile_structure_before_typed_visitors( + #[case] bytes: &[u8], + #[case] expected: Failure, +) { + rejected(bytes, LIMITS, expected); +} + +#[rstest] +#[case(&[0xa2, 0x61, b'x', 0x00, 0x61, b'x', 0x01])] +#[case(&[0xa2, 0x61, b'x', 0x00, 0x78, 0x01, b'x', 0x01])] +#[case(&[0xa2, 0x01, 0x00, 0x18, 0x01, 0x01])] +#[case(&[0xa2, 0x20, 0x00, 0x38, 0x00, 0x01])] +#[case(&[0xbf, 0x61, b'x', 0x00, 0x61, b'x', 0x01, 0xff])] +fn test_should_reject_semantically_duplicate_keys(#[case] bytes: &[u8]) { + rejected(bytes, LIMITS, Failure::Duplicate); +} + +fn numeric_header(major: u8, width: usize, value: u8) -> Vec { + match width { + 0 => vec![(major << 5) | value], + 1 => vec![(major << 5) | 0x18, value], + 2 => vec![(major << 5) | 0x19, 0, value], + 4 => vec![(major << 5) | 0x1a, 0, 0, 0, value], + 8 => vec![(major << 5) | 0x1b, 0, 0, 0, 0, 0, 0, 0, value], + _ => Vec::new(), + } +} + +#[rstest] +#[case(0)] +#[case(1)] +#[case(2)] +#[case(4)] +#[case(8)] +fn test_should_accept_every_header_width_without_requiring_canonical_order( + #[case] width: usize, +) -> TestResult { + assert_eq!(decode::(&numeric_header(0, width, 1))?, 1); + assert_eq!(decode::(&numeric_header(1, width, 1))?, -2); + let mut text = numeric_header(3, width, 1); + text.push(b'a'); + assert_eq!(decode::(&text)?, "a"); + let mut bytes = numeric_header(2, width, 1); + bytes.push(0xff); + assert_eq!(decode::(&bytes)?, Value::Bytes(vec![0xff])); + let mut array = numeric_header(4, width, 1); + array.push(0x00); + assert_eq!(decode::>(&array)?, vec![0]); + let mut map = numeric_header(5, width, 1); + map.extend_from_slice(&[0x61, b'a', 0x00]); + assert_eq!(decode::>(&map)?.get("a"), Some(&0)); + Ok(()) +} + +#[rstest] +#[case(&[0xf4], Value::Bool(false))] +#[case(&[0xf5], Value::Bool(true))] +#[case(&[0xf6], Value::Null)] +#[case(&[0x40], Value::Bytes(vec![]))] +#[case(&[0x60], Value::Text(String::new()))] +#[case(&[0x80], Value::Array(vec![]))] +#[case(&[0xa0], Value::Map(vec![]))] +#[case(&[0x9f, 0x00, 0x01, 0xff], Value::Array(vec![0.into(), 1.into()]))] +#[case(&[0xbf, 0x00, 0x00, 0xff], Value::Map(vec![(0.into(), 0.into())]))] +#[case(&[0xf9, 0x3e, 0x00], Value::Float(1.5))] +#[case(&[0xfa, 0x3f, 0xc0, 0x00, 0x00], Value::Float(1.5))] +#[case(&[0xfb, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], Value::Float(1.5))] +fn test_should_preserve_supported_cbor_values( + #[case] bytes: &[u8], + #[case] expected: Value, +) -> TestResult { + assert_eq!(decode::(bytes)?, expected); + Ok(()) +} + +#[test] +fn test_should_allow_integer_keyed_versions_and_nested_key_reuse() -> TestResult { + let value = BTreeMap::from([ + (0_u64, BTreeMap::from([("same".to_owned(), vec![0, 1, 2])])), + ( + u64::MAX, + BTreeMap::from([("same".to_owned(), vec![3, 4, 5])]), + ), + ]); + assert_eq!( + decode::>>>(&encode(&value)?)?, + value + ); + let negative: BTreeMap = decode(&[0xa2, 0x20, 0x01, 0x00, 0x02])?; + assert_eq!(negative, BTreeMap::from([(-1, 1), (0, 2)])); + Ok(()) +} + +#[test] +fn test_should_bound_section_bytes_at_the_exact_boundary() -> TestResult { + let bytes = [0x61, b'a']; + let limits = Limits { bytes: 2, ..LIMITS }; + assert_eq!(decode_with_limits::(&bytes, limits)?, "a"); + assert_eq!(encode_with_limits(&"a", limits)?, bytes); + rejected(&bytes, Limits { bytes: 1, ..limits }, Failure::Bytes); + assert_eq!( + encode_with_limits(&"a", Limits { bytes: 1, ..limits }), + Err(Failure::Bytes) + ); + Ok(()) +} + +#[test] +fn test_should_bound_all_aggregate_nodes_including_keys_and_containers() -> TestResult { + // The root map, key, array and two scalar elements occupy five nodes. + let bytes = [0xa1, 0x61, b'k', 0x82, 0x00, 0x01]; + let limits = Limits { nodes: 5, ..LIMITS }; + assert!(decode_with_limits::(&bytes, limits).is_ok()); + rejected(&bytes, Limits { nodes: 4, ..limits }, Failure::Nodes); + rejected(&[0x00], Limits { nodes: 0, ..limits }, Failure::Nodes); + Scan::validate(&[0x00], Limits { nodes: 1, ..limits })?; + Ok(()) +} + +#[rstest] +#[case(&[0x82, 0x00, 0x01])] +#[case(&[0x9f, 0x00, 0x01, 0xff])] +#[case(&[0xa2, 0x00, 0x00, 0x01, 0x01])] +#[case(&[0xbf, 0x00, 0x00, 0x01, 0x01, 0xff])] +fn test_should_bound_definite_and_indefinite_collection_counts(#[case] bytes: &[u8]) { + let limits = Limits { + collection: 2, + ..LIMITS + }; + assert!(decode_with_limits::(bytes, limits).is_ok()); + rejected( + bytes, + Limits { + collection: 1, + ..limits + }, + Failure::Collection, + ); +} + +#[test] +fn test_should_bound_actual_container_depth_without_counting_scalars() -> TestResult { + Scan::validate(&[0x00], Limits { depth: 0, ..LIMITS })?; + rejected(&[0x80], Limits { depth: 0, ..LIMITS }, Failure::Depth); + let limits = Limits { depth: 2, ..LIMITS }; + Scan::validate(&[0x81, 0x81, 0x00], limits)?; + rejected(&[0x81, 0x81, 0x80], limits, Failure::Depth); + let mut bytes = vec![0x81; LIMITS.depth]; + bytes.push(0x00); + let _: Value = decode(&bytes)?; + bytes.insert(0, 0x81); + rejected(&bytes, LIMITS, Failure::Depth); + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +enum Tree { + Leaf, + Seq(Vec), +} + +#[test] +fn test_should_round_trip_derived_enums_at_exact_wire_depth_with_typed_headroom() -> TestResult { + let mut value = Tree::Leaf; + for _ in 0..64 { + value = Tree::Seq(vec![value]); + } + let bytes = encode(&value)?; + Scan::validate(&bytes, LIMITS)?; + // W190: a scalar Leaf also consumes the typed decoder's enum guard. + assert!( + ciborium::de::from_reader_with_recursion_limit::(bytes.as_slice(), 128).is_err() + ); + assert_eq!(decode::(&bytes)?, value); + Ok(()) +} + +#[rstest] +#[case(&[0x63, b'a', b'b', b'c'])] +#[case(&[0x43, 0x00, 0x01, 0xff])] +fn test_should_bound_definite_scalar_lengths_before_content(#[case] bytes: &[u8]) -> TestResult { + Scan::validate( + bytes, + Limits { + scalar: 3, + ..LIMITS + }, + )?; + rejected( + bytes, + Limits { + scalar: 2, + ..LIMITS + }, + Failure::Scalar, + ); + let claim = numeric_header(3, 8, 3); + rejected( + &claim, + Limits { + scalar: 2, + ..LIMITS + }, + Failure::Scalar, + ); + Ok(()) +} + +#[test] +fn test_should_bound_decoded_text_keys_in_bytes() -> TestResult { + // The Unicode key contains two bytes, one character. + let bytes = [0xa1, 0x62, 0xc3, 0xa9, 0x00]; + Scan::validate(&bytes, Limits { key: 2, ..LIMITS })?; + rejected(&bytes, Limits { key: 1, ..LIMITS }, Failure::Key); + let claim = [0xa1, 0x62, 0x00]; + rejected(&claim, Limits { key: 1, ..LIMITS }, Failure::Key); + Ok(()) +} + +#[test] +fn test_should_reject_extreme_external_lengths_without_allocation() { + for (major, expected) in [ + (2, Failure::Scalar), + (3, Failure::Scalar), + (4, Failure::Collection), + (5, Failure::Collection), + ] { + let mut bytes = vec![(major << 5) | 0x1b]; + bytes.extend_from_slice(&u64::MAX.to_be_bytes()); + rejected(&bytes, LIMITS, expected); + } +} + +#[test] +fn test_should_reject_all_truncated_prefixes_of_a_nested_value() -> TestResult { + let value = BTreeMap::from([("outer", vec![vec!["hello", "world"], vec!["é"]])]); + let bytes = encode(&value)?; + for len in 0..bytes.len() { + let prefix = bytes.get(..len).ok_or("invalid test prefix")?; + VISITS.set(0); + assert!(decode::(prefix).is_err()); + assert_eq!(VISITS.get(), 0); + } + Ok(()) +} + +#[test] +fn test_should_remove_attacker_data_from_schema_errors_and_sources() { + let secret = "private-snapshot-content"; + let mut raw = Vec::new(); + assert!(ciborium::into_writer(&secret, &mut raw).is_ok()); + let error = decode::(&raw).err(); + assert_eq!(error, Some(Failure::Schema)); + assert!(!format!("{error:?}").contains(secret)); + if let Some(error) = error { + assert!(error.source().is_none()); + assert_eq!(error.to_string(), "invalid snapshot CBOR schema"); + } +} + +struct FailingSerializer; +impl Serialize for FailingSerializer { + fn serialize(&self, _serializer: S) -> Result { + Err(S::Error::custom("private-serialization-content")) + } +} + +#[test] +fn test_should_redact_serializer_failures() { + let error = encode(&FailingSerializer).err(); + assert_eq!(error, Some(Failure::Encoding)); + assert!(!format!("{error:?}").contains("private-serialization-content")); +} + +struct ManyValues; +impl Serialize for ManyValues { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(None)?; + for _ in 0..1_000_000 { + VISITS.set(VISITS.get().saturating_add(1)); + sequence.serialize_element(&0_u8)?; + } + sequence.end() + } +} + +#[test] +fn test_should_stop_oversized_serialization_before_finishing_the_input() { + VISITS.set(0); + assert_eq!( + encode_with_limits(&ManyValues, Limits { bytes: 8, ..LIMITS }), + Err(Failure::Bytes) + ); + assert!(VISITS.get() <= 8); +} + +#[test] +fn test_should_keep_capped_writer_bytes_unchanged_on_oversize() -> TestResult { + let mut writer = CappedWriter { + bytes: Vec::new(), + limit: 3, + failure: None, + }; + assert_eq!(writer.append(&[0, 1])?, 2); + assert_eq!(writer.append(&[2, 3]), Err(Failure::Bytes)); + assert_eq!(writer.bytes, [0, 1]); + assert_eq!(writer.append(&[2])?, 1); + assert_eq!(writer.bytes, [0, 1, 2]); + Ok(()) +} + +#[test] +fn test_should_apply_profile_checks_to_newly_encoded_values() { + assert_eq!( + encode(&Value::Tag(0, Box::new(Value::Text("date".into())))), + Err(Failure::Profile) + ); + assert_eq!( + encode(&Value::Map(vec![(Value::Bool(true), Value::Null)])), + Err(Failure::Profile) + ); + assert_eq!( + encode(&Value::Map(vec![ + (Value::Text("x".into()), Value::Null), + (Value::Text("x".into()), Value::Null) + ])), + Err(Failure::Duplicate) + ); + assert_eq!( + encode_with_limits( + &vec![0; 3], + Limits { + collection: 2, + ..LIMITS + } + ), + Err(Failure::Collection) + ); + assert_eq!( + encode_with_limits( + &"abc", + Limits { + scalar: 2, + ..LIMITS + } + ), + Err(Failure::Scalar) + ); +} + +struct Early; +impl<'de> Deserialize<'de> for Early { + fn deserialize>(_decoder: D) -> Result { + Ok(Self) + } +} + +#[test] +fn test_should_require_the_typed_reader_to_consume_the_single_root() { + assert!(matches!(decode::(&[0x00]), Err(Failure::Trailing))); +} + +#[test] +fn test_should_demonstrate_preflight_is_required_for_trailing_and_duplicate_input() -> TestResult { + assert_eq!(ciborium::from_reader::([0x00, 0x01].as_slice())?, 0); + rejected(&[0x00, 0x01], LIMITS, Failure::Trailing); + let duplicate = [0xa2, 0x61, b'x', 0x00, 0x61, b'x', 0x01]; + assert_eq!( + ciborium::from_reader::, _>(duplicate.as_slice())?.get("x"), + Some(&1) + ); + rejected(&duplicate, LIMITS, Failure::Duplicate); + Ok(()) +} + +fn profile_values() -> impl Strategy { + prop_oneof![ + any::().prop_map(Value::Bool), + any::().prop_map(|v| Value::Integer(v.into())), + "[a-zé水0-9]{0,32}".prop_map(Value::Text), + collection::vec(any::(), 0..64).prop_map(Value::Bytes), + Just(Value::Null), + ] + .prop_recursive(6, 128, 12, |inner| { + prop_oneof![ + collection::vec(inner.clone(), 0..12).prop_map(Value::Array), + collection::btree_map("[a-zA-Z0-9]{0,16}", inner, 0..8).prop_map(|map| Value::Map( + map.into_iter() + .map(|(key, value)| (Value::Text(key), value)) + .collect() + )), + ] + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn test_should_round_trip_bounded_nested_profile_values(value in profile_values()) { + let encoded = encode(&value); + prop_assert!(encoded.is_ok()); + if let Ok(bytes) = encoded { + let mut original = Vec::new(); + let original_result = ciborium::into_writer(&value, &mut original); + prop_assert!(original_result.is_ok()); + prop_assert_eq!(bytes.as_slice(), original.as_slice()); + let decoded = decode::(&bytes); + prop_assert!(decoded.is_ok()); + if let Ok(decoded) = decoded { + prop_assert_eq!(decoded, value); + } + } + } + + #[test] + fn test_should_never_deserialize_rejected_arbitrary_bytes(bytes in collection::vec(any::(), 0..4096)) { + let rejected = Scan::validate(&bytes, LIMITS).is_err(); + VISITS.set(0); + let _result = decode::(&bytes); + if rejected { + prop_assert_eq!(VISITS.get(), 0); + } + } +} diff --git a/apps/rustack/src/snapshot/fixtures/legacy-v2-meta-provenance.json b/apps/rustack/src/snapshot/fixtures/legacy-v2-meta-provenance.json new file mode 100644 index 00000000..cc60bc38 --- /dev/null +++ b/apps/rustack/src/snapshot/fixtures/legacy-v2-meta-provenance.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "fixture": "legacy-v2-meta.ss.zst", + "sha256": "fdc3d7c32dc2d84cd8bc5590bce9d1e894b680380ce54aef686df0f71ebcd870", + "generatedWith": { + "zstd": "0.13.3", + "zstd-safe": "7.3.0", + "zstd-sys": "2.1.0+zstd.1.5.7", + "compressionLevel": 1 + }, + "archive": { + "magicHex": "5253534e41500002", + "kind": 2, + "flags": 0, + "headerBytes": 36, + "fileBytes": 88, + "section": { + "kind": 1, + "flags": 0, + "offset": 64, + "length": 12, + "rows": 7, + "data": "legacy-state" + }, + "footer": "IEEE CRC32 and little-endian payload length" + }, + "generation": "Generated before dependency migration using the original resolved Cargo.lock codec cohort; no credentials or business payloads." +} diff --git a/apps/rustack/src/snapshot/fixtures/legacy-v2-meta.ss.zst b/apps/rustack/src/snapshot/fixtures/legacy-v2-meta.ss.zst new file mode 100644 index 00000000..8e7818fd Binary files /dev/null and b/apps/rustack/src/snapshot/fixtures/legacy-v2-meta.ss.zst differ diff --git a/apps/rustack/src/snapshot/image_tests.rs b/apps/rustack/src/snapshot/image_tests.rs new file mode 100644 index 00000000..4bf0f2f7 --- /dev/null +++ b/apps/rustack/src/snapshot/image_tests.rs @@ -0,0 +1,233 @@ +//! Image export budget failures must preserve both existing archive generations. + +use std::io::{Cursor, Write}; + +use anyhow::ensure; +use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; +use rustack_lambda_core::{ + artifact::Architecture, + config::LambdaConfig, + executor::{Executor, ExecutorBackend, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::InvocationContext, + storage::{LayerRecord, LayerStoreSnapshot, PolicyStatement}, +}; +use rustack_lambda_model::{ + input::{CreateFunctionInput, PublishVersionInput}, + types::FunctionCode, +}; +use zip::{ZipWriter, write::SimpleFileOptions}; + +use super::*; + +#[derive(Debug)] +struct MetadataOnlyExecutor; +#[async_trait] +impl Executor for MetadataOnlyExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn available(&self, _: &InvokeRequest) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported("metadata-only fixture".into())) + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported("metadata-only fixture".into())) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_preserve_generations_when_live_image_metadata_cannot_be_exported( + #[case] deep: bool, +) -> Result<()> { + let directory = tempfile::tempdir()?; + let config = SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("image-export".to_owned())?, + }; + config.save(&RuntimeProviders::default(), "good").await?; + let previous = directory.path().join(".image-export.previous"); + fs::create_dir_all(&previous).await?; + let original = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + fs::write(previous.join(MANIFEST_FILE), &original).await?; + let provider = Arc::new(RustackLambda::with_executor( + rustack_lambda_core::storage::FunctionStore::new(directory.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..LambdaConfig::default() + }, + Arc::new(MetadataOnlyExecutor), + )); + let mut layer: LayerRecord = serde_json::from_value( + serde_json::json!({"name":"bad","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:bad","nextVersion":2, + "versions":{"1":{"version":1,"description":"","compatibleRuntimes":[],"compatibleArchitectures":[],"licenseInfo":null,"codeSha256":"unused","codeSize":0,"createdDate":"2026-10-27T00:00:00Z","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:bad","layerVersionArn":"arn:aws:lambda:us-east-1:000000000000:layer:bad:1","policy":{"statements":[]}}}}), + )?; + let version = layer + .versions + .get_mut(&1) + .context("layer fixture version")?; + if deep { + let mut condition = serde_json::Value::Null; + for _ in 0..50_000 { + condition = serde_json::Value::Array(vec![condition]); + } + version.policy.statements.push(PolicyStatement { + sid: "deep".into(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:GetLayerVersion".into(), + resource: "*".into(), + condition: Some(condition), + }); + } else { + version.description = "x".repeat(20 * 1024 * 1024); + } + provider.layer_store().import_snapshot(LayerStoreSnapshot { + layers: vec![layer], + }); + let mut providers = RuntimeProviders::default(); + providers.register_lambda(provider); + let error = config + .save(&providers, "bad") + .await + .err() + .context("accepted oversized/deep live Image export")?; + assert!(format!("{error:#}").contains("Invalid or oversized Docker archive metadata")); + assert_eq!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?, + original + ); + assert_eq!(fs::read(previous.join(MANIFEST_FILE)).await?, original); + Ok(()) +} + +#[derive(Debug)] +struct ManagedZipMetadataExecutor; +#[async_trait] +impl Executor for ManagedZipMetadataExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn docker_zip_architecture(&self) -> Option { + Some(Architecture::Arm64) + } + fn zip_available(&self, architecture: Architecture) -> Result<(), ExecutorError> { + if architecture == Architecture::Arm64 { + Ok(()) + } else { + Err(ExecutorError::Unsupported( + "controlled ZIP architecture".into(), + )) + } + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported("snapshot-only fixture".into())) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +fn deep_tree_zip() -> Result { + let mut elf = vec![0_u8; 64]; + elf.get_mut(..7) + .context("ELF prefix")? + .copy_from_slice(b"\x7fELF\x02\x01\x01"); + elf.get_mut(16..18) + .context("ELF type")? + .copy_from_slice(&2_u16.to_le_bytes()); + elf.get_mut(18..20) + .context("ELF architecture")? + .copy_from_slice(&183_u16.to_le_bytes()); + elf.get_mut(20..24) + .context("ELF version")? + .copy_from_slice(&1_u32.to_le_bytes()); + elf.get_mut(52..54) + .context("ELF header")? + .copy_from_slice(&64_u16.to_le_bytes()); + let mut zip = ZipWriter::new(Cursor::new(Vec::new())); + zip.start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + )?; + zip.write_all(&elf)?; + for branch in 0..16 { + let path = format!("branch{branch}/{}/payload", vec!["d"; 32].join("/")); + zip.start_file(path, SimpleFileOptions::default())?; + } + Ok(STANDARD.encode(zip.finish()?.into_inner())) +} +#[tokio::test] +async fn test_should_preserve_both_generations_when_shared_zip_revisions_exceed_restore_node_budget() +-> Result<()> { + let directory = tempfile::tempdir()?; + let config = SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("zip-export".to_owned())?, + }; + config.save(&RuntimeProviders::default(), "good").await?; + let previous = directory.path().join(".zip-export.previous"); + fs::create_dir_all(&previous).await?; + let original = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + fs::write(previous.join(MANIFEST_FILE), &original).await?; + let provider = Arc::new(RustackLambda::with_executor( + rustack_lambda_core::storage::FunctionStore::new(directory.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ManagedZipMetadataExecutor), + )); + provider + .create_function(CreateFunctionInput { + function_name: "bounded-zip".into(), + runtime: Some("provided.al2023".into()), + handler: Some("handler.entry".into()), + role: "arn:aws:iam::000000000000:role/test".into(), + architectures: Some(vec!["arm64".into()]), + code: FunctionCode { + zip_file: Some(deep_tree_zip()?), + ..Default::default() + }, + ..Default::default() + }) + .await?; + // Original artifact ownership is shared; each saved revision still charges nodes. + for _ in 0..39 { + provider.publish_version("bounded-zip", &PublishVersionInput::default())?; + } + let mut providers = RuntimeProviders::default(); + providers.register_lambda(provider.clone()); + let error = config + .save(&providers, "over-profile") + .await + .err() + .context("accepted unrestorable ZIP generation")?; + ensure!( + format!("{error:#}").contains("Invalid or oversized Docker ZIP archive candidate"), + "wrong producer guard rejected ZIP archive" + ); + ensure!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await? == original, + "current generation replaced" + ); + ensure!( + fs::read(previous.join(MANIFEST_FILE)).await? == original, + "previous generation replaced" + ); + provider + .shutdown_before(Deadline::now() + Duration::from_secs(10)) + .await?; + Ok(()) +} diff --git a/apps/rustack/src/snapshot/lease.rs b/apps/rustack/src/snapshot/lease.rs new file mode 100644 index 00000000..259730c9 --- /dev/null +++ b/apps/rustack/src/snapshot/lease.rs @@ -0,0 +1,123 @@ +//! Unique retained OS-lease acquisition survives observing timeout/drop. + +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::{io::ErrorKind, mem, sync::Arc}; + +use anyhow::{Context, Result, bail}; +#[cfg(unix)] +use nix::fcntl::OFlag; +#[cfg(test)] +use tokio::time::Duration; +use tokio::{ + task::JoinHandle, + time::{Instant, timeout_at}, +}; + +use super::{SnapshotConfig, SnapshotLease, task::Failure, worker::WorkerFailure}; + +#[derive(Debug)] +pub(crate) struct LeaseAcquisition { + join: Option>>, + result: Option, Failure>>, +} +impl SnapshotConfig { + /// Start the same blocking acquisition once; keep its owner above timeouts. + pub(crate) fn begin_lease(&self) -> LeaseAcquisition { + self.begin_owned_lease( + #[cfg(test)] + None, + ) + } + #[allow(clippy::disallowed_types, clippy::disallowed_methods)] // Existing lease std-IO boundary; Tokio has no advisory lock equivalent. + fn begin_owned_lease(&self, #[cfg(test)] gate: Option>) -> LeaseAcquisition { + let root = self.root.clone(); + let name = self.name.as_str().to_owned(); + let join = tokio::task::spawn_blocking(move || { + #[cfg(test)] + if let Some(gate) = &gate { + gate.enter(); + } + std::fs::create_dir_all(&root).context("create snapshot root for lease")?; + let path = root.join(format!(".{name}.lock")); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if !metadata.file_type().is_file() => { + bail!("snapshot lock must be a regular file") + } + Ok(_) => {} + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(error).context("inspect snapshot lease path"), + } + let mut options = std::fs::OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + options.custom_flags((OFlag::O_NONBLOCK | OFlag::O_NOFOLLOW).bits()); + } + let file = options.open(&path).context("open snapshot lease")?; + if !file + .metadata() + .context("inspect opened snapshot lease")? + .is_file() + { + bail!("snapshot lock must be a regular file"); + } + file.try_lock() + .context("snapshot is already owned by another runtime")?; + Ok(SnapshotLease { _file: file }) + }); + LeaseAcquisition { + join: Some(join), + result: None, + } + } + #[cfg(test)] + pub(crate) async fn acquire_lease(&self) -> Result> { + self.begin_lease() + .wait(Instant::now() + Duration::from_secs(5)) + .await + } +} +impl LeaseAcquisition { + pub(super) fn quarantine(&mut self) { + if let Some(join) = self.join.take() { + mem::forget(join); + } + if let Some(Ok(lease)) = self.result.take() { + lease.quarantine(); + } + } + pub(crate) async fn wait(&mut self, deadline: Instant) -> Result> { + if let Some(join) = &mut self.join { + let result = timeout_at(deadline, join) + .await + .context("snapshot lease acquisition observation expired")?; + self.join = None; + let result = result + .map_err(WorkerFailure::new) + .map_err(anyhow::Error::from) + .and_then(|result| result); + self.result = Some(result.map(Arc::new).map_err(Failure::new)); + if Instant::now() >= deadline { + bail!("snapshot lease acquisition observation expired"); + } + } + match &self.result { + Some(Ok(lease)) => Ok(lease.clone()), + Some(Err(error)) => Err(error.clone().into()), + None => bail!("snapshot lease acquisition result missing"), + } + } +} +impl Drop for LeaseAcquisition { + fn drop(&mut self) { + // At most one pending lock acquisition exists in main. A lost owner is + // not proof that its late-acquired OS lock is safe to release. + if let Some(join) = self.join.take() { + mem::forget(join); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/rustack/src/snapshot/lease/tests.rs b/apps/rustack/src/snapshot/lease/tests.rs new file mode 100644 index 00000000..8edbdc58 --- /dev/null +++ b/apps/rustack/src/snapshot/lease/tests.rs @@ -0,0 +1,76 @@ +//! Both inner deadline expiry and a dropped observer keep the exact acquisition. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use tokio::{sync::Notify, time::timeout_at}; + +use super::*; +use crate::snapshot::SnapshotName; + +#[derive(Debug, Default)] +pub(super) struct Gate { + entered: Notify, + released: AtomicBool, +} +impl Gate { + pub(super) fn enter(&self) { + self.entered.notify_one(); + while !self.released.load(Ordering::Acquire) { + std::thread::park_timeout(Duration::from_millis(1)); + } + } +} +#[derive(Debug)] +struct Release(Arc); +impl Drop for Release { + fn drop(&mut self) { + self.0.released.store(true, Ordering::Release); + } +} +#[tokio::test] +async fn test_should_retain_delayed_lease_acquisition_after_timeout_or_observer_drop() -> Result<()> +{ + for inner_deadline in [false, true] { + let directory = tempfile::tempdir()?; + let config = SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("lease".to_owned())?, + }; + let gate = Arc::new(Gate::default()); + let release = Release(gate.clone()); + let mut acquisition = config.begin_owned_lease(Some(gate.clone())); + timeout_at( + Instant::now() + Duration::from_secs(2), + gate.entered.notified(), + ) + .await?; + let short = Instant::now() + Duration::from_millis(5); + if inner_deadline { + assert!(acquisition.wait(short).await.is_err()); + } else { + assert!( + timeout_at( + short, + acquisition.wait(Instant::now() + Duration::from_secs(2)) + ) + .await + .is_err() + ); + } + assert!( + acquisition + .join + .as_ref() + .is_some_and(|join| !join.is_finished()) + ); + drop(release); + let lease = acquisition + .wait(Instant::now() + Duration::from_secs(2)) + .await?; + assert!(config.acquire_lease().await.is_err()); + drop(lease); + drop(acquisition); + let _next = config.acquire_lease().await?; + } + Ok(()) +} diff --git a/apps/rustack/src/snapshot/policy_tests.rs b/apps/rustack/src/snapshot/policy_tests.rs new file mode 100644 index 00000000..c3522c6c --- /dev/null +++ b/apps/rustack/src/snapshot/policy_tests.rs @@ -0,0 +1,74 @@ +//! A configured limit is not evidence of corruption of a committed generation. + +use super::*; + +async fn generation(root: &Path, directory: &str, version: &str, bytes: usize) -> Result { + let target = root.join(directory); + let meta = target.join("services/limited/meta.ss.zst"); + let stats = write_archive( + &meta, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new( + SECTION_STATE_CBOR, + to_cbor(&"x".repeat(bytes))?, + 1, + )], + ) + .await?; + let mut manifest = SnapshotManifest::new("budget", version)?; + manifest.add_service( + "limited".into(), + SnapshotServiceManifest { + kind: SnapshotKind::Resource, + meta_file: "services/limited/meta.ss.zst".into(), + data_file: None, + meta: stats, + data: None, + }, + ); + write_manifest(&target.join(MANIFEST_FILE), &manifest).await?; + Ok(target) +} +#[tokio::test] +async fn test_should_refuse_reduced_policy_without_replacing_valid_current_or_previous_generation() +-> Result<()> { + let directory = tempfile::tempdir()?; + let current = generation(directory.path(), "budget", "new", 4096).await?; + let previous = generation(directory.path(), ".budget.previous", "old", 1).await?; + let new_bytes = fs::read(current.join(MANIFEST_FILE)).await?; + let old_bytes = fs::read(previous.join(MANIFEST_FILE)).await?; + let error = recover_snapshot( + directory.path(), + ¤t, + "budget", + &BTreeMap::from([("limited", 4098)]), + ) + .await + .err() + .context("accepted reduced budget")?; + assert!(error.is::()); + assert_eq!(fs::read(current.join(MANIFEST_FILE)).await?, new_bytes); + assert_eq!(fs::read(previous.join(MANIFEST_FILE)).await?, old_bytes); + recover_snapshot( + directory.path(), + ¤t, + "budget", + &BTreeMap::from([("limited", 8192)]), + ) + .await?; + assert_eq!( + read_manifest(¤t.join(MANIFEST_FILE)) + .await? + .rustack_version, + "new" + ); + assert!(fs::try_exists(&previous).await?); + SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("budget".to_owned())?, + } + .load(&RuntimeProviders::default()) + .await?; + assert!(!fs::try_exists(previous).await?); + Ok(()) +} diff --git a/apps/rustack/src/snapshot/session.rs b/apps/rustack/src/snapshot/session.rs new file mode 100644 index 00000000..2c3a15d1 --- /dev/null +++ b/apps/rustack/src/snapshot/session.rs @@ -0,0 +1,129 @@ +//! Main keeps one session above all startup/save/final-cleanup observations. + +use std::fmt; + +use anyhow::{Result, bail}; +use tokio::time::Instant; + +use super::{RuntimeProviders, SnapshotConfig, SnapshotTask, lease::LeaseAcquisition}; + +#[derive(Debug)] +enum Operation { + Load(SnapshotTask), + Save(SnapshotTask), +} +impl Operation { + async fn wait(&mut self, deadline: Instant) -> Result<()> { + match self { + Self::Load(task) | Self::Save(task) => task.wait(deadline).await, + } + } +} +#[derive(Debug)] +struct Named { + config: SnapshotConfig, + lease: LeaseAcquisition, + transaction: Option, + loaded: bool, +} +pub(crate) struct SnapshotSession { + named: Option, +} +impl fmt::Debug for SnapshotSession { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SnapshotSession()") + } +} +impl SnapshotSession { + pub(crate) fn new(config: Option) -> Self { + Self { + named: config.map(|config| Named { + lease: config.begin_lease(), + config, + transaction: None, + loaded: false, + }), + } + } + pub(crate) async fn load( + &mut self, + providers: &RuntimeProviders, + deadline: Instant, + ) -> Result<()> { + let Some(named) = &mut self.named else { + return Ok(()); + }; + if named.loaded { + return Ok(()); + } + let lease = named.lease.wait(deadline).await?; + if named.transaction.is_none() { + if Instant::now() >= deadline { + bail!("snapshot startup observation expired"); + } + tracing::info!(coverage = ?providers.coverage(), "snapshot includes only declared state; other services and SQS messages are not persisted"); + named.transaction = Some(Operation::Load(SnapshotTask::start( + named.config.clone(), + providers.clone(), + lease, + None, + deadline, + ))); + } + match &mut named.transaction { + Some(Operation::Load(task)) => task.wait(deadline).await?, + _ => bail!("invalid snapshot startup transaction"), + } + named.loaded = true; + named.transaction = None; + Ok(()) + } + pub(crate) async fn save( + &mut self, + providers: &RuntimeProviders, + version: &'static str, + deadline: Instant, + ) -> Result<()> { + let Some(named) = &mut self.named else { + return Ok(()); + }; + if !named.loaded { + bail!("cannot publish snapshot after failed startup"); + } + let lease = named.lease.wait(deadline).await?; + if named.transaction.is_none() { + if Instant::now() >= deadline { + bail!("snapshot save observation expired"); + } + named.transaction = Some(Operation::Save(SnapshotTask::start( + named.config.clone(), + providers.clone(), + lease, + Some(version), + deadline, + ))); + } + match &mut named.transaction { + Some(Operation::Save(task)) => task.wait(deadline).await, + _ => bail!("invalid snapshot save transaction"), + } + } + pub(crate) async fn finish(&mut self, deadline: Instant) -> Result<()> { + let Some(named) = &mut self.named else { + return Ok(()); + }; + let lease = named.lease.wait(deadline).await; + if let Some(operation) = &mut named.transaction { + operation.wait(deadline).await?; + } + lease.map(|_| ()) + } + pub(crate) fn quarantine(&mut self) { + if let Some(named) = &mut self.named { + named.lease.quarantine(); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/rustack/src/snapshot/session/tests.rs b/apps/rustack/src/snapshot/session/tests.rs new file mode 100644 index 00000000..a8ee6c98 --- /dev/null +++ b/apps/rustack/src/snapshot/session/tests.rs @@ -0,0 +1,65 @@ +//! One named session owns acquisition and never reissues accepted transactions. + +use tokio::{fs, time::Duration}; + +use super::*; +use crate::snapshot::{MANIFEST_FILE, SnapshotName, read_manifest}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} +#[tokio::test] +async fn test_should_retain_named_ownership_and_observe_the_same_publication() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("session".to_owned())?, + }; + let providers = RuntimeProviders::default(); + let mut session = SnapshotSession::new(Some(config.clone())); + session.load(&providers, deadline()).await?; + session.load(&providers, deadline()).await?; + assert!(config.acquire_lease().await.is_err()); + session.save(&providers, "original", deadline()).await?; + let original = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + session + .save(&providers, "must-not-reissue", deadline()) + .await?; + assert_eq!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?, + original + ); + assert_eq!( + read_manifest(&config.snapshot_dir().join(MANIFEST_FILE)) + .await? + .rustack_version, + "original" + ); + session.finish(deadline()).await?; + drop(session); + let _next = config.acquire_lease().await?; + Ok(()) +} +#[tokio::test] +async fn test_should_refuse_publication_after_failed_startup() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("session".to_owned())?, + }; + let providers = RuntimeProviders::default(); + config.save(&providers, "old").await?; + let manifest = config.snapshot_dir().join(MANIFEST_FILE); + fs::write(&manifest, b"corrupt generation").await?; + let mut session = SnapshotSession::new(Some(config)); + assert!(session.load(&providers, deadline()).await.is_err()); + assert!( + session + .save(&providers, "invalid", deadline()) + .await + .is_err() + ); + assert_eq!(fs::read(&manifest).await?, b"corrupt generation"); + assert!(session.finish(deadline()).await.is_err()); + Ok(()) +} diff --git a/apps/rustack/src/snapshot/task.rs b/apps/rustack/src/snapshot/task.rs new file mode 100644 index 00000000..2958ab86 --- /dev/null +++ b/apps/rustack/src/snapshot/task.rs @@ -0,0 +1,121 @@ +//! An observing deadline never aborts the snapshot transaction or its IO joins. + +use std::{error::Error, fmt, mem, sync::Arc}; + +use anyhow::{Context, Result}; +use tokio::{ + task::JoinHandle, + time::{Instant, timeout_at}, +}; + +use super::{RuntimeProviders, SnapshotConfig, SnapshotLease, worker::WorkerFailure}; + +pub(crate) struct SnapshotTask { + join: Option>>, + failure: Option, +} +impl fmt::Debug for SnapshotTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SnapshotTask()") + } +} +#[derive(Clone)] +pub(super) struct Failure(Arc); +impl Failure { + pub(super) fn new(error: anyhow::Error) -> Self { + Self(Arc::new(error)) + } +} +impl fmt::Debug for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl fmt::Display for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("snapshot transaction failed") + } +} +impl Error for Failure { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.0.as_ref().as_ref()) + } +} +struct Guard { + lease: Option>, + providers: Option, + finished: bool, +} +impl fmt::Debug for Guard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SnapshotGuard()") + } +} +impl Drop for Guard { + fn drop(&mut self) { + if !self.finished { + // A panic/cancellation cannot prove sibling filesystem retirement. + if let Some(lease) = self.lease.take() { + mem::forget(lease); + } + if let Some(providers) = self.providers.take() { + mem::forget(providers); + } + } + } +} +impl SnapshotTask { + pub(crate) fn start( + config: SnapshotConfig, + providers: RuntimeProviders, + lease: Arc, + version: Option<&'static str>, + deadline: Instant, + ) -> Self { + let join = tokio::spawn(async move { + let mut guard = Guard { + lease: Some(lease), + providers: Some(providers), + finished: false, + }; + let providers = guard + .providers + .as_ref() + .context("snapshot transaction providers missing")?; + let result = match version { + Some(version) => config.save(providers, version).await, + None => config.load_before(providers, deadline).await, + }; + guard.finished = true; + result + }); + Self { + join: Some(join), + failure: None, + } + } + pub(crate) async fn wait(&mut self, deadline: Instant) -> Result<()> { + if let Some(join) = &mut self.join { + let result = timeout_at(deadline, join) + .await + .context("snapshot transaction observation expired")?; + self.join = None; + let result = result + .map_err(WorkerFailure::new) + .map_err(anyhow::Error::from) + .and_then(|result| result); + if let Err(error) = result { + self.failure = Some(Failure::new(error)); + } + if Instant::now() >= deadline { + anyhow::bail!("snapshot transaction observation expired"); + } + } + self.failure + .as_ref() + .map_or(Ok(()), |error| Err(error.clone().into())) + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/rustack/src/snapshot/task/tests.rs b/apps/rustack/src/snapshot/task/tests.rs new file mode 100644 index 00000000..9a99339d --- /dev/null +++ b/apps/rustack/src/snapshot/task/tests.rs @@ -0,0 +1,240 @@ +//! Retained transaction observation and complete sibling retirement on error. + +use std::path::Path; + +use async_trait::async_trait; +use tokio::{ + fs, + sync::{Notify, watch}, + time::{Duration, advance, timeout}, +}; + +use super::*; +use crate::snapshot::{MANIFEST_FILE, SnapshotName, SnapshotService, to_cbor}; + +#[derive(Debug)] +struct GateService { + name: &'static str, + fail: bool, + entered: Arc, + release: watch::Receiver, +} +impl GateService { + async fn hold(&self) -> Result<()> { + self.entered.notify_one(); + let mut release = self.release.clone(); + release + .wait_for(|ready| *ready) + .await + .context("test gate lost")?; + Ok(()) + } +} +#[async_trait] +impl SnapshotService for GateService { + fn service_name(&self) -> &'static str { + self.name + } + async fn save_meta(&self, _directory: &Path) -> Result> { + if self.fail { + anyhow::bail!("controlled service failure"); + } + self.hold().await?; + to_cbor(&()) + } + async fn load_meta(&self, _metadata: Vec, directory: &Path) -> Result<()> { + if self.fail { + anyhow::bail!("controlled service failure"); + } + self.hold().await?; + anyhow::ensure!( + fs::metadata(directory).await?.is_dir(), + "staging removed before sibling retirement" + ); + Ok(()) + } +} +fn config(directory: &tempfile::TempDir) -> Result { + Ok(SnapshotConfig { + root: directory.path().to_owned(), + name: SnapshotName::try_from("transaction".to_owned())?, + }) +} +#[tokio::test] +async fn test_should_keep_transaction_and_named_lease_after_observer_expiry() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let entered = Arc::new(Notify::new()); + let (release, receiver) = watch::channel(false); + let providers = RuntimeProviders { + services: vec![Arc::new(GateService { + name: "owned", + fail: false, + entered: entered.clone(), + release: receiver, + })], + }; + let lease = config.acquire_lease().await?; + let mut task = SnapshotTask::start( + config.clone(), + providers, + lease, + Some("test"), + Instant::now() + Duration::from_secs(30), + ); + timeout(Duration::from_secs(2), entered.notified()).await?; + assert!( + task.wait(Instant::now() + Duration::from_millis(5)) + .await + .is_err() + ); + assert!(task.join.as_ref().is_some_and(|join| !join.is_finished())); + assert!(config.acquire_lease().await.is_err()); + release.send_replace(true); + task.wait(Instant::now() + Duration::from_secs(5)).await?; + let _new = config.acquire_lease().await?; + Ok(()) +} +#[tokio::test(start_paused = true)] +async fn test_should_cache_ready_completion_but_reject_first_late_observation() -> Result<()> { + let entered = Arc::new(Notify::new()); + let signal = entered.clone(); + let mut task = SnapshotTask { + join: Some(tokio::spawn(async move { + signal.notify_one(); + Ok(()) + })), + failure: None, + }; + let deadline = Instant::now() + Duration::from_secs(1); + entered.notified().await; + assert!(task.join.as_ref().is_some_and(JoinHandle::is_finished)); + advance(Duration::from_secs(2)).await; + assert!(task.wait(deadline).await.is_err()); + assert!(task.join.is_none()); + task.wait(Instant::now() + Duration::from_secs(1)).await?; + Ok(()) +} +#[tokio::test] +async fn test_should_drain_started_load_and_save_siblings_before_returning_first_error() +-> Result<()> { + for saving in [false, true] { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let (ready, receiver) = watch::channel(true); + let original = RuntimeProviders { + services: ["broken", "slow"] + .into_iter() + .map(|name| { + Arc::new(GateService { + name, + fail: false, + entered: Arc::new(Notify::new()), + release: receiver.clone(), + }) as Arc + }) + .collect(), + }; + config.save(&original, "old").await?; + let original_manifest = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + let entered = Arc::new(Notify::new()); + let (release, receiver) = watch::channel(false); + let replacement = RuntimeProviders { + services: vec![ + Arc::new(GateService { + name: "broken", + fail: true, + entered: Arc::new(Notify::new()), + release: receiver.clone(), + }), + Arc::new(GateService { + name: "slow", + fail: false, + entered: entered.clone(), + release: receiver, + }), + ], + }; + let requested = config.clone(); + let mut task = tokio::spawn(async move { + if saving { + requested.save(&replacement, "invalid").await + } else { + requested.load(&replacement).await + } + }); + timeout(Duration::from_secs(2), entered.notified()).await?; + assert!(timeout(Duration::from_millis(5), &mut task).await.is_err()); + assert!(!task.is_finished(), "returned before sibling retirement"); + release.send_replace(true); + assert!(timeout(Duration::from_secs(5), task).await??.is_err()); + assert_eq!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?, + original_manifest + ); + drop(ready); + } + Ok(()) +} + +#[tokio::test] +async fn test_should_join_expired_domain_import_without_retiring_previous_generation() -> Result<()> +{ + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let (_, ready) = watch::channel(true); + let source = RuntimeProviders { + services: vec![Arc::new(GateService { + name: "owned", + fail: false, + entered: Arc::new(Notify::new()), + release: ready, + })], + }; + config.save(&source, "current").await?; + let previous = directory.path().join(".transaction.previous"); + fs::create_dir_all(previous.join("services/owned")).await?; + let files = [MANIFEST_FILE, "services/owned/meta.ss.zst"]; + let mut original = Vec::new(); + for file in files { + let bytes = fs::read(config.snapshot_dir().join(file)).await?; + fs::write(previous.join(file), &bytes).await?; + original.push(bytes); + } + let entered = Arc::new(Notify::new()); + let (release, wait) = watch::channel(false); + let target = RuntimeProviders { + services: vec![Arc::new(GateService { + name: "owned", + fail: false, + entered: entered.clone(), + release: wait, + })], + }; + let lease = config.acquire_lease().await?; + let deadline = Instant::now() + Duration::from_secs(3600); + let mut task = SnapshotTask::start(config.clone(), target, lease, None, deadline); + timeout(Duration::from_secs(10), entered.notified()).await?; + // Advance only after positive import entry, then resume real IO observation. + tokio::time::pause(); + advance(Duration::from_secs(3601)).await; + tokio::time::resume(); + assert!(task.wait(Instant::now()).await.is_err()); + assert!(task.join.is_some()); + release.send_replace(true); + assert!( + task.wait(Instant::now() + Duration::from_secs(10)) + .await + .is_err() + ); + assert!( + task.join.is_none(), + "transaction must join rather than merely expire its observer" + ); + assert!(task.failure.is_some()); + for (file, bytes) in files.into_iter().zip(original) { + assert_eq!(fs::read(config.snapshot_dir().join(file)).await?, bytes); + assert_eq!(fs::read(previous.join(file)).await?, bytes); + } + Ok(()) +} diff --git a/apps/rustack/src/snapshot/worker.rs b/apps/rustack/src/snapshot/worker.rs new file mode 100644 index 00000000..6e983a8c --- /dev/null +++ b/apps/rustack/src/snapshot/worker.rs @@ -0,0 +1,77 @@ +//! Retain typed worker evidence without displaying potentially secret panic text. + +use std::{error::Error, fmt}; + +use tokio::task::JoinError; + +pub(super) struct WorkerFailure { + source: JoinError, +} +impl WorkerFailure { + pub(super) fn new(source: JoinError) -> Self { + Self { source } + } + /// Explicit typed inspection; raw `JoinError` Display must not reach diagnostics. + pub(super) fn join_error(&self) -> &JoinError { + &self.source + } +} +impl fmt::Debug for WorkerFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl fmt::Display for WorkerFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(if self.join_error().is_panic() { + "snapshot worker panicked" + } else { + "snapshot worker cancelled" + }) + } +} +// The original is retained for explicit typed inspection. Chaining it would leak +// JoinError's panic payload through main's formatted anyhow diagnostic chain. +impl Error for WorkerFailure {} + +#[cfg(test)] +mod tests { + use std::future::pending; + + use anyhow::{Context, Result}; + + use super::*; + + fn panic_source() { + panic!("private-panic-payload"); + } + #[tokio::test] + async fn test_should_retain_panic_and_cancellation_without_exposing_panic_text() -> Result<()> { + for panic in [false, true] { + let task = tokio::spawn(async move { + if panic { + panic_source(); + } else { + pending::<()>().await; + } + }); + if !panic { + task.abort(); + } + let original = task.await.err().context("missing controlled failure")?; + let failure = WorkerFailure::new(original); + assert_eq!(failure.join_error().is_panic(), panic); + assert_eq!(failure.join_error().is_cancelled(), !panic); + let error = anyhow::Error::new(failure).context("outer archive owner"); + for diagnostic in [ + format!("{error}"), + format!("{error:#}"), + format!("{error:?}"), + ] { + assert!(!diagnostic.contains("private-panic-payload")); + } + assert!(error.downcast_ref::().is_some()); + } + Ok(()) + } +} diff --git a/apps/rustack/src/snapshot/workflows.rs b/apps/rustack/src/snapshot/workflows.rs new file mode 100644 index 00000000..c8d8dd05 --- /dev/null +++ b/apps/rustack/src/snapshot/workflows.rs @@ -0,0 +1,287 @@ +//! One retained archive operation/CPU/IO lane above observer cancellation. + +use std::{ + error::Error, + fmt, mem, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use rustack_stepfunctions_core::{ + PreparedSnapshot, Registry, RegistryError, RegistryReceipt, RegistryResult, RegistrySnapshot, + SnapshotLimits, +}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; + +use super::{ + SnapshotKind, SnapshotService, + archive::{read_file_bounded, to_cbor_bounded}, + decode_state, + worker::WorkerFailure, +}; + +mod io; +mod owner; + +const HISTORY_FILE: &str = "history.cbor"; + +#[derive(Clone)] +pub(crate) struct WorkflowArchive { + sender: mpsc::Sender, + capacity: Arc, + closing: watch::Sender, + ready: watch::Receiver, + bytes: usize, + io_budget: Duration, +} +impl fmt::Debug for WorkflowArchive { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("WorkflowArchive()") + } +} +#[derive(Debug)] +pub(crate) struct ArchiveSupervisor { + closing: watch::Sender, + join: Option>>, + failure: Option, +} +#[derive(Clone)] +struct Failure(Arc); +impl fmt::Debug for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl fmt::Display for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("workflow archive owner failed") + } +} +impl Error for Failure { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.0.as_ref().as_ref()) + } +} +enum Command { + Save(PathBuf), + Load(Vec, PathBuf), +} +impl fmt::Debug for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ArchiveCommand()") + } +} +#[derive(Debug)] +struct Envelope { + command: Option, + deadline: Instant, + reply: oneshot::Sender>, + permit: OwnedSemaphorePermit, +} +struct Reply { + bytes: Vec, + _permit: OwnedSemaphorePermit, +} +impl fmt::Debug for Reply { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ArchiveReply()") + } +} +enum Materialized { + Saved { metadata: Vec, history: Vec }, + Prepared(PreparedSnapshot), +} +impl fmt::Debug for Materialized { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ArchiveMaterialized()") + } +} +#[derive(Debug)] +struct Owner { + registry: Registry, + bytes: usize, + inbox: mpsc::Receiver, + closing: watch::Sender, + ready: watch::Sender, + active: Option, + cpu: Option>>, + io: Option>>, + receipt: Option, + fatal: bool, + #[cfg(test)] + hooks: Option>, +} + +impl WorkflowArchive { + pub(crate) fn new( + registry: Registry, + bytes: usize, + io_budget: Duration, + ) -> Result<(Self, ArchiveSupervisor)> { + Self::new_owned( + registry, + bytes, + io_budget, + #[cfg(test)] + None, + ) + } + fn new_owned( + registry: Registry, + bytes: usize, + io_budget: Duration, + #[cfg(test)] hooks: Option>, + ) -> Result<(Self, ArchiveSupervisor)> { + SnapshotLimits::try_from(bytes)?; + if io_budget.is_zero() { + bail!("invalid workflow archive IO deadline"); + } + let (sender, inbox) = mpsc::channel(1); + let (closing, _) = watch::channel(false); + let (ready, status) = watch::channel(true); + let owner = Owner { + registry, + bytes, + inbox, + closing: closing.clone(), + ready, + active: None, + cpu: None, + io: None, + receipt: None, + fatal: false, + #[cfg(test)] + hooks, + }; + let join = tokio::spawn(owner.run()); + Ok(( + Self { + sender, + capacity: Arc::new(Semaphore::new(1)), + closing: closing.clone(), + ready: status, + bytes, + io_budget, + }, + ArchiveSupervisor { + closing, + join: Some(join), + failure: None, + }, + )) + } + pub(crate) fn is_ready(&self) -> bool { + !*self.closing.borrow() && *self.ready.borrow() + } + async fn execute(&self, command: Command) -> Result> { + if !self.is_ready() { + bail!("workflow archive owner unavailable"); + } + let permit = self + .capacity + .clone() + .try_acquire_owned() + .context("workflow archive capacity exhausted")?; + let deadline = Instant::now() + .checked_add(self.io_budget) + .context("workflow archive deadline overflow")?; + let (reply, receipt) = oneshot::channel(); + self.sender + .try_send(Envelope { + command: Some(command), + deadline, + reply, + permit, + }) + .map_err(|_| anyhow::anyhow!("workflow archive admission unavailable"))?; + let response = receipt + .await + .context("workflow archive reply owner lost")??; + Ok(response.bytes) + } +} +impl ArchiveSupervisor { + pub(crate) fn close(&self) { + self.closing.send_replace(true); + } + pub(crate) async fn shutdown(&mut self, deadline: Instant) -> Result<()> { + self.close(); + if let Some(join) = &mut self.join { + let result = timeout_at(deadline, join) + .await + .context("workflow archive shutdown observation expired")?; + self.join = None; + let result = result + .map_err(WorkerFailure::new) + .map_err(anyhow::Error::from) + .and_then(|result| result); + if let Err(error) = result { + self.failure = Some(Failure(Arc::new(error))); + } + } + self.failure + .as_ref() + .map_or(Ok(()), |error| Err(error.clone().into())) + } +} +impl Drop for ArchiveSupervisor { + fn drop(&mut self) { + // The retained root keeps accepted CPU/IO/commit work until joined completion. + self.close(); + } +} + +#[async_trait] +impl SnapshotService for WorkflowArchive { + fn service_name(&self) -> &'static str { + "stepfunctions" + } + fn snapshot_kind(&self) -> SnapshotKind { + SnapshotKind::Data + } + fn payload_limit(&self) -> Option { + Some(self.bytes) + } + fn data_file(&self) -> Option<&'static str> { + Some(HISTORY_FILE) + } + async fn save_meta(&self, staging: &Path) -> Result> { + self.execute(Command::Save(staging.to_owned())).await + } + async fn load_meta(&self, metadata: Vec, staging: &Path) -> Result<()> { + self.execute(Command::Load(metadata, staging.to_owned())) + .await?; + Ok(()) + } +} +impl Drop for Owner { + fn drop(&mut self) { + self.closing.send_replace(true); + self.ready.send_replace(false); + if self.active.is_some() { + // A root panic is not a cleanup receipt. Preserve the bounded job graph. + if let Some(cpu) = self.cpu.take() { + mem::forget(cpu); + } + if let Some(io) = self.io.take() { + mem::forget(io); + } + if let Some(receipt) = self.receipt.take() { + mem::forget(receipt); + } + if let Some(active) = self.active.take() { + mem::forget(active); + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/rustack/src/snapshot/workflows/io.rs b/apps/rustack/src/snapshot/workflows/io.rs new file mode 100644 index 00000000..e889033f --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/io.rs @@ -0,0 +1,59 @@ +//! Disk futures are retained in a joined task, including after observation expiry. + +use std::{ + fmt, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, bail}; +use tokio::fs; + +use super::{HISTORY_FILE, read_file_bounded}; + +pub(super) enum Output { + History(Vec), + Written, +} +impl fmt::Debug for Output { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ArchiveIoOutput()") + } +} +async fn check(directory: &Path, required: bool) -> Result<()> { + let mut entries = fs::read_dir(directory) + .await + .context("read workflow archive staging directory")?; + let first = entries + .next_entry() + .await + .context("read workflow history entry")?; + match first { + Some(entry) => { + if entry.file_name() != HISTORY_FILE + || !entry.file_type().await?.is_file() + || entries.next_entry().await?.is_some() + { + bail!("invalid workflow history payload directory"); + } + } + None if required => bail!("missing workflow history payload"), + None => {} + } + Ok(()) +} +pub(super) async fn read(directory: PathBuf, bytes: usize) -> Result { + check(&directory, true).await?; + let history = read_file_bounded( + &directory.join(HISTORY_FILE), + u64::try_from(bytes).context("workflow archive byte overflow")?, + ) + .await?; + Ok(Output::History(history)) +} +pub(super) async fn write(directory: PathBuf, bytes: Vec) -> Result { + check(&directory, false).await?; + fs::write(directory.join(HISTORY_FILE), bytes) + .await + .context("write workflow history payload")?; + Ok(Output::Written) +} diff --git a/apps/rustack/src/snapshot/workflows/owner.rs b/apps/rustack/src/snapshot/workflows/owner.rs new file mode 100644 index 00000000..494edd34 --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/owner.rs @@ -0,0 +1,213 @@ +//! Retained CPU/IO and accepted registry commits outlive individual observers. + +use std::{future::Future, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, bail}; +use rustack_stepfunctions_core::{RegistryHistories, RegistryMetadata}; +use tokio::{ + task::{spawn, spawn_blocking}, + time::{Instant, timeout_at}, +}; + +use super::{ + Command, Failure, Materialized, Owner, RegistryError, RegistryResult, RegistrySnapshot, Reply, + SnapshotLimits, WorkerFailure, decode_state, io, to_cbor_bounded, +}; + +impl Owner { + pub(super) async fn run(mut self) -> Result<()> { + let mut closing = self.closing.subscribe(); + loop { + if *closing.borrow() { + self.inbox.close(); + } + let envelope = tokio::select! { + biased; + _ = closing.changed(), if !self.inbox.is_closed() => { self.inbox.close(); continue; } + envelope = self.inbox.recv() => envelope, + }; + let Some(envelope) = envelope else { + break; + }; + self.active = Some(envelope); + let result = self.process().await; + let (result, fatal) = if self.fatal { + let error = match result { + Err(error) => error, + Ok(_) => anyhow::anyhow!("workflow archive fatal-state invariant failed"), + }; + let failure = Failure(Arc::new(error)); + (Err(failure.clone().into()), Some(failure)) + } else { + (result, None) + }; + let envelope = self + .active + .take() + .context("workflow archive envelope lost")?; + let reply = result.map(|bytes| Reply { + bytes, + _permit: envelope.permit, + }); + let _observed = envelope.reply.send(reply); + if let Some(failure) = fatal { + return Err(failure.into()); + } + } + Ok(()) + } + async fn process(&mut self) -> Result> { + let active = self + .active + .as_mut() + .context("workflow archive envelope missing")?; + let deadline = active.deadline; + let command = active + .command + .take() + .context("workflow archive command missing")?; + Self::check_deadline(deadline)?; + let registry = self.registry.clone(); + let bytes = self.bytes; + match command { + Command::Save(directory) => { + #[cfg(test)] + let hooks = self.hooks.clone(); + self.cpu = Some(spawn_blocking(move || { + let document = registry.export_snapshot(SnapshotLimits::try_from(bytes)?)?; + let (metadata, history) = document.into_archive_parts()?; + let result = Ok(Materialized::Saved { + metadata: to_cbor_bounded(&metadata, bytes)?, + history: to_cbor_bounded(&history, bytes)?, + }); + #[cfg(test)] + if let Some(hooks) = hooks { + hooks.cpu_wait(); + } + result + })); + let Materialized::Saved { metadata, history } = self.join_cpu().await? else { + bail!("invalid workflow archive save result"); + }; + Self::check_deadline(deadline)?; + self.start_io(io::write(directory, history)); + if !matches!(self.join_io(deadline).await?, io::Output::Written) { + bail!("invalid workflow archive write result"); + } + Ok(metadata) + } + Command::Load(metadata, directory) => { + if metadata.len() > bytes { + bail!("workflow archive metadata byte limit exceeded"); + } + self.start_io(io::read(directory, bytes)); + let io::Output::History(history) = self.join_io(deadline).await? else { + bail!("invalid workflow archive read result"); + }; + Self::check_deadline(deadline)?; + #[cfg(test)] + let hooks = self.hooks.clone(); + self.cpu = Some(spawn_blocking(move || { + let metadata: RegistryMetadata = decode_state(&metadata)?; + let history: RegistryHistories = decode_state(&history)?; + let document = registry.prepare_snapshot( + RegistrySnapshot::from_archive_parts(metadata, history)?, + SnapshotLimits::try_from(bytes)?, + )?; + #[cfg(test)] + if let Some(hooks) = hooks { + hooks.cpu_wait(); + } + Ok(Materialized::Prepared(document)) + })); + let Materialized::Prepared(prepared) = self.join_cpu().await? else { + bail!("invalid workflow archive prepare result"); + }; + Self::check_deadline(deadline)?; + self.receipt = Some(self.registry.restore_snapshot(prepared)?); + // Accepted commit ownership is independent of the command observer's deadline. + loop { + let receipt = self + .receipt + .as_mut() + .context("workflow archive restore receipt missing")?; + match receipt.wait(Instant::now() + Duration::from_secs(1)).await { + Err(RegistryError::Deadline) => {} + Ok(RegistryResult::Restored) => { + self.receipt = None; + return Ok(Vec::new()); + } + Ok(_) => { + self.receipt = None; + bail!("invalid workflow archive restore acknowledgement"); + } + Err(error) => { + self.receipt = None; + return Err(error.into()); + } + } + } + } + } + } + fn check_deadline(deadline: Instant) -> Result<()> { + if Instant::now() >= deadline { + bail!("workflow archive operation deadline exceeded"); + } + Ok(()) + } + fn start_io(&mut self, operation: impl Future> + Send + 'static) { + #[cfg(test)] + let task = { + let hooks = self.hooks.clone(); + spawn(async move { + if let Some(hooks) = hooks { + hooks.io_wait().await?; + } + operation.await + }) + }; + #[cfg(not(test))] + let task = spawn(operation); + self.io = Some(task); + } + async fn join_cpu(&mut self) -> Result { + let result = self + .cpu + .as_mut() + .context("workflow archive CPU join missing")? + .await; + self.cpu = None; + result.map_err(|error| { + self.fatal = true; + anyhow::Error::from(WorkerFailure::new(error)) + })? + } + async fn join_io(&mut self, deadline: Instant) -> Result { + let task = self + .io + .as_mut() + .context("workflow archive IO join missing")?; + let (expired, result) = if let Ok(result) = timeout_at(deadline, &mut *task).await { + (false, result) + } else { + self.ready.send_replace(false); + (true, task.await) + }; + self.io = None; + let result = result.map_err(|error| { + self.fatal = true; + anyhow::Error::from(WorkerFailure::new(error)) + })?; + if expired { + self.ready.send_replace(true); + bail!("workflow archive IO observation expired"); + } + let output = result?; + Self::check_deadline(deadline)?; + Ok(output) + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/rustack/src/snapshot/workflows/owner/tests.rs b/apps/rustack/src/snapshot/workflows/owner/tests.rs new file mode 100644 index 00000000..76fcef98 --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/owner/tests.rs @@ -0,0 +1,105 @@ +//! Ready-before-timer ordering and retained typed worker failures. + +use std::sync::Arc; + +use rustack_stepfunctions_core::{ + ExecutionLimits, ExecutionSupervisor, HistoryLimits, Registry, RegistryLimits, + RegistrySupervisor, RuntimeLimits, TenantScope, +}; +use tokio::{ + sync::{Notify, mpsc, watch}, + task::JoinHandle, + time::advance, +}; + +use super::*; + +fn owner() -> Result<(Owner, RegistrySupervisor)> { + let runtime = ExecutionSupervisor::new( + RuntimeLimits::default(), + HistoryLimits::new(25000, 1024 * 1024)?, + 8 * 1024 * 1024, + None, + )?; + let (registry, supervisor) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + )?; + let (_, inbox) = mpsc::channel(1); + let (closing, _) = watch::channel(false); + let (ready, _) = watch::channel(true); + Ok(( + Owner { + registry, + bytes: 1024 * 1024, + inbox, + closing, + ready, + active: None, + cpu: None, + io: None, + receipt: None, + fatal: false, + hooks: None, + }, + supervisor, + )) +} +#[tokio::test(start_paused = true)] +async fn test_should_reject_ready_io_first_observed_after_absolute_deadline() -> Result<()> { + let (mut owner, mut registry) = owner()?; + let gate = Arc::new(Notify::new()); + let signal = gate.clone(); + owner.io = Some(tokio::spawn(async move { + signal.notify_one(); + Ok(io::Output::Written) + })); + let deadline = Instant::now() + Duration::from_secs(1); + gate.notified().await; + assert!(owner.io.as_ref().is_some_and(JoinHandle::is_finished)); + advance(Duration::from_secs(2)).await; + assert!(owner.join_io(deadline).await.is_err()); + assert!(owner.io.is_none(), "physical task was not joined"); + drop(owner); + registry + .shutdown(Instant::now() + Duration::from_secs(1)) + .await?; + Ok(()) +} +fn panic_source() -> Result { + panic!("private-cpu-payload"); +} +fn panic_io() -> Result { + panic!("private-io-payload"); +} +#[tokio::test] +async fn test_should_retain_typed_cpu_and_io_failures() -> Result<()> { + for cpu in [false, true] { + let (mut owner, mut registry) = owner()?; + let result = if cpu { + owner.cpu = Some(spawn_blocking(panic_source)); + owner.join_cpu().await.map(|_| ()) + } else { + owner.io = Some(tokio::spawn(async { panic_io() })); + owner + .join_io(Instant::now() + Duration::from_secs(2)) + .await + .map(|_| ()) + }; + let error = result.err().context("missing worker failure")?; + assert!( + error + .downcast_ref::() + .is_some_and(|failure| failure.join_error().is_panic()) + ); + assert!(!format!("{error:#?}").contains("private-")); + assert!(owner.fatal); + drop(owner); + registry + .shutdown(Instant::now() + Duration::from_secs(2)) + .await?; + } + Ok(()) +} diff --git a/apps/rustack/src/snapshot/workflows/tests.rs b/apps/rustack/src/snapshot/workflows/tests.rs new file mode 100644 index 00000000..ef2d79e1 --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/tests.rs @@ -0,0 +1,189 @@ +//! Actual service archives, metadata/history joins and atomic startup binding. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use http::StatusCode; +use rustack_core::settings::RuntimeBudgets; +use rustack_stepfunctions_core::RegistryMetadata; +use serde_json::json; +use tempfile::TempDir; +use tokio::{ + fs, + sync::{Notify, watch}, +}; + +use super::*; +use crate::{ + snapshot::{ + MANIFEST_FILE, RuntimeProviders, SnapshotConfig, SnapshotName, + archive::{ArchiveKind, from_cbor, read_archive}, + }, + workflows::tests::{request, runtime, start}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn config(root: &TempDir) -> Result { + Ok(SnapshotConfig { + name: SnapshotName::try_from("workflows".to_owned())?, + root: root.path().to_owned(), + }) +} +fn providers(client: WorkflowArchive) -> RuntimeProviders { + let mut providers = RuntimeProviders::default(); + providers.register_workflows(client); + providers +} +#[tokio::test] +async fn test_should_use_real_meta_data_generation_and_restore_exact_old_execution() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let mut source = runtime(RuntimeBudgets::default())?; + let mut target = runtime(RuntimeBudgets::default())?; + let result = async { + let captured = start(&source, 0).await?; + source.owner.quiesce(deadline()).await?; + assert!(source.archive.is_ready(), "quiesce closed the save lane"); + let expected = serde_json::to_value(captured.handle().snapshot().history().page(1, 1000, true)?.events())?; + config.save(&providers(source.archive.clone()), "test").await?; + let service = config.snapshot_dir().join("services/stepfunctions"); + assert!(fs::metadata(service.join("meta.ss.zst")).await?.is_file()); + assert!(fs::metadata(service.join("data.ss.zst")).await?.is_file()); + assert!(fs::metadata(config.snapshot_dir().join(MANIFEST_FILE)).await?.is_file()); + let parts = read_archive(&service.join("meta.ss.zst"), ArchiveKind::ServiceMeta).await?; + let metadata: RegistryMetadata = from_cbor(&parts.first().context("missing metadata section")?.bytes)?; + assert_eq!(serde_json::to_value(metadata)?.pointer("/executions/0/history"), Some(&json!([]))); + config.load(&providers(target.archive.clone())).await?; + let (status, saved) = request(target.api.clone(), "DescribeExecution", json!({"executionArn":captured.arn().as_str()})).await?; + assert_eq!(status, StatusCode::OK, "{saved}"); + assert_eq!(saved.get("status"), Some(&json!("SUCCEEDED"))); + assert_eq!(saved.get("input"), Some(&json!("{\"answer\":42}"))); + let (status, history) = request(target.api.clone(), "GetExecutionHistory", json!({"executionArn":captured.arn().as_str()})).await?; + assert_eq!(status, StatusCode::OK, "{history}"); + assert_eq!(history.get("events"), Some(&expected)); + let (status, duplicate) = request(target.api.clone(), "StartExecution", json!({"stateMachineArn":captured.arn().machine().as_str(),"name":"Run","input":"{\"answer\":42}"})).await?; + assert_eq!(status, StatusCode::BAD_REQUEST, "{duplicate}"); + let (status, next) = request(target.api.clone(), "StartExecution", json!({"stateMachineArn":captured.arn().machine().as_str(),"name":"After","input":"{}"})).await?; + assert_eq!(status, StatusCode::OK, "{next}"); + assert_eq!(target.status.capabilities()["snapshot"], "resources-and-history"); + assert_eq!(providers(target.archive.clone()).coverage(), vec![("stepfunctions", "resources-and-history")]); + Ok(()) + }.await; + let one = source.owner.shutdown(deadline()).await; + let two = target.owner.shutdown(deadline()).await; + one?; + two?; + result +} +#[tokio::test] +async fn test_should_save_empty_registry_with_required_history_data() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let mut source = runtime(RuntimeBudgets::default())?; + let mut target = runtime(RuntimeBudgets::default())?; + let result = async { + source.owner.quiesce(deadline()).await?; + config + .save(&providers(source.archive.clone()), "test") + .await?; + config.load(&providers(target.archive.clone())).await?; + let (status, rows) = request(target.api.clone(), "ListStateMachines", json!({})).await?; + assert_eq!(status, StatusCode::OK); + assert_eq!(rows.get("stateMachines"), Some(&json!([]))); + Ok(()) + } + .await; + let one = source.owner.shutdown(deadline()).await; + let two = target.owner.shutdown(deadline()).await; + one?; + two?; + result +} +#[tokio::test] +async fn test_should_preserve_old_generation_when_export_is_not_sealed() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let mut source = runtime(RuntimeBudgets::default())?; + let mut live = runtime(RuntimeBudgets::default())?; + let result = async { + start(&source, 0).await?; + source.owner.quiesce(deadline()).await?; + config + .save(&providers(source.archive.clone()), "old") + .await?; + let before = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + assert!( + config + .save(&providers(live.archive.clone()), "unproven") + .await + .is_err() + ); + assert_eq!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?, + before + ); + Ok(()) + } + .await; + let one = source.owner.shutdown(deadline()).await; + let two = live.owner.shutdown(deadline()).await; + one?; + two?; + result +} + +mod rejection; + +#[derive(Debug)] +pub(super) struct Hooks { + cpu: bool, + cpu_entered: Notify, + cpu_released: AtomicBool, + io: bool, + io_entered: Notify, + io_released: watch::Sender, +} +impl Hooks { + fn new(cpu: bool, io: bool) -> Arc { + Arc::new(Self { + cpu, + cpu_entered: Notify::new(), + cpu_released: AtomicBool::new(false), + io, + io_entered: Notify::new(), + io_released: watch::channel(false).0, + }) + } + pub(super) fn cpu_wait(&self) { + if self.cpu { + self.cpu_entered.notify_one(); + while !self.cpu_released.load(Ordering::Acquire) { + std::thread::park_timeout(Duration::from_millis(1)); + } + } + } + pub(super) async fn io_wait(&self) -> Result<()> { + if self.io { + self.io_entered.notify_one(); + let mut release = self.io_released.subscribe(); + release + .wait_for(|ready| *ready) + .await + .context("owned IO test gate lost")?; + } + Ok(()) + } + fn release(&self) { + self.cpu_released.store(true, Ordering::Release); + self.io_released.send_replace(true); + } +} +#[derive(Debug)] +struct Release(Arc); +impl Drop for Release { + fn drop(&mut self) { + self.0.release(); + } +} +mod ownership; diff --git a/apps/rustack/src/snapshot/workflows/tests/ownership.rs b/apps/rustack/src/snapshot/workflows/tests/ownership.rs new file mode 100644 index 00000000..7b5a783e --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/tests/ownership.rs @@ -0,0 +1,172 @@ +//! Loss of service observers preserves prepared graphs and actual IO retirement. + +use rustack_stepfunctions_core::{ + ExecutionLimits, ExecutionSupervisor, HistoryLimits, Payload, RegistryCapacities, + RegistryCommand, RegistryLimits, RegistrySupervisor, RuntimeLimits, StateSupport, TenantScope, +}; +use tokio::time::{advance, pause, resume, timeout}; + +use super::*; + +fn registry() -> Result<(Registry, RegistrySupervisor)> { + let runtime = ExecutionSupervisor::new( + RuntimeLimits::default(), + HistoryLimits::new(25000, 1024 * 1024)?, + 8 * 1024 * 1024, + None, + )?; + Ok(Registry::new_with_state_support( + TenantScope::default(), + RegistryLimits::new( + RegistryCapacities::new(1, 2, 1, 2)?, + Duration::from_secs(60), + 8, + )?, + runtime, + ExecutionLimits::default(), + StateSupport::Structured, + )?) +} +#[tokio::test] +async fn test_should_keep_prepared_graph_and_commit_once_after_service_observer_drop() -> Result<()> +{ + let directory = tempfile::tempdir()?; + let mut source = runtime(RuntimeBudgets::default())?; + let (registry, mut registry_owner) = registry()?; + let hooks = Hooks::new(true, false); + let release = Release(hooks.clone()); + let (archive, mut owner) = WorkflowArchive::new_owned( + registry.clone(), + 1024 * 1024, + Duration::from_secs(10), + Some(hooks.clone()), + )?; + let result = async { + start(&source, 0).await?; + source.owner.quiesce(deadline()).await?; + let metadata = source.archive.save_meta(directory.path()).await?; + let client = archive.clone(); + let path = directory.path().to_owned(); + let observer = tokio::spawn(async move { client.load_meta(metadata, &path).await }); + timeout(Duration::from_secs(2), hooks.cpu_entered.notified()).await?; + assert!( + !registry.status().ready, + "prepared lease did not block business admission" + ); + assert!(registry.submit(RegistryCommand::Machines).is_err()); + observer.abort(); + assert!( + observer + .await + .err() + .is_some_and(|error| error.is_cancelled()) + ); + assert!( + owner + .shutdown(Instant::now() + Duration::from_millis(5)) + .await + .is_err() + ); + assert!(!registry.status().ready); + hooks.release(); + owner.shutdown(deadline()).await?; + let rows = registry + .submit(RegistryCommand::Machines)? + .wait(deadline()) + .await?; + let RegistryResult::Machines(machines) = rows else { + anyhow::bail!("missing committed machine"); + }; + assert_eq!(machines.len(), 1); + let machine = machines.first().context("missing imported machine")?; + let capacity = registry + .submit(RegistryCommand::Start( + machine.arn().clone(), + Some("next".parse()?), + Payload::parse(b"{}")?, + None, + ))? + .wait(deadline()) + .await; + assert!( + matches!(capacity, Err(RegistryError::Capacity)), + "import bypassed original record pool" + ); + Ok(()) + } + .await; + drop(release); + let archive = owner.shutdown(deadline()).await; + let records = registry_owner.shutdown(deadline()).await; + let original = source.owner.shutdown(deadline()).await; + archive?; + records?; + original?; + result +} +#[tokio::test] +async fn test_should_join_expired_io_after_observer_drop_without_starting_preparation() -> Result<()> +{ + let directory = tempfile::tempdir()?; + let mut source = runtime(RuntimeBudgets::default())?; + let (registry, mut registry_owner) = registry()?; + let hooks = Hooks::new(false, true); + let release = Release(hooks.clone()); + let (archive, mut owner) = WorkflowArchive::new_owned( + registry.clone(), + 1024 * 1024, + Duration::from_secs(1), + Some(hooks.clone()), + )?; + let mut paused = false; + let result = async { + start(&source, 0).await?; + source.owner.quiesce(deadline()).await?; + let metadata = source.archive.save_meta(directory.path()).await?; + pause(); + paused = true; + let client = archive.clone(); + let path = directory.path().to_owned(); + let observer = tokio::spawn(async move { client.load_meta(metadata, &path).await }); + timeout(Duration::from_secs(2), hooks.io_entered.notified()).await?; + advance(Duration::from_secs(2)).await; + resume(); + paused = false; + let mut ready = archive.ready.clone(); + timeout(Duration::from_secs(2), ready.wait_for(|ready| !*ready)).await??; + observer.abort(); + assert!( + observer + .await + .err() + .is_some_and(|error| error.is_cancelled()) + ); + assert!( + owner + .shutdown(Instant::now() + Duration::from_millis(5)) + .await + .is_err() + ); + assert!(registry.status().ready, "IO delay started preparation"); + hooks.release(); + owner.shutdown(deadline()).await?; + let rows = registry + .submit(RegistryCommand::Machines)? + .wait(deadline()) + .await?; + assert!(matches!(rows,RegistryResult::Machines(machines) if machines.is_empty())); + Ok(()) + } + .await; + if paused { + resume(); + } + drop(release); + let archive = owner.shutdown(deadline()).await; + let records = registry_owner.shutdown(deadline()).await; + let original = source.owner.shutdown(deadline()).await; + archive?; + records?; + original?; + result +} diff --git a/apps/rustack/src/snapshot/workflows/tests/rejection.rs b/apps/rustack/src/snapshot/workflows/tests/rejection.rs new file mode 100644 index 00000000..0a00d4e8 --- /dev/null +++ b/apps/rustack/src/snapshot/workflows/tests/rejection.rs @@ -0,0 +1,241 @@ +//! Valid-envelope corruption cannot publish a partial workflow graph. + +use rstest::rstest; +use rustack_stepfunctions_core::RegistryHistories; + +use super::*; +use crate::snapshot::{ + archive::{ + ArchiveSection, SECTION_STATE_CBOR, pack_data_archive, to_cbor, unpack_data_archive, + write_archive, + }, + read_manifest, write_manifest, +}; + +#[rstest] +#[case("schema")] +#[case("profile")] +#[case("hash")] +#[case("missing-data")] +#[case("orphan-data")] +#[case("unknown-field")] +#[case("quota")] +#[case("missing-file")] +#[tokio::test] +async fn test_should_reject_invalid_archive_before_publishing_any_workflow( + #[case] corruption: &str, +) -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let mut source = runtime(RuntimeBudgets::default())?; + let mut budgets = RuntimeBudgets::default(); + if corruption == "quota" { + budgets.workflow_registry_records = 1; + } + let mut target = runtime(budgets)?; + let result = async { + let first = start(&source, 0).await?; + let (status, reply) = request( + source.api.clone(), + "StartExecution", + json!({"stateMachineArn":first.arn().machine().as_str(),"name":"second","input":"{}"}), + ) + .await?; + assert_eq!(status, StatusCode::OK, "{reply}"); + source.owner.quiesce(deadline()).await?; + config + .save(&providers(source.archive.clone()), "source") + .await?; + let service = config.snapshot_dir().join("services/stepfunctions"); + let meta_path = service.join("meta.ss.zst"); + let data_path = service.join("data.ss.zst"); + let sections = read_archive(&meta_path, ArchiveKind::ServiceMeta).await?; + let mut metadata = serde_json::to_value(from_cbor::( + §ions.first().context("missing metadata")?.bytes, + )?)?; + let history_dir = directory.path().join("mutated-history"); + unpack_data_archive( + &data_path, + &history_dir, + Some(1024 * 1024), + Some(HISTORY_FILE), + ) + .await?; + let mut histories = serde_json::to_value(from_cbor::( + &fs::read(history_dir.join(HISTORY_FILE)).await?, + )?)?; + match corruption { + "schema" => metadata["schemaVersion"] = json!(2), + "profile" => metadata["profileVersion"] = json!(2), + "hash" => metadata["revisions"][0]["sourceHash"] = json!(vec![0; 32]), + "missing-data" => histories["histories"] = json!([]), + "orphan-data" => { + histories["histories"][0]["arn"] = + json!("arn:aws:states:us-east-1:000000000000:execution:Machine:orphan"); + } + "unknown-field" => metadata["private-archive-marker"] = json!("private-archive-marker"), + "quota" => {} + "missing-file" => { + let mut manifest = + read_manifest(&config.snapshot_dir().join(MANIFEST_FILE)).await?; + let service = manifest + .services + .get_mut("stepfunctions") + .context("missing service entry")?; + service.data_file = None; + service.data = None; + write_manifest(&config.snapshot_dir().join(MANIFEST_FILE), &manifest).await?; + } + _ => anyhow::bail!("unknown corruption"), + } + write_archive( + &meta_path, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new( + SECTION_STATE_CBOR, + to_cbor(&metadata)?, + 1, + )], + ) + .await?; + fs::write(history_dir.join(HISTORY_FILE), to_cbor(&histories)?).await?; + pack_data_archive(&history_dir, &data_path).await?; + let before = fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?; + let error = config + .load(&providers(target.archive.clone())) + .await + .err() + .context("accepted invalid workflow archive")?; + assert!(!format!("{error:#?}").contains("private-archive-marker")); + let (status, machines) = + request(target.api.clone(), "ListStateMachines", json!({})).await?; + assert_eq!(status, StatusCode::OK, "{machines}"); + assert_eq!(machines.get("stateMachines"), Some(&json!([]))); + assert_eq!( + fs::read(config.snapshot_dir().join(MANIFEST_FILE)).await?, + before + ); + Ok(()) + } + .await; + let one = source.owner.shutdown(deadline()).await; + let two = target.owner.shutdown(deadline()).await; + one?; + two?; + result +} +#[tokio::test] +async fn test_should_load_old_generation_without_workflow_service_as_empty() -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + config.save(&RuntimeProviders::default(), "old").await?; + let mut target = runtime(RuntimeBudgets::default())?; + let result = async { + config.load(&providers(target.archive.clone())).await?; + let (status, machines) = + request(target.api.clone(), "ListStateMachines", json!({})).await?; + assert_eq!(status, StatusCode::OK); + assert_eq!(machines.get("stateMachines"), Some(&json!([]))); + Ok(()) + } + .await; + target.owner.shutdown(deadline()).await?; + result +} + +#[rstest] +#[case("profile")] +#[case("hash")] +#[case("scope")] +#[tokio::test] +async fn test_should_retain_both_generations_until_domain_import_succeeds( + #[case] corruption: &str, +) -> Result<()> { + let directory = tempfile::tempdir()?; + let config = config(&directory)?; + let mut source = runtime(RuntimeBudgets::default())?; + let mut target = runtime(RuntimeBudgets::default())?; + let result = async { + start(&source, 0).await?; + source.owner.quiesce(deadline()).await?; + let providers = providers(source.archive.clone()); + config.save(&providers, "previous").await?; + let previous = directory.path().join(".workflows.previous"); + fs::rename(config.snapshot_dir(), &previous).await?; + config.save(&providers, "current").await?; + // Recreate the post-publish/pre-retirement crash window with two + // physically valid generations, before corrupting only domain data. + fs::create_dir_all(previous.join("services/stepfunctions")).await?; + let files = [ + MANIFEST_FILE, + "services/stepfunctions/meta.ss.zst", + "services/stepfunctions/data.ss.zst", + ]; + let mut original = Vec::new(); + for relative in files { + let bytes = fs::read(config.snapshot_dir().join(relative)).await?; + fs::write(previous.join(relative), &bytes).await?; + original.push(bytes); + } + let meta_path = config.snapshot_dir().join(files[1]); + let sections = read_archive(&meta_path, ArchiveKind::ServiceMeta).await?; + let mut metadata = serde_json::to_value(from_cbor::( + §ions.first().context("missing metadata")?.bytes, + )?)?; + match corruption { + "profile" => metadata["profileVersion"] = json!(2), + "hash" => metadata["revisions"][0]["sourceHash"] = json!(vec![0; 32]), + "scope" => metadata["scope"]["account"] = json!("111111111111"), + _ => anyhow::bail!("unknown corruption"), + } + write_archive( + &meta_path, + ArchiveKind::ServiceMeta, + vec![ArchiveSection::new( + SECTION_STATE_CBOR, + to_cbor(&metadata)?, + 1, + )], + ) + .await?; + let corrupted = fs::read(&meta_path).await?; + let target_providers = super::providers(target.archive.clone()); + assert!(config.load(&target_providers).await.is_err()); + for (relative, bytes) in files.into_iter().zip(&original) { + assert_eq!(&fs::read(previous.join(relative)).await?, bytes); + } + assert_eq!(fs::read(&meta_path).await?, corrupted); + let (status, machines) = + request(target.api.clone(), "ListStateMachines", json!({})).await?; + assert_eq!(status, StatusCode::OK); + assert_eq!(machines.get("stateMachines"), Some(&json!([]))); + fs::write( + &meta_path, + original.get(1).context("missing original metadata")?, + ) + .await?; + target.owner.shutdown(deadline()).await?; + target = runtime(RuntimeBudgets::default())?; + config + .load(&super::providers(target.archive.clone())) + .await?; + assert!(!fs::try_exists(&previous).await?); + let (status, machines) = + request(target.api.clone(), "ListStateMachines", json!({})).await?; + assert_eq!(status, StatusCode::OK); + assert_eq!( + machines + .get("stateMachines") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(1) + ); + Ok(()) + } + .await; + let one = source.owner.shutdown(deadline()).await; + let two = target.owner.shutdown(deadline()).await; + one?; + two?; + result +} diff --git a/apps/rustack/src/stepfunctions_api.rs b/apps/rustack/src/stepfunctions_api.rs new file mode 100644 index 00000000..a18877ce --- /dev/null +++ b/apps/rustack/src/stepfunctions_api.rs @@ -0,0 +1,427 @@ +//! Registry-backed API, retained preparation and charged stable cursors. +//! +//! Compilation follows the registry's immutable installed instruction profile. +//! async-trait is required by the transport's object-safe handler port. + +use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use rustack_stepfunctions_core::{Registry, TaskSupport, TenantScope}; +use rustack_stepfunctions_http::{Request, RequestContext, Response, StepFunctionsHandler}; +use rustack_stepfunctions_model::{StepFunctionsError, StepFunctionsErrorCode as Code}; +use tokio::{ + runtime::Handle, + spawn, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinError, JoinHandle, JoinSet}, + time::{Instant, MissedTickBehavior, interval, timeout_at}, +}; + +mod error; +mod pages; +mod prepare; +mod render; +#[cfg(test)] +mod tests; +mod worker; + +pub use error::ApiError; + +/// Validated API, preparation and stable-pagination admission limits. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ApiLimits { + accepted: usize, + cpu: usize, + tokens: usize, + bytes: usize, +} +impl ApiLimits { + /// Validate accepted1..128, workers1..32≤accepted, tokens1..100k and8MiB..1GiB. + /// + /// # Errors + /// Rejects zero/excessive or inconsistent trusted limits. + /// + /// ``` + /// let limits=rustack_cli::stepfunctions_api::ApiLimits::new(32,8,10000,64*1024*1024)?; + /// # Ok::<(),rustack_cli::stepfunctions_api::ApiError>(()) + /// ``` + pub fn new(accepted: usize, cpu: usize, tokens: usize, bytes: usize) -> Result { + if !(1..=128).contains(&accepted) + || !(1..=32).contains(&cpu) + || cpu > accepted + || !(1..=100_000).contains(&tokens) + || !(8 * 1024 * 1024..=1024 * 1024 * 1024).contains(&bytes) + { + return Err(ApiError::Configuration); + } + Ok(Self { + accepted, + cpu, + tokens, + bytes, + }) + } +} +impl Default for ApiLimits { + fn default() -> Self { + Self { + accepted: 32, + cpu: 8, + tokens: 10_000, + bytes: 64 * 1024 * 1024, + } + } +} +struct Admission { + sender: mpsc::Sender, + capacity: Arc, + closing: watch::Sender, + diagnostic: watch::Receiver>>, +} +struct Context { + scope: TenantScope, + tasks: TaskSupport, + registry: Registry, + cpu: Arc, + pages: mpsc::Sender, +} +struct Envelope { + request: Request, + deadline: Instant, + reply: oneshot::Sender, + permit: OwnedSemaphorePermit, +} +struct Reply { + result: Result, + permit: Option, +} +struct Slot { + reply: oneshot::Sender, + permit: OwnedSemaphorePermit, +} +struct Io { + requests: mpsc::Receiver, + pages: mpsc::Receiver, + context: Arc, + closing: watch::Sender, + diagnostic: watch::Sender>>, +} +struct State { + io: Io, + jobs: JoinSet>, + slots: HashMap, + quarantine: Vec, + store: pages::Store, +} +struct Owner { + state: Option, +} + +/// Real typed handler with bounded caller-independent work ownership. +#[non_exhaustive] +pub struct StepFunctionsApi { + admission: Arc, + scope: TenantScope, +} +impl fmt::Debug for StepFunctionsApi { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsApi()") + } +} +impl StepFunctionsApi { + /// Compose the real registry with compiler capability supplied by its owner. + /// + /// # Errors + /// Requires active Tokio and matching trusted registry/operator scope. + /// + /// ```no_run + /// # fn compose(registry:rustack_stepfunctions_core::Registry)->Result<(),rustack_cli::stepfunctions_api::ApiError>{ + /// let(handler,owner)=rustack_cli::stepfunctions_api::StepFunctionsApi::new(Default::default(),rustack_stepfunctions_core::TaskSupport::Disabled,registry,Default::default())?; + /// # Ok(()) } + /// ``` + pub fn new( + scope: TenantScope, + tasks: TaskSupport, + registry: Registry, + limits: ApiLimits, + ) -> Result<(Arc, ApiSupervisor), ApiError> { + let (api, owner, stop) = Self::compose(scope, tasks, registry, limits)?; + let supervisor = ApiSupervisor { + closing: api.admission.closing.clone(), + join: Some(spawn(owner.run(stop))), + result: None, + }; + Ok((api, supervisor)) + } + fn compose( + scope: TenantScope, + tasks: TaskSupport, + registry: Registry, + limits: ApiLimits, + ) -> Result<(Arc, Owner, watch::Receiver), ApiError> { + Handle::try_current().map_err(|_| ApiError::Configuration)?; + let other = registry.scope(); + if scope.partition() != other.partition() + || scope.region() != other.region() + || scope.account() != other.account() + || registry.callback_support() && tasks != TaskSupport::Lambda + { + return Err(ApiError::Configuration); + } + let (sender, requests) = mpsc::channel(limits.accepted); + let (page_tx, pages) = mpsc::channel(limits.accepted); + let (closing, stop) = watch::channel(false); + let (diagnostic, diagnostics) = watch::channel(None); + let context = Arc::new(Context { + scope: scope.clone(), + tasks, + registry, + cpu: Arc::new(Semaphore::new(limits.cpu)), + pages: page_tx, + }); + let owner = Owner { + state: Some(State { + io: Io { + requests, + pages, + context, + closing: closing.clone(), + diagnostic, + }, + jobs: JoinSet::new(), + slots: HashMap::new(), + quarantine: Vec::new(), + store: pages::Store::new(limits), + }), + }; + let api = Arc::new(Self { + scope, + admission: Arc::new(Admission { + sender, + capacity: Arc::new(Semaphore::new(limits.accepted)), + closing: closing.clone(), + diagnostic: diagnostics, + }), + }); + Ok((api, owner, stop)) + } + /// Inspect a retained typed failure without exposing source text in Debug. + /// + /// ``` + /// # fn inspect(api:&rustack_cli::stepfunctions_api::StepFunctionsApi){let diagnostic=api.diagnostic();} + /// ``` + #[must_use] + pub fn diagnostic(&self) -> Option> { + self.admission.diagnostic.borrow().clone() + } + /// Inspect admission/root health without exposing request data. + /// + /// ``` + /// # fn inspect(api:&rustack_cli::stepfunctions_api::StepFunctionsApi){let ready=api.is_ready();} + /// ``` + #[must_use] + pub fn is_ready(&self) -> bool { + !*self.admission.closing.borrow() && self.admission.diagnostic.borrow().is_none() + } +} +#[async_trait] +impl StepFunctionsHandler for StepFunctionsApi { + #[tracing::instrument(name = "stepfunctions.api", skip_all)] + async fn handle( + &self, + context: RequestContext, + request: Request, + ) -> Result { + let scope = context.scope(); + if scope.partition() != self.scope.partition() + || scope.region() != self.scope.region() + || scope.account() != self.scope.account() + { + return Err(error::error(Code::AccessDeniedException)); + } + let deadline = context.deadline(); + error::check(deadline)?; + if *self.admission.closing.borrow() { + return Err(error::error(Code::ServiceUnavailable)); + } + let permit = self + .admission + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| error::error(Code::ThrottlingException))?; + let (reply, receiver) = oneshot::channel(); + self.admission + .sender + .try_send(Envelope { + request, + deadline, + reply, + permit, + }) + .map_err(|e| match e { + mpsc::error::TrySendError::Full(_) => error::error(Code::ThrottlingException), + mpsc::error::TrySendError::Closed(_) => error::error(Code::ServiceUnavailable), + })?; + let reply = timeout_at(deadline, receiver) + .await + .map_err(|_| error::error(Code::RequestTimeoutException))? + .map_err(|_| error::error(Code::ServiceUnavailable))?; + let result = reply.result; + drop(reply.permit); + error::check(deadline)?; + result + } +} +impl Owner { + async fn run(mut self, stop: watch::Receiver) -> Result<(), ApiError> { + let result = match self.state.as_mut() { + Some(state) => state.run(stop).await, + None => Err(ApiError::Quarantined), + }; + if result.is_ok() { + drop(self.state.take()); + } + result + } +} +impl Drop for Owner { + fn drop(&mut self) { + if let Some(mut state) = self.state.take() { + state.io.closing.send_replace(true); + state.io.diagnostic.send_if_modified(|current| { + if current.is_some() { + false + } else { + *current = Some(Arc::new(ApiError::Quarantined)); + true + } + }); + state.io.requests.close(); + while let Ok(envelope) = state.io.requests.try_recv() { + let _sent = envelope.reply.send(Reply { + result: Err(error::error(Code::ServiceUnavailable)), + permit: Some(envelope.permit), + }); + } + let _retained = Box::leak(Box::new(state)); + } + } +} +impl State { + async fn run(&mut self, mut stop: watch::Receiver) -> Result<(), ApiError> { + let mut closed = false; + let mut incoming = true; + let mut timer = interval(Duration::from_secs(1)); + timer.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + if !closed && (*stop.borrow() || stop.has_changed().is_err()) { + closed = true; + self.io.closing.send_replace(true); + self.io.requests.close(); + } + if !incoming && self.jobs.is_empty() { + return if self.quarantine.is_empty() { + Ok(()) + } else { + Err(ApiError::Quarantined) + }; + } + tokio::select! {biased; + _=stop.changed(),if !closed=>{}, + joined=self.jobs.join_next_with_id(),if !self.jobs.is_empty()=>if let Some(joined)=joined{self.finish(joined);}, + command=self.io.pages.recv()=>if let Some(command)=command{self.store.dispatch(command);}, + envelope=self.io.requests.recv(),if incoming=>if let Some(envelope)=envelope{self.start(envelope);}else{incoming=false;closed=true;self.io.closing.send_replace(true);}, + _=timer.tick()=>self.store.expire(Instant::now()), + } + } + } + fn start(&mut self, envelope: Envelope) { + let context = self.io.context.clone(); + let job = self + .jobs + .spawn(worker::run(context, envelope.request, envelope.deadline)); + self.slots.insert( + job.id(), + Slot { + reply: envelope.reply, + permit: envelope.permit, + }, + ); + } + fn finish(&mut self, joined: Result<(Id, Result), JoinError>) { + match joined { + Ok((id, result)) => { + if let Some(slot) = self.slots.remove(&id) { + let _sent = slot.reply.send(Reply { + result, + permit: Some(slot.permit), + }); + } + } + Err(error) => { + let id = error.id(); + let diagnostic = Arc::new(ApiError::Worker(error)); + self.io.diagnostic.send_replace(Some(diagnostic.clone())); + if let Some(slot) = self.slots.remove(&id) { + let _sent = slot.reply.send(Reply { + result: Err(error::source(Code::InternalFailure, diagnostic)), + permit: None, + }); + self.quarantine.push(slot.permit); + } + } + } + } +} +/// Unique resumable API-root shutdown controller. +#[derive(Debug)] +#[non_exhaustive] +pub struct ApiSupervisor { + closing: watch::Sender, + join: Option>>, + result: Option>>, +} +impl ApiSupervisor { + /// Close admission; all accepted work remains owned until joined. + /// + /// ``` + /// # fn close(owner:&rustack_cli::stepfunctions_api::ApiSupervisor){owner.close();} + /// ``` + pub fn close(&self) { + self.closing.send_replace(true); + } + /// Join the same root through an absolute observation deadline. + /// + /// # Errors + /// Deadline preserves its handle; worker/quarantine errors remain cached. + /// + /// ```no_run + /// # async fn join(owner:&mut rustack_cli::stepfunctions_api::ApiSupervisor,deadline:tokio::time::Instant){let result=owner.shutdown(deadline).await;} + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), ApiError> { + self.close(); + if self.result.is_none() { + let join = self.join.as_mut().ok_or(ApiError::Quarantined)?; + let joined = timeout_at(deadline, join) + .await + .map_err(|_| ApiError::Deadline)?; + self.join = None; + self.result = Some(match joined { + Ok(result) => result.map_err(Arc::new), + Err(error) => Err(Arc::new(ApiError::Worker(error))), + }); + } + match self.result.as_ref() { + Some(Ok(())) => Ok(()), + Some(Err(error)) => Err(ApiError::Owner(error.clone())), + None => Err(ApiError::Quarantined), + } + } +} +impl Drop for ApiSupervisor { + fn drop(&mut self) { + self.close(); /* Detach only the owning root; it joins its accepted jobs. */ + } +} diff --git a/apps/rustack/src/stepfunctions_api/error.rs b/apps/rustack/src/stepfunctions_api/error.rs new file mode 100644 index 00000000..9ed93a58 --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/error.rs @@ -0,0 +1,102 @@ +//! Static service diagnostics and retained typed sources. + +use std::{error::Error, fmt, sync::Arc}; + +use http::StatusCode; +use rustack_stepfunctions_core::{CallbackError, ExecutionError, RegistryError}; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode as Code, StepFunctionsOperation, +}; +use thiserror::Error; +use tokio::{task::JoinError, time::Instant}; + +/// API-owner failures; Debug and public error text never project business data. +#[derive(Error)] +#[non_exhaustive] +pub enum ApiError { + /// Invalid trusted capacities/scope/runtime configuration. + #[error("invalid workflow API configuration")] + Configuration, + /// An observation expired while its owner remains live. + #[error("workflow API observation expired")] + Deadline, + /// Work remains uncertain after unexpected worker failure. + #[error("workflow API cleanup retained")] + Quarantined, + /// Supervised root/worker failure with redacted diagnostics. + #[error("workflow API worker failed")] + Worker(#[source] JoinError), + /// Cached root failure. + #[error("workflow API owner failed")] + Owner(#[source] Arc), +} +impl fmt::Debug for ApiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +pub(super) fn error(code: Code) -> StepFunctionsError { + StepFunctionsError::new(code) +} +pub(super) fn source(code: Code, cause: impl Error + Send + Sync + 'static) -> StepFunctionsError { + let mut error = error(code); + error.source = Some(Box::new(cause)); + error +} +pub(super) fn registry( + cause: RegistryError, + operation: StepFunctionsOperation, +) -> StepFunctionsError { + let code = match &cause { + RegistryError::Validation => Code::ValidationException, + RegistryError::MachineMissing => match operation { + StepFunctionsOperation::TagResource + | StepFunctionsOperation::UntagResource + | StepFunctionsOperation::ListTagsForResource => Code::ResourceNotFound, + _ => Code::StateMachineDoesNotExist, + }, + RegistryError::TagLimit => Code::TooManyTags, + RegistryError::MachineExists => Code::StateMachineAlreadyExists, + RegistryError::Deleting => Code::StateMachineDeleting, + RegistryError::ExecutionMissing => Code::ExecutionDoesNotExist, + RegistryError::ExecutionExists => Code::ExecutionAlreadyExists, + RegistryError::Capacity => match operation { + StepFunctionsOperation::CreateStateMachine => Code::StateMachineLimitExceeded, + StepFunctionsOperation::StartExecution => Code::ExecutionLimitExceeded, + _ => Code::ThrottlingException, + }, + RegistryError::Deadline => Code::RequestTimeoutException, + RegistryError::Unavailable => Code::ServiceUnavailable, + RegistryError::Execution(ExecutionError::Capacity) => Code::ExecutionLimitExceeded, + _ => Code::InternalFailure, + }; + let mut result = source(code, cause); + if code == Code::ResourceNotFound { + result.status_code = StatusCode::BAD_REQUEST; + } + result +} +pub(super) fn check(deadline: Instant) -> Result<(), StepFunctionsError> { + if Instant::now() >= deadline { + Err(error(Code::RequestTimeoutException)) + } else { + Ok(()) + } +} + +pub(super) fn callback(cause: CallbackError) -> StepFunctionsError { + source(callback_code(&cause), cause) +} +fn callback_code(cause: &CallbackError) -> Code { + match cause { + CallbackError::Cached(cause) => callback_code(cause), + CallbackError::InvalidToken => Code::InvalidToken, + CallbackError::TaskTimedOut => Code::TaskTimedOut, + CallbackError::Data(_) => Code::InvalidOutput, + CallbackError::Validation => Code::ValidationException, + CallbackError::Capacity => Code::ThrottlingException, + CallbackError::Deadline => Code::RequestTimeoutException, + CallbackError::Unavailable => Code::ServiceUnavailable, + _ => Code::InternalFailure, + } +} diff --git a/apps/rustack/src/stepfunctions_api/pages.rs b/apps/rustack/src/stepfunctions_api/pages.rs new file mode 100644 index 00000000..1bd66178 --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/pages.rs @@ -0,0 +1,383 @@ +//! Actor-owned opaque cursors over shared charged immutable captures. + +use std::{ + collections::HashMap, + fmt::{self, Write}, + mem::size_of, + sync::Arc, + time::Duration, +}; + +use getrandom::fill; +use rustack_stepfunctions_core::{ExecutionSnapshot, MachineSnapshot, RegistryResult}; +use rustack_stepfunctions_http::Response; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode as Code, StepFunctionsOperation as Op, output, + types::ExecutionStatus, +}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore, oneshot}, + time::Instant, +}; + +use super::{ + ApiLimits, + error::{check, error, source}, + render, +}; + +#[derive(Clone, PartialEq, Eq)] +pub(super) struct Flags { + pub(super) size: usize, + pub(super) status: Option, + pub(super) reverse: bool, + pub(super) data: bool, +} +impl Flags { + pub(super) fn new( + size: Option, + status: Option, + reverse: bool, + data: bool, + ) -> Result { + let size = match size { + None | Some(0) => 100, + Some(n) => usize::try_from(n) + .ok() + .filter(|n| *n <= 1000) + .ok_or_else(|| error(Code::ValidationException))?, + }; + Ok(Self { + size, + status, + reverse, + data, + }) + } +} +#[derive(Clone, PartialEq, Eq)] +pub(super) struct Binding { + pub(super) op: Op, + pub(super) resource: String, + pub(super) flags: Flags, +} +pub(super) enum Data { + Machines(Vec), + Executions(Vec), + History(ExecutionSnapshot), +} +impl Data { + pub(super) fn capture( + binding: &Binding, + result: RegistryResult, + ) -> Result { + match (binding.op, result) { + (Op::ListStateMachines, RegistryResult::Machines(values)) => Ok(Self::Machines(values)), + (Op::ListExecutions, RegistryResult::Executions(values)) => { + let mut captured = Vec::new(); + for value in values { + let status = render::status(&value)?; + if binding + .flags + .status + .as_ref() + .is_none_or(|filter| filter == &status) + { + captured.push(value); + } + } + Ok(Self::Executions(captured)) + } + (Op::GetExecutionHistory, RegistryResult::Execution(value)) => Ok(Self::History(value)), + _ => Err(error(Code::InternalFailure)), + } + } + fn bytes(&self) -> Result { + let bytes = match self { + Self::Machines(v) => v.capacity().checked_mul(size_of::()), + Self::Executions(v) => v.capacity().checked_mul(size_of::()), + Self::History(v) => Some(v.view().history().charged_bytes()), + } + .and_then(|n| n.checked_add(512)) + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| error(Code::ThrottlingException))?; + Ok(bytes) + } +} +pub(super) struct Snapshot { + data: Data, + _permit: OwnedSemaphorePermit, +} +#[derive(Clone)] +pub(super) struct Cursor { + pub(super) binding: Binding, + pub(super) snapshot: Arc, + pub(super) position: u32, +} +struct Token { + cursor: Cursor, + expires: Instant, +} +pub(super) struct Store { + tokens: HashMap, + capacity: usize, + bytes: Arc, +} +impl Store { + pub(super) fn new(limits: ApiLimits) -> Self { + Self { + tokens: HashMap::new(), + capacity: limits.tokens, + bytes: Arc::new(Semaphore::new(limits.bytes)), + } + } + pub(super) fn expire(&mut self, now: Instant) { + self.tokens.retain(|_, token| token.expires > now); + } + fn reserve(&mut self, data: Data) -> Result, StepFunctionsError> { + self.expire(Instant::now()); + let permit = self + .bytes + .clone() + .try_acquire_many_owned(data.bytes()?) + .map_err(|_| error(Code::ThrottlingException))?; + Ok(Arc::new(Snapshot { + data, + _permit: permit, + })) + } + fn read(&mut self, token: &str, binding: &Binding) -> Result { + self.expire(Instant::now()); + let value = self + .tokens + .get(token) + .filter(|v| v.cursor.binding == *binding) + .ok_or_else(|| error(Code::InvalidToken))?; + Ok(value.cursor.clone()) + } + fn save(&mut self, token: String, cursor: Cursor) -> Result<(), StepFunctionsError> { + self.expire(Instant::now()); + if self.tokens.len() >= self.capacity { + return Err(error(Code::ThrottlingException)); + } + if self.tokens.contains_key(&token) { + return Err(error(Code::InternalFailure)); + } + let expires = Instant::now() + .checked_add(Duration::from_hours(24)) + .ok_or_else(|| error(Code::InternalFailure))?; + self.tokens.insert(token, Token { cursor, expires }); + Ok(()) + } + pub(super) fn dispatch(&mut self, command: Command) { + let deadline = command.deadline; + match command.request { + Request::Reserve(data, reply) => { + let result = check(deadline).and_then(|()| self.reserve(data)); + let _sent = reply.send(result); + } + Request::Read(token, binding, reply) => { + let result = check(deadline).and_then(|()| self.read(&token, &binding)); + let _sent = reply.send(result); + } + Request::Save(token, cursor, reply) => { + let result = check(deadline).and_then(|()| self.save(token, cursor)); + let _sent = reply.send(result); + } + } + } +} +pub(super) struct Command { + pub(super) deadline: Instant, + pub(super) request: Request, +} +pub(super) enum Request { + Reserve( + Data, + oneshot::Sender, StepFunctionsError>>, + ), + Read( + String, + Binding, + oneshot::Sender>, + ), + Save( + String, + Cursor, + oneshot::Sender>, + ), +} +pub(super) fn first(binding: Binding, snapshot: Arc) -> Cursor { + let position = match &snapshot.data { + Data::History(e) if binding.flags.reverse => e.view().history().len(), + Data::History(_) => 1, + _ => 0, + }; + Cursor { + binding, + snapshot, + position, + } +} +pub(super) struct Page { + pub(super) response: Response, + pub(super) next: Option, + pub(super) token: String, +} +pub(super) fn page(cursor: Cursor) -> Result { + let flags = &cursor.binding.flags; + let (response, next) = match &cursor.snapshot.data { + Data::Machines(values) => { + let (slice, next) = slice(values, cursor.position, flags.size)?; + let mut out = output::ListStateMachinesOutput::default(); + out.state_machines = slice.iter().map(render::machine_item).collect(); + (out.into(), next) + } + Data::Executions(values) => { + let (slice, next) = slice(values, cursor.position, flags.size)?; + let mut out = output::ListExecutionsOutput::default(); + out.executions = slice + .iter() + .map(render::execution_item) + .collect::, _>>()?; + (out.into(), next) + } + Data::History(value) => { + let history = value.view().history(); + let page = if flags.reverse { + history.page_reverse(cursor.position, flags.size, flags.data) + } else { + history.page(cursor.position, flags.size, flags.data) + } + .map_err(|e| source(Code::InternalFailure, e))?; + let mut out = output::GetExecutionHistoryOutput::default(); + out.events = page.events().to_vec(); + (out.into(), page.next_id()) + } + }; + let token = if next.is_some() { + nonce()? + } else { + String::new() + }; + Ok(Page { + response, + next: next.map(|position| Cursor { position, ..cursor }), + token, + }) +} +fn slice( + values: &[T], + position: u32, + size: usize, +) -> Result<(&[T], Option), StepFunctionsError> { + let start = usize::try_from(position).map_err(|e| source(Code::InternalFailure, e))?; + let end = start + .checked_add(size) + .ok_or_else(|| error(Code::InternalFailure))? + .min(values.len()); + let slice = values + .get(start..end) + .ok_or_else(|| error(Code::InternalFailure))?; + let next = if end < values.len() { + Some(u32::try_from(end).map_err(|e| source(Code::InternalFailure, e))?) + } else { + None + }; + Ok((slice, next)) +} +fn nonce() -> Result { + let mut bytes = [0_u8; 32]; + fill(&mut bytes).map_err(|e| source(Code::InternalFailure, e))?; + let mut out = String::with_capacity(64); + for byte in bytes { + write!(out, "{byte:02x}").map_err(|e| source(Code::InternalFailure, e))?; + } + Ok(out) +} +pub(super) fn attach(response: &mut Response, token: String) -> Result<(), StepFunctionsError> { + match response { + Response::ListStateMachines(v) => v.next_token = Some(token), + Response::ListExecutions(v) => v.next_token = Some(token), + Response::GetExecutionHistory(v) => v.next_token = Some(token), + _ => return Err(error(Code::InternalFailure)), + } + Ok(()) +} +impl fmt::Debug for Store { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PageStore()") + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn binding() -> Binding { + Binding { + op: Op::ListStateMachines, + resource: String::new(), + flags: Flags::new(Some(1), None, false, true).unwrap(), + } + } + #[test] + fn test_should_charge_sparse_vector_capacity_and_release_only_after_final_snapshot() { + let mut store = Store::new(ApiLimits::new(1, 1, 100, 8 * 1024 * 1024).unwrap()); + let mut snapshots = Vec::new(); + for _ in 0..5 { + snapshots.push( + store + .reserve(Data::Executions(Vec::with_capacity(100_000))) + .unwrap(), + ); + } + assert!( + store + .reserve(Data::Executions(Vec::with_capacity(100_000))) + .is_err() + ); + let held = snapshots.pop().unwrap(); + let clone = held.clone(); + drop(held); + assert!( + store + .reserve(Data::Executions(Vec::with_capacity(100_000))) + .is_err() + ); + drop(clone); + assert!( + store + .reserve(Data::Executions(Vec::with_capacity(100_000))) + .is_ok() + ); + } + #[test] + fn test_should_preserve_live_tokens_at_capacity_and_expire_them_before_read() { + let mut store = Store::new(ApiLimits::new(1, 1, 1, 8 * 1024 * 1024).unwrap()); + let snapshot = store.reserve(Data::Machines(Vec::new())).unwrap(); + let cursor = first(binding(), snapshot); + store.save("a".repeat(64), cursor.clone()).unwrap(); + assert!(store.save("b".repeat(64), cursor.clone()).is_err()); + assert!(store.read(&"a".repeat(64), &binding()).is_ok()); + let expired = Instant::now() + Duration::from_hours(24); + store.expire(expired); + assert!(store.read(&"a".repeat(64), &binding()).is_err()); + store.save("b".repeat(64), cursor).unwrap(); + assert!(store.read(&"b".repeat(64), &binding()).is_ok()); + } + #[test] + fn test_should_reject_binding_changes_and_random_handle_collisions_without_overwriting() { + let mut store = Store::new(ApiLimits::default()); + let snapshot = store.reserve(Data::Machines(Vec::new())).unwrap(); + let cursor = first(binding(), snapshot); + store.save("a".repeat(64), cursor.clone()).unwrap(); + assert!(store.save("a".repeat(64), cursor).is_err()); + let mut changed = binding(); + changed.flags.data = false; + assert_eq!( + store.read(&"a".repeat(64), &changed).err().unwrap().code, + Code::InvalidToken + ); + assert!(store.read(&"a".repeat(64), &binding()).is_ok()); + } +} diff --git a/apps/rustack/src/stepfunctions_api/prepare.rs b/apps/rustack/src/stepfunctions_api/prepare.rs new file mode 100644 index 00000000..fe00937e --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/prepare.rs @@ -0,0 +1,375 @@ +//! Retained blocking preparation under the installed STANDARD/JSONPath profile. + +use rustack_stepfunctions_core::{ + AslCompiler, CallbackCommand, CallbackToken, DiagnosticSeverity, ExecutionArn, Payload, + PreparedDefinition, RegistryCommand, RoleArn, StateMachineArn, StopReason, Tags, TenantScope, +}; +use rustack_stepfunctions_http::{HttpError, Request, Response}; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode as Code, StepFunctionsOperation as Op, input, + output, + types::{ + EncryptionConfiguration, EncryptionType, ExecutionStatus, IncludedData, LogLevel, + LoggingConfiguration, StateMachineType, Tag, TracingConfiguration, + ValidateStateMachineDefinitionDiagnostic, ValidateStateMachineDefinitionResultCode, + ValidateStateMachineDefinitionSeverity, + }, +}; + +use super::{ + error::{callback, error, source}, + pages::{Binding, Flags}, +}; + +pub(super) enum Prepared { + Command(RegistryCommand, Op), + Page(Binding, Option, RegistryCommand), + Callback(CallbackCommand, Op), + Ready(Response), +} +pub(super) fn prepare( + request: Request, + compiler: &AslCompiler, + scope: &TenantScope, + callbacks: bool, +) -> Result { + request.validate().map_err(|e| { + let code = match &e { + HttpError::Field(code) => *code, + _ => Code::ValidationException, + }; + source(code, e) + })?; + let op = request.operation(); + let command = match request { + Request::CreateStateMachine(v) => create(*v, compiler, scope)?, + Request::UpdateStateMachine(v) => update(&v, compiler, scope)?, + Request::DeleteStateMachine(v) => { + RegistryCommand::Delete(machine(&v.state_machine_arn, scope)?) + } + Request::DescribeStateMachine(v) => { + included(v.included_data.as_ref())?; + RegistryCommand::Machine(machine(&v.state_machine_arn, scope)?) + } + Request::ListStateMachineVersions(v) => { + let arn = machine(&v.state_machine_arn, scope)?; + if v.next_token.is_some() { + return Err(error(Code::InvalidToken)); + } + RegistryCommand::Machine(arn) + } + Request::StartExecution(v) => RegistryCommand::Start( + machine(&v.state_machine_arn, scope)?, + v.name + .as_deref() + .map(str::parse) + .transpose() + .map_err(|e| source(Code::InvalidName, e))?, + Payload::parse(v.input.as_deref().unwrap_or("{}").as_bytes()) + .map_err(|e| source(Code::InvalidExecutionInput, e))?, + v.trace_header, + ), + Request::DescribeExecution(v) => { + included(v.included_data.as_ref())?; + RegistryCommand::Execution(execution(&v.execution_arn, scope)?) + } + Request::StopExecution(v) => RegistryCommand::Stop( + execution(&v.execution_arn, scope)?, + StopReason::new(v.error, v.cause).map_err(|e| source(Code::ValidationException, e))?, + ), + Request::DescribeStateMachineForExecution(v) => { + included(v.included_data.as_ref())?; + RegistryCommand::Execution(execution(&v.execution_arn, scope)?) + } + Request::TagResource(v) => { + RegistryCommand::Tag(machine(&v.resource_arn, scope)?, tags(v.tags)?) + } + Request::UntagResource(v) => RegistryCommand::Untag( + machine(&v.resource_arn, scope)?, + Tags::new(v.tag_keys.into_iter().map(|k| (k, String::new())).collect()) + .map_err(|e| source(Code::ValidationException, e))?, + ), + Request::ListTagsForResource(v) => { + RegistryCommand::Machine(machine(&v.resource_arn, scope)?) + } + Request::ValidateStateMachineDefinition(v) => { + return Ok(Prepared::Ready(validate(&v, compiler)?.into())); + } + Request::ListStateMachines(v) => { + return Ok(Prepared::Page( + Binding { + op, + resource: String::new(), + flags: Flags::new(v.max_results, None, false, true)?, + }, + token(v.next_token)?, + RegistryCommand::Machines, + )); + } + Request::ListExecutions(v) => { + if v.map_run_arn.is_some() + || v.redrive_filter.is_some() + || matches!(v.status_filter, Some(ExecutionStatus::PendingRedrive)) + { + return Err(error(Code::UnsupportedOperation)); + } + let arn = machine( + v.state_machine_arn + .as_deref() + .ok_or_else(|| error(Code::ValidationException))?, + scope, + )?; + return Ok(Prepared::Page( + Binding { + op, + resource: arn.as_str().into(), + flags: Flags::new(v.max_results, v.status_filter, false, true)?, + }, + token(v.next_token)?, + RegistryCommand::Executions(arn), + )); + } + Request::GetExecutionHistory(v) => return history(*v, scope), + request => return callback_request(request, callbacks), + }; + Ok(Prepared::Command(command, op)) +} +fn callback_request(request: Request, enabled: bool) -> Result { + if !enabled { + return Err(error(Code::UnsupportedOperation)); + } + let op = request.operation(); + let command = match request { + Request::SendTaskSuccess(v) => CallbackCommand::success( + CallbackToken::try_from(v.task_token).map_err(callback)?, + v.output.as_bytes(), + ) + .map_err(callback)?, + Request::SendTaskFailure(v) => CallbackCommand::failure( + CallbackToken::try_from(v.task_token).map_err(callback)?, + v.error, + v.cause, + ) + .map_err(callback)?, + Request::SendTaskHeartbeat(v) => { + CallbackCommand::heartbeat(CallbackToken::try_from(v.task_token).map_err(callback)?) + } + _ => return Err(error(Code::UnsupportedOperation)), + }; + Ok(Prepared::Callback(command, op)) +} + +fn history( + v: input::GetExecutionHistoryInput, + scope: &TenantScope, +) -> Result { + let arn = execution(&v.execution_arn, scope)?; + Ok(Prepared::Page( + Binding { + op: Op::GetExecutionHistory, + resource: arn.as_str().into(), + flags: Flags::new( + v.max_results, + None, + v.reverse_order.unwrap_or(false), + v.include_execution_data.unwrap_or(true), + )?, + }, + token(v.next_token)?, + RegistryCommand::Execution(arn), + )) +} +fn create( + v: input::CreateStateMachineInput, + compiler: &AslCompiler, + scope: &TenantScope, +) -> Result { + profile( + v.r#type.as_ref(), + v.logging_configuration.as_ref(), + v.tracing_configuration.as_ref(), + v.encryption_configuration.as_ref(), + v.publish, + v.version_description.as_deref(), + )?; + Ok(RegistryCommand::Create( + v.name.parse().map_err(|e| source(Code::InvalidName, e))?, + definition(compiler, &v.definition)?, + role(&v.role_arn, scope)?, + tags(v.tags)?, + )) +} +fn update( + v: &input::UpdateStateMachineInput, + compiler: &AslCompiler, + scope: &TenantScope, +) -> Result { + profile( + None, + v.logging_configuration.as_ref(), + v.tracing_configuration.as_ref(), + v.encryption_configuration.as_ref(), + v.publish, + v.version_description.as_deref(), + )?; + if v.definition.is_none() && v.role_arn.is_none() { + return Err(error(Code::MissingRequiredParameter)); + } + Ok(RegistryCommand::Update( + machine(&v.state_machine_arn, scope)?, + v.definition + .as_deref() + .map(|v| definition(compiler, v)) + .transpose()?, + v.role_arn.as_deref().map(|v| role(v, scope)).transpose()?, + )) +} +fn definition( + compiler: &AslCompiler, + text: &str, +) -> Result { + Ok(PreparedDefinition::new( + compiler + .compile(text.as_bytes()) + .map_err(|e| source(Code::InvalidDefinition, e))? + .machine() + .clone(), + )) +} +fn role(text: &str, scope: &TenantScope) -> Result { + let role: RoleArn = text.parse().map_err(|e| source(Code::InvalidArn, e))?; + if !role.as_str().starts_with(&format!( + "arn:{}:iam::{}:role/", + scope.partition(), + scope.account() + )) { + return Err(error(Code::InvalidArn)); + } + Ok(role) +} +fn tags(values: Vec) -> Result { + Tags::new( + values + .into_iter() + .map(|t| { + Ok(( + t.key.ok_or_else(|| error(Code::ValidationException))?, + t.value.unwrap_or_default(), + )) + }) + .collect::, StepFunctionsError>>()?, + ) + .map_err(|e| source(Code::ValidationException, e)) +} +fn scoped(text: &str, scope: &TenantScope) -> Result<(), StepFunctionsError> { + if text.starts_with(&format!( + "arn:{}:states:{}:{}:", + scope.partition(), + scope.region(), + scope.account() + )) { + Ok(()) + } else { + Err(error(Code::InvalidArn)) + } +} +fn machine(text: &str, scope: &TenantScope) -> Result { + let value: StateMachineArn = text.parse().map_err(|e| source(Code::InvalidArn, e))?; + scoped(value.as_str(), scope)?; + Ok(value) +} +fn execution(text: &str, scope: &TenantScope) -> Result { + let value: ExecutionArn = text.parse().map_err(|e| source(Code::InvalidArn, e))?; + scoped(value.as_str(), scope)?; + Ok(value) +} +fn token(value: Option) -> Result, StepFunctionsError> { + if value.as_ref().is_some_and(|t| { + t.len() != 64 + || !t + .bytes() + .all(|b| b.is_ascii_digit() || matches!(b, b'a'..=b'f')) + }) { + return Err(error(Code::InvalidToken)); + } + Ok(value) +} +fn included(value: Option<&IncludedData>) -> Result<(), StepFunctionsError> { + if value.is_some_and(|v| v != &IncludedData::AllData) { + Err(error(Code::UnsupportedOperation)) + } else { + Ok(()) + } +} +fn profile( + kind: Option<&StateMachineType>, + logging: Option<&LoggingConfiguration>, + tracing: Option<&TracingConfiguration>, + encryption: Option<&EncryptionConfiguration>, + publish: Option, + version: Option<&str>, +) -> Result<(), StepFunctionsError> { + if kind.is_some_and(|v| v != &StateMachineType::Standard) { + return Err(error(Code::StateMachineTypeNotSupported)); + } + if publish == Some(true) + || version.is_some() + || logging.is_some_and(|v| { + v.level.as_ref().is_some_and(|l| l != &LogLevel::Off) || !v.destinations.is_empty() + }) + || tracing.is_some_and(|v| v.enabled == Some(true)) + || encryption.is_some_and(|v| { + v.r#type != EncryptionType::AwsOwnedKey + || v.kms_key_id.is_some() + || v.kms_data_key_reuse_period_seconds.is_some() + }) + { + return Err(error(Code::ValidationException)); + } + Ok(()) +} +fn validate( + v: &input::ValidateStateMachineDefinitionInput, + compiler: &AslCompiler, +) -> Result { + profile(v.r#type.as_ref(), None, None, None, None, None)?; + let limit = match v.max_results { + None | Some(0) => 100, + Some(n) => usize::try_from(n) + .ok() + .filter(|n| *n <= 100) + .ok_or_else(|| error(Code::ValidationException))?, + }; + let mut response = output::ValidateStateMachineDefinitionOutput::default(); + let diagnostics = match compiler.compile(v.definition.as_bytes()) { + Ok(artifact) => { + response.result = ValidateStateMachineDefinitionResultCode::Ok; + artifact.warnings().to_vec() + } + Err(error) => { + response.result = ValidateStateMachineDefinitionResultCode::Fail; + vec![error.diagnostic().clone()] + } + }; + let warnings = v.severity == Some(ValidateStateMachineDefinitionSeverity::Warning); + let mut diagnostics: Vec<_> = diagnostics + .iter() + .filter(|d| warnings || d.severity() == DiagnosticSeverity::Error) + .map(|d| { + let mut out = ValidateStateMachineDefinitionDiagnostic::default(); + out.code = d.code().into(); + out.message = d.message().into(); + out.location = Some(d.location().into()); + out.severity = if d.severity() == DiagnosticSeverity::Error { + ValidateStateMachineDefinitionSeverity::Error + } else { + ValidateStateMachineDefinitionSeverity::Warning + }; + out + }) + .collect(); + response.truncated = Some(diagnostics.len() > limit); + diagnostics.truncate(limit); + response.diagnostics = diagnostics; + Ok(response) +} diff --git a/apps/rustack/src/stepfunctions_api/render.rs b/apps/rustack/src/stepfunctions_api/render.rs new file mode 100644 index 00000000..1c7fd799 --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/render.rs @@ -0,0 +1,187 @@ +//! Exact modeled M1 replies from immutable core captures. + +use std::{str, sync::Arc}; + +use rustack_stepfunctions_core::{ + ExecutionOutcome, ExecutionSnapshot, MachineSnapshot, MachineStatus, RegistryResult, +}; +use rustack_stepfunctions_http::Response; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode as Code, StepFunctionsOperation as Op, output, + types::{ + CloudWatchEventsExecutionDataDetails, EncryptionConfiguration, EncryptionType, + ExecutionListItem, ExecutionStatus, LogLevel, LoggingConfiguration, StateMachineListItem, + StateMachineStatus, StateMachineType, Tag, TracingConfiguration, + }, +}; + +use super::error::{error, source}; + +pub(super) fn text(bytes: &[u8]) -> Result { + str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|e| source(Code::InternalFailure, e)) +} +pub(super) fn status(value: &ExecutionSnapshot) -> Result { + Ok(match value.view().outcome().map(Arc::as_ref) { + None => ExecutionStatus::Running, + Some(ExecutionOutcome::Succeeded(_)) => ExecutionStatus::Succeeded, + Some(ExecutionOutcome::Failed(_)) => ExecutionStatus::Failed, + Some(ExecutionOutcome::TimedOut) => ExecutionStatus::TimedOut, + Some(ExecutionOutcome::Aborted(_)) => ExecutionStatus::Aborted, + _ => return Err(error(Code::InternalFailure)), + }) +} +fn off() -> LoggingConfiguration { + let mut value = LoggingConfiguration::default(); + value.level = Some(LogLevel::Off); + value +} +fn tracing() -> TracingConfiguration { + let mut value = TracingConfiguration::default(); + value.enabled = Some(false); + value +} +fn encryption() -> EncryptionConfiguration { + let mut value = EncryptionConfiguration::default(); + value.r#type = EncryptionType::AwsOwnedKey; + value +} +pub(super) fn tag(key: &str, value: &str) -> Tag { + let mut out = Tag::default(); + out.key = Some(key.into()); + out.value = Some(value.into()); + out +} +pub(super) fn machine_item(m: &MachineSnapshot) -> StateMachineListItem { + let mut out = StateMachineListItem::default(); + out.creation_date = m.creation_date(); + out.name = m.arn().name().as_str().into(); + out.state_machine_arn = m.arn().as_str().into(); + out.r#type = StateMachineType::Standard; + out +} +pub(super) fn execution_item( + e: &ExecutionSnapshot, +) -> Result { + let mut out = ExecutionListItem::default(); + out.execution_arn = e.arn().as_str().into(); + out.name = e.arn().name().as_str().into(); + out.start_date = e.start_date(); + out.state_machine_arn = e.arn().machine().as_str().into(); + out.status = status(e)?; + out.stop_date = e.view().stop_date(); + Ok(out) +} +pub(super) fn render(op: Op, result: RegistryResult) -> Result { + match (op, result) { + (Op::CreateStateMachine, RegistryResult::Machine(m)) => { + let mut out = output::CreateStateMachineOutput::default(); + out.creation_date = m.creation_date(); + out.state_machine_arn = m.arn().as_str().into(); + Ok(out.into()) + } + (Op::UpdateStateMachine, RegistryResult::Machine(m)) => { + let mut out = output::UpdateStateMachineOutput::default(); + out.update_date = m.update_date(); + out.revision_id = Some(m.revision().id().into()); + Ok(out.into()) + } + (Op::DeleteStateMachine, RegistryResult::Deleted) => { + Ok(output::DeleteStateMachineOutput::default().into()) + } + (Op::TagResource, RegistryResult::Tags(_)) => { + Ok(output::TagResourceOutput::default().into()) + } + (Op::UntagResource, RegistryResult::Tags(_)) => { + Ok(output::UntagResourceOutput::default().into()) + } + (Op::ListTagsForResource, RegistryResult::Machine(m)) => { + let mut out = output::ListTagsForResourceOutput::default(); + out.tags = m.tags().values().iter().map(|(k, v)| tag(k, v)).collect(); + Ok(out.into()) + } + (Op::ListStateMachineVersions, RegistryResult::Machine(_)) => { + Ok(output::ListStateMachineVersionsOutput::default().into()) + } + (Op::DescribeStateMachine, RegistryResult::Machine(m)) => describe_machine(&m), + (Op::StartExecution, RegistryResult::Execution(e)) => { + let mut out = output::StartExecutionOutput::default(); + out.execution_arn = e.arn().as_str().into(); + out.start_date = e.start_date(); + Ok(out.into()) + } + (Op::DescribeExecution, RegistryResult::Execution(e)) => describe_execution(&e), + (Op::StopExecution, RegistryResult::Execution(e)) => { + let mut out = output::StopExecutionOutput::default(); + out.stop_date = e + .view() + .stop_date() + .ok_or_else(|| error(Code::InternalFailure))?; + Ok(out.into()) + } + (Op::DescribeStateMachineForExecution, RegistryResult::Execution(e)) => { + let mut out = output::DescribeStateMachineForExecutionOutput::default(); + out.definition = text(e.revision().machine().original_bytes())?; + out.name = e.arn().machine().name().as_str().into(); + out.role_arn = e.revision().role().as_str().into(); + out.state_machine_arn = e.arn().machine().as_str().into(); + out.revision_id = Some(e.revision().id().into()); + out.update_date = e.revision().publication_date(); + out.logging_configuration = Some(off()); + out.tracing_configuration = Some(tracing()); + out.encryption_configuration = Some(encryption()); + Ok(out.into()) + } + _ => Err(error(Code::InternalFailure)), + } +} +fn describe_machine(m: &MachineSnapshot) -> Result { + let mut out = output::DescribeStateMachineOutput::default(); + out.definition = text(m.revision().machine().original_bytes())?; + out.creation_date = m.creation_date(); + out.name = m.arn().name().as_str().into(); + out.role_arn = m.revision().role().as_str().into(); + out.state_machine_arn = m.arn().as_str().into(); + out.revision_id = Some(m.revision().id().into()); + out.r#type = StateMachineType::Standard; + out.status = Some(match m.status() { + MachineStatus::Active => StateMachineStatus::Active, + MachineStatus::Deleting => StateMachineStatus::Deleting, + _ => return Err(error(Code::InternalFailure)), + }); + out.logging_configuration = Some(off()); + out.tracing_configuration = Some(tracing()); + out.encryption_configuration = Some(encryption()); + Ok(out.into()) +} +fn describe_execution(e: &ExecutionSnapshot) -> Result { + let mut out = output::DescribeExecutionOutput::default(); + out.execution_arn = e.arn().as_str().into(); + out.state_machine_arn = e.arn().machine().as_str().into(); + out.start_date = e.start_date(); + out.stop_date = e.view().stop_date(); + out.status = status(e)?; + out.name = Some(e.arn().name().as_str().into()); + out.input = Some(text(e.input().original_bytes())?); + out.trace_header = e.trace_header().map(str::to_owned); + let mut details = CloudWatchEventsExecutionDataDetails::default(); + details.included = Some(true); + out.input_details = Some(details.clone()); + match e.view().outcome().map(Arc::as_ref) { + Some(ExecutionOutcome::Succeeded(payload)) => { + out.output = Some(text(payload.original_bytes())?); + out.output_details = Some(details); + } + Some(ExecutionOutcome::Failed(f)) => { + out.error = f.error().map(str::to_owned); + out.cause = f.cause().map(str::to_owned); + } + Some(ExecutionOutcome::Aborted(f)) => { + out.error = f.details().error().map(str::to_owned); + out.cause = f.details().cause().map(str::to_owned); + } + _ => {} + } + Ok(out.into()) +} diff --git a/apps/rustack/src/stepfunctions_api/tests.rs b/apps/rustack/src/stepfunctions_api/tests.rs new file mode 100644 index 00000000..8f51bd62 --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/tests.rs @@ -0,0 +1,942 @@ +//! Real typed HTTP → API owner → registry → execution semantics. + +use std::{sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::{Request as HttpRequest, StatusCode}; +use http_body_util::{BodyExt, Full}; +use rustack_stepfunctions_core::{ + ExecutionLimits, ExecutionSupervisor, HistoryLimits, RegistryLimits, RegistrySupervisor, + RuntimeLimits, +}; +use rustack_stepfunctions_http::{ + CONTENT_TYPE, RequestScope, StepFunctionsHttpConfig, StepFunctionsHttpService, +}; +use serde_json::{Value, json}; + +use super::*; + +struct Fixture { + service: StepFunctionsHttpService, + api: ApiSupervisor, + registry: RegistrySupervisor, + handle: Arc, +} +impl Fixture { + fn new(limits: ApiLimits) -> Self { + let scope = TenantScope::default(); + let runtime = ExecutionSupervisor::new( + RuntimeLimits::default(), + HistoryLimits::default(), + 256 * 1024 * 1024, + None, + ) + .unwrap(); + let (registry, owner) = Registry::new( + scope.clone(), + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + ) + .unwrap(); + Self::from_registry(registry, owner, TaskSupport::Disabled, limits) + } + fn from_registry( + registry: Registry, + owner: RegistrySupervisor, + tasks: TaskSupport, + limits: ApiLimits, + ) -> Self { + let scope = registry.scope().clone(); + let (handle, api) = StepFunctionsApi::new(scope.clone(), tasks, registry, limits).unwrap(); + let config = StepFunctionsHttpConfig::new( + RequestScope::new( + scope.partition().into(), + scope.region().into(), + scope.account().into(), + ) + .unwrap(), + true, + None, + ) + .unwrap(); + Self { + service: StepFunctionsHttpService::new(handle.clone(), config), + api, + registry: owner, + handle, + } + } + async fn call(&self, action: &str, input: Value) -> (StatusCode, Value) { + let request = HttpRequest::builder() + .method("POST") + .uri("/") + .header("content-type", CONTENT_TYPE) + .header("x-amz-target", format!("AWSStepFunctions.{action}")) + .body(Full::new(Bytes::from(serde_json::to_vec(&input).unwrap()))) + .unwrap(); + let response = self.service.handle(request).await; + let status = response.status(); + assert!(response.headers().contains_key("x-amzn-requestid")); + let body = response.into_body().collect().await.unwrap().to_bytes(); + ( + status, + if body.is_empty() { + Value::Null + } else { + serde_json::from_slice(&body).unwrap() + }, + ) + } + async fn create(&self, name: &str, definition: Value) -> String { + let(status,response)=self.call("CreateStateMachine",json!({"name":name,"roleArn":"arn:aws:iam::000000000000:role/Workflow","definition":definition.to_string()})).await; + assert_eq!(status, StatusCode::OK, "{response}"); + response["stateMachineArn"].as_str().unwrap().into() + } + async fn wait(&self, arn: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let (status, response) = self + .call("DescribeExecution", json!({"executionArn":arn})) + .await; + assert_eq!(status, StatusCode::OK); + if response["status"] != "RUNNING" { + return response; + } + assert!(Instant::now() < deadline); + tokio::task::yield_now().await; + } + } + async fn close(&mut self) { + let deadline = Instant::now() + Duration::from_secs(5); + let api = self.api.shutdown(deadline).await; + let registry = self.registry.shutdown(deadline).await; + assert!(api.is_ok(), "{api:?}"); + assert!(registry.is_ok(), "{registry:?}"); + } +} + +mod callback; +fn succeed() -> Value { + json!({"StartAt":"Done","States":{"Done":{"Type":"Succeed"}}}) +} +fn wait() -> Value { + json!({"StartAt":"Waiting","States":{"Waiting":{"Type":"Wait","Seconds":60,"Next":"Done"},"Done":{"Type":"Succeed"}}}) +} +async fn tag_roundtrip(f: &Fixture, arn: &str) { + let (status, _) = f + .call( + "TagResource", + json!({"resourceArn":arn,"tags":[{"key":"team","value":"workflow"}]}), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (_, tags) = f + .call("ListTagsForResource", json!({"resourceArn":arn})) + .await; + assert_eq!(tags["tags"][0]["value"], "workflow"); + assert_eq!( + f.call( + "UntagResource", + json!({"resourceArn":arn,"tagKeys":["team"]}) + ) + .await + .0, + StatusCode::OK + ); +} +#[tokio::test] +async fn test_should_serve_all_fifteen_real_m1_operations_and_captured_revisions() { + let mut f = Fixture::new(ApiLimits::default()); + let arn = f.create("Workflow", succeed()).await; + let (status, original) = f + .call("DescribeStateMachine", json!({"stateMachineArn":arn})) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(original["status"], "ACTIVE"); + assert!(original["creationDate"].is_number()); + tag_roundtrip(&f, &arn).await; + let (_, started) = f + .call( + "StartExecution", + json!({"stateMachineArn":arn,"name":"first","input":" {\"n\":7} ","traceHeader":""}), + ) + .await; + let execution = started["executionArn"].as_str().unwrap(); + let finished = f.wait(execution).await; + assert_eq!(finished["status"], "SUCCEEDED"); + assert_eq!(finished["input"], " {\"n\":7} "); + assert_eq!( + serde_json::from_str::(finished["output"].as_str().unwrap()).unwrap(), + json!({"n":7}) + ); + let (_, updated) = f + .call( + "UpdateStateMachine", + json!({"stateMachineArn":arn,"definition":wait().to_string()}), + ) + .await; + assert_ne!(updated["revisionId"], original["revisionId"]); + let (_, captured) = f + .call( + "DescribeStateMachineForExecution", + json!({"executionArn":execution}), + ) + .await; + assert_eq!(captured["revisionId"], original["revisionId"]); + assert_eq!(captured["definition"], original["definition"]); + assert_eq!(captured["updateDate"], original["creationDate"]); + let (_, list) = f + .call("ListExecutions", json!({"stateMachineArn":arn})) + .await; + assert_eq!(list["executions"].as_array().unwrap().len(), 1); + let (_, history) = f + .call("GetExecutionHistory", json!({"executionArn":execution})) + .await; + assert_eq!( + history["events"].as_array().unwrap().last().unwrap()["type"], + "ExecutionSucceeded" + ); + assert_eq!( + f.call( + "ValidateStateMachineDefinition", + json!({"definition":succeed().to_string()}) + ) + .await + .1["result"], + "OK" + ); + assert_eq!( + f.call("ListStateMachines", json!({})).await.1["stateMachines"] + .as_array() + .unwrap() + .len(), + 1 + ); + let (_, running) = f + .call( + "StartExecution", + json!({"stateMachineArn":arn,"name":"stop"}), + ) + .await; + let stop = running["executionArn"].as_str().unwrap(); + let (_, stopped) = f + .call( + "StopExecution", + json!({"executionArn":stop,"error":"","cause":""}), + ) + .await; + let terminal = f.wait(stop).await; + assert_eq!(terminal["status"], "ABORTED"); + assert_eq!(terminal["stopDate"], stopped["stopDate"]); + assert_eq!(terminal["error"], ""); + assert_eq!( + f.call("DeleteStateMachine", json!({"stateMachineArn":arn})) + .await + .0, + StatusCode::OK + ); + assert_eq!( + f.call("DescribeExecution", json!({"executionArn":execution})) + .await + .0, + StatusCode::OK + ); + f.close().await; +} +#[tokio::test] +async fn test_should_filter_validate_diagnostics_without_mutation_and_reject_profiles() { + let mut f = Fixture::new(ApiLimits::default()); + let definition = json!({"StartAt":"Done","States":{"Done":{"Type":"Succeed"},"Unused":{"Type":"Succeed"},"Other":{"Type":"Succeed"}}}); + let (_, errors) = f + .call( + "ValidateStateMachineDefinition", + json!({"definition":definition.to_string()}), + ) + .await; + assert_eq!(errors["result"], "OK"); + assert_eq!(errors["diagnostics"], json!([])); + let (_, warnings) = f + .call( + "ValidateStateMachineDefinition", + json!({"definition":definition.to_string(),"severity":"WARNING","maxResults":1}), + ) + .await; + assert_eq!(warnings["diagnostics"].as_array().unwrap().len(), 1); + assert_eq!(warnings["truncated"], true); + let (status, invalid) = f + .call( + "ValidateStateMachineDefinition", + json!({"definition":"not json"}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(invalid["result"], "FAIL"); + for extra in [ + json!({"type":"EXPRESS"}), + json!({"publish":true}), + json!({"tracingConfiguration":{"enabled":true}}), + json!({"loggingConfiguration":{"level":"ALL"}}), + json!({"encryptionConfiguration":{"type":"CUSTOMER_MANAGED_KMS_KEY"}}), + ] { + let mut input = json!({"name":"Rejected","roleArn":"arn:aws:iam::000000000000:role/Workflow","definition":succeed().to_string()}); + input + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + assert_eq!( + f.call("CreateStateMachine", input).await.0, + StatusCode::BAD_REQUEST + ); + } + assert_eq!( + f.call("ListStateMachines", json!({})).await.1["stateMachines"] + .as_array() + .unwrap() + .len(), + 0 + ); + f.close().await; +} +#[tokio::test] +async fn test_should_bind_stable_machine_cursor_to_full_query_and_replay_after_deletion() { + let mut f = Fixture::new(ApiLimits::default()); + let a = f.create("A", succeed()).await; + let b = f.create("B", succeed()).await; + let (_, first) = f.call("ListStateMachines", json!({"maxResults":1})).await; + let token = first["nextToken"].as_str().unwrap(); + assert_eq!(token.len(), 64); + f.call("DeleteStateMachine", json!({"stateMachineArn":b})) + .await; + f.create("C", succeed()).await; + for _ in 0..2 { + let (status, next) = f + .call( + "ListStateMachines", + json!({"maxResults":1,"nextToken":token}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(next["stateMachines"][0]["name"], "B"); + assert!(next.get("nextToken").is_none()); + } + for (op, input) in [ + ( + "ListStateMachines", + json!({"maxResults":2,"nextToken":token}), + ), + ( + "ListExecutions", + json!({"stateMachineArn":a,"maxResults":1,"nextToken":token}), + ), + ( + "ListStateMachines", + json!({"maxResults":1,"nextToken":"f".repeat(64)}), + ), + ] { + let (status, error) = f.call(op, input).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error["__type"], "InvalidToken"); + } + f.close().await; +} +#[tokio::test] +async fn test_should_page_reverse_and_data_disabled_history_with_fixed_high_water() { + let mut f = Fixture::new(ApiLimits::default()); + let arn = f.create("History", succeed()).await; + let (_, started) = f + .call( + "StartExecution", + json!({"stateMachineArn":arn,"input":"{\"secret\":7}"}), + ) + .await; + let arn = started["executionArn"].as_str().unwrap(); + f.wait(arn).await; + let(_,first)=f.call("GetExecutionHistory",json!({"executionArn":arn,"maxResults":1,"reverseOrder":true,"includeExecutionData":false})).await; + assert_eq!(first["events"][0]["type"], "ExecutionSucceeded"); + assert!( + first["events"][0]["executionSucceededEventDetails"] + .get("output") + .is_none() + ); + let token = first["nextToken"].clone(); + let(_,next)=f.call("GetExecutionHistory",json!({"executionArn":arn,"maxResults":1,"reverseOrder":true,"includeExecutionData":false,"nextToken":token})).await; + assert!(next["events"][0]["id"].as_i64().unwrap() < first["events"][0]["id"].as_i64().unwrap()); + let(_,invalid)=f.call("GetExecutionHistory",json!({"executionArn":arn,"maxResults":1,"reverseOrder":false,"includeExecutionData":false,"nextToken":token})).await; + assert_eq!(invalid["__type"], "InvalidToken"); + f.close().await; +} +#[tokio::test] +async fn test_should_preserve_running_start_idempotency_and_raw_input_rejection_over_http() { + let mut f = Fixture::new(ApiLimits::default()); + let arn = f.create("Wait", wait()).await; + let input = json!({"stateMachineArn":arn,"name":"run","input":" {\"n\":7}"}); + let (_, first) = f.call("StartExecution", input.clone()).await; + let (_, second) = f.call("StartExecution", input).await; + assert_eq!(first, second); + assert_eq!( + f.call( + "StartExecution", + json!({"stateMachineArn":arn,"name":"run","input":"{\"n\":7}"}) + ) + .await + .1["__type"], + "ExecutionAlreadyExists" + ); + assert_eq!( + f.call("StartExecution", json!({"stateMachineArn":arn,"input":""})) + .await + .0, + StatusCode::BAD_REQUEST + ); + f.close().await; +} +#[tokio::test] +async fn test_should_report_explicit_unsupported_fields_and_keep_scope_and_readiness() { + let mut f = Fixture::new(ApiLimits::default()); + assert!(f.handle.is_ready()); + let arn = f.create("Scope", succeed()).await; + assert_eq!( + f.call( + "UpdateStateMachine", + json!({"stateMachineArn":arn,"loggingConfiguration":{"level":"OFF"}}) + ) + .await + .1["__type"], + "MissingRequiredParameter" + ); + for (op, input) in [ + ( + "DescribeStateMachine", + json!({"stateMachineArn":arn,"includedData":"METADATA_ONLY"}), + ), + ( + "ListExecutions", + json!({"stateMachineArn":arn,"statusFilter":"PENDING_REDRIVE"}), + ), + ("SendTaskHeartbeat", json!({"taskToken":"opaque"})), + ] { + assert_eq!(f.call(op, input).await.1["__type"], "UnsupportedOperation"); + } + assert_eq!( + f.call( + "DescribeStateMachine", + json!({"stateMachineArn":"arn:aws:states:us-west-2:000000000000:stateMachine:Scope"}) + ) + .await + .1["__type"], + "InvalidArn" + ); + f.close().await; + assert!(!f.handle.is_ready()); +} + +#[tokio::test] +async fn test_should_map_missing_tag_resources_and_reject_cumulative_tag_quota_atomically() { + let mut f = Fixture::new(ApiLimits::default()); + let missing = "arn:aws:states:us-east-1:000000000000:stateMachine:Missing"; + for (op, input) in [ + ("TagResource", json!({"resourceArn":missing,"tags":[]})), + ("UntagResource", json!({"resourceArn":missing,"tagKeys":[]})), + ("ListTagsForResource", json!({"resourceArn":missing})), + ] { + let (status, reply) = f.call(op, input).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(reply["__type"], "ResourceNotFound"); + } + let arn = f.create("Tags", succeed()).await; + let tags: Vec<_> = (0..50) + .map(|n| json!({"key":format!("key{n}"),"value":"v"})) + .collect(); + assert_eq!( + f.call("TagResource", json!({"resourceArn":arn,"tags":tags})) + .await + .0, + StatusCode::OK + ); + let (status, reply) = f + .call( + "TagResource", + json!({"resourceArn":arn,"tags":[{"key":"extra"}]}), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(reply["__type"], "TooManyTags"); + assert_eq!( + f.call("ListTagsForResource", json!({"resourceArn":arn})) + .await + .1["tags"] + .as_array() + .unwrap() + .len(), + 50 + ); + f.close().await; +} +#[tokio::test] +async fn test_should_hold_execution_status_cursor_boundary_across_later_stop() { + let mut f = Fixture::new(ApiLimits::default()); + let arn = f.create("Filtered", wait()).await; + let mut executions = Vec::new(); + for name in ["a", "b", "c"] { + executions.push( + f.call("StartExecution", json!({"stateMachineArn":arn,"name":name})) + .await + .1["executionArn"] + .as_str() + .unwrap() + .to_owned(), + ); + } + let (_, first) = f + .call( + "ListExecutions", + json!({"stateMachineArn":arn,"statusFilter":"RUNNING","maxResults":1}), + ) + .await; + let token = first["nextToken"].clone(); + for execution in executions { + f.call("StopExecution", json!({"executionArn":execution})) + .await; + } + let(status,second)=f.call("ListExecutions",json!({"stateMachineArn":arn,"statusFilter":"RUNNING","maxResults":1,"nextToken":token})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(second["executions"][0]["status"], "RUNNING"); + assert_ne!( + first["executions"][0]["executionArn"], + second["executions"][0]["executionArn"] + ); + assert_eq!( + f.call( + "ListExecutions", + json!({"stateMachineArn":arn,"statusFilter":"ABORTED","maxResults":1,"nextToken":token}) + ) + .await + .1["__type"], + "InvalidToken" + ); + f.close().await; +} +fn controlled( + work: impl FnOnce() -> Result + Send + 'static, + deadline: Instant, + panic_worker: bool, +) -> ( + Arc, + ApiSupervisor, + RegistrySupervisor, + oneshot::Receiver, +) { + let runtime = ExecutionSupervisor::new( + RuntimeLimits::default(), + HistoryLimits::default(), + 256 * 1024 * 1024, + None, + ) + .unwrap(); + let (registry, registry_owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + ) + .unwrap(); + let (api, mut owner, stop) = StepFunctionsApi::compose( + TenantScope::default(), + TaskSupport::Disabled, + registry, + ApiLimits::new(1, 1, 32, 8 * 1024 * 1024).unwrap(), + ) + .unwrap(); + let state = owner.state.as_mut().unwrap(); + let context = state.io.context.clone(); + let (reply, receiver) = oneshot::channel(); + let permit = api.admission.capacity.clone().try_acquire_owned().unwrap(); + let job = state.jobs.spawn(async move { + assert!(!panic_worker, "controlled API worker failure"); + worker::cpu(&context, deadline, work).await + }); + state.slots.insert(job.id(), Slot { reply, permit }); + let supervisor = ApiSupervisor { + closing: api.admission.closing.clone(), + join: Some(spawn(owner.run(stop))), + result: None, + }; + (api, supervisor, registry_owner, receiver) +} +fn ready_response() -> Response { + rustack_stepfunctions_model::output::ListStateMachinesOutput::default().into() +} +#[tokio::test] +async fn test_should_join_same_blocking_work_after_shutdown_observation_expires() { + let (entered, ready) = oneshot::channel(); + let (release, gate) = std::sync::mpsc::channel::<()>(); + let (api, mut owner, mut registry, receiver) = controlled( + move || { + let _sent = entered.send(()); + let _released = gate.recv(); + Ok(ready_response()) + }, + Instant::now() + Duration::from_secs(5), + false, + ); + ready.await.unwrap(); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(5)) + .await, + Err(ApiError::Deadline) + )); + assert_eq!(api.admission.capacity.available_permits(), 0); + drop(release); + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); + let reply = receiver.await.unwrap(); + assert!(reply.result.is_ok()); + assert_eq!(api.admission.capacity.available_permits(), 0); + drop(reply.permit); + assert_eq!(api.admission.capacity.available_permits(), 1); + registry + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_retain_root_slot_after_abort_even_when_blocking_work_finishes() { + let (entered, ready) = oneshot::channel(); + let (release, gate) = std::sync::mpsc::channel::<()>(); + let (done, finished) = oneshot::channel(); + let (api, mut owner, mut registry, _receiver) = controlled( + move || { + let _sent = entered.send(()); + let _released = gate.recv(); + let _done = done.send(()); + Ok(ready_response()) + }, + Instant::now() + Duration::from_secs(5), + false, + ); + ready.await.unwrap(); + owner.join.as_ref().unwrap().abort(); + assert!( + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + drop(release); + finished.await.unwrap(); + assert_eq!(api.admission.capacity.available_permits(), 0); + assert!(!api.is_ready()); + registry + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_keep_worker_diagnostic_after_quarantined_shutdown() { + let (api, mut owner, mut registry, receiver) = controlled( + || Ok(ready_response()), + Instant::now() + Duration::from_secs(5), + true, + ); + let reply = receiver.await.unwrap(); + assert_eq!(reply.result.unwrap_err().code, Code::InternalFailure); + assert!(matches!( + api.diagnostic().as_deref(), + Some(ApiError::Worker(_)) + )); + assert!( + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + assert!(matches!( + api.diagnostic().as_deref(), + Some(ApiError::Worker(_)) + )); + assert_eq!(api.admission.capacity.available_permits(), 0); + registry + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} +#[tokio::test(start_paused = true)] +async fn test_should_join_cpu_then_reject_a_ready_result_after_its_deadline() { + let (entered, mut ready) = oneshot::channel(); + let (release, gate) = std::sync::mpsc::channel::<()>(); + let deadline = Instant::now() + Duration::from_secs(5); + let (_, mut owner, mut registry, receiver) = controlled( + move || { + let _sent = entered.send(()); + let _released = gate.recv(); + Ok(ready_response()) + }, + deadline, + false, + ); + let observed = std::time::Instant::now() + Duration::from_secs(5); + loop { + match ready.try_recv() { + Ok(()) => break, + Err(oneshot::error::TryRecvError::Empty) => { + assert!(std::time::Instant::now() < observed); + tokio::task::yield_now().await; + } + Err(oneshot::error::TryRecvError::Closed) => panic!("CPU worker did not enter"), + } + } + tokio::time::advance(Duration::from_secs(6)).await; + drop(release); + let error = receiver.await.unwrap().result.unwrap_err(); + assert_eq!(error.code, Code::RequestTimeoutException); + assert_eq!(error.status_code, StatusCode::REQUEST_TIMEOUT); + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); + registry + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} +#[derive(Default)] +struct WakeFlag(std::sync::atomic::AtomicBool); +impl std::task::Wake for WakeFlag { + fn wake(self: Arc) { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } +} +struct CaptureContext(mpsc::Sender); +#[async_trait] +impl StepFunctionsHandler for CaptureContext { + async fn handle( + &self, + context: RequestContext, + _request: Request, + ) -> Result { + self.0.send(context).await.unwrap(); + Ok(ready_response()) + } +} +#[tokio::test(start_paused = true)] +async fn test_should_reject_an_already_ready_typed_reply_when_observer_resumes_after_deadline() { + let mut fixture = Fixture::new(ApiLimits::default()); + let (tx, mut rx) = mpsc::channel(1); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into()).unwrap(), + true, + None, + ) + .unwrap() + .with_budgets( + rustack_core::http::BodyBudget::control(), + Duration::from_secs(1), + ) + .unwrap(); + let service = StepFunctionsHttpService::new(Arc::new(CaptureContext(tx)), config); + let request = HttpRequest::builder() + .method("POST") + .uri("/") + .header("content-type", CONTENT_TYPE) + .header("x-amz-target", "AWSStepFunctions.ListStateMachines") + .body(Full::new(Bytes::from_static(b"{}"))) + .unwrap(); + service.handle(request).await; + let context = rx.recv().await.unwrap(); + let deadline = context.deadline(); + let api = fixture.handle.clone(); + let request = Request::decode( + rustack_stepfunctions_model::StepFunctionsOperation::ListStateMachines, + b"{}", + ) + .unwrap(); + let mut future = api.handle(context, request); + let flag = Arc::new(WakeFlag::default()); + let waker = std::task::Waker::from(flag.clone()); + let mut task = std::task::Context::from_waker(&waker); + assert!(future.as_mut().poll(&mut task).is_pending()); + let observed = std::time::Instant::now() + Duration::from_secs(5); + while !flag.0.load(std::sync::atomic::Ordering::SeqCst) { + assert!(std::time::Instant::now() < observed); + assert!(Instant::now() < deadline); + tokio::task::yield_now().await; + } + assert!(Instant::now() < deadline); + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!( + future.await.unwrap_err().code, + Code::RequestTimeoutException + ); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_reject_mismatched_trusted_http_scope_before_dispatch() { + let mut fixture = Fixture::new(ApiLimits::default()); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-west-2".into(), "111111111111".into()).unwrap(), + true, + None, + ) + .unwrap(); + fixture.service = StepFunctionsHttpService::new(fixture.handle.clone(), config); + let (status, body) = fixture.call("ListStateMachines", json!({})).await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(body["__type"], "AccessDeniedException"); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_preserve_optional_failure_fields_in_describe_and_terminal_history() { + let mut f = Fixture::new(ApiLimits::default()); + for (name, state) in [ + ("OmittedFailure", json!({"Type":"Fail"})), + ( + "NamedFailure", + json!({"Type":"Fail","Error":"BusinessError","Cause":"quoted \"cause\" 中文"}), + ), + ] { + let arn = f + .create(name, json!({"StartAt":"F","States":{"F":state}})) + .await; + let (_, started) = f + .call("StartExecution", json!({"stateMachineArn":arn})) + .await; + let execution = started["executionArn"].as_str().unwrap(); + let description = f.wait(execution).await; + let (_, history) = f + .call( + "GetExecutionHistory", + json!({"executionArn":execution,"reverseOrder":true}), + ) + .await; + assert_eq!(description["status"], "FAILED"); + assert_eq!(description.get("error"), state.get("Error")); + assert_eq!(description.get("cause"), state.get("Cause")); + assert_eq!(description.get("output"), None); + let details = &history["events"][0]["executionFailedEventDetails"]; + assert_eq!(details.get("error"), description.get("error")); + assert_eq!(details.get("cause"), description.get("cause")); + } + f.close().await; +} +#[tokio::test] +async fn test_should_map_execution_timeout_consistently_after_start_request_has_completed() { + let mut f = Fixture::new(ApiLimits::default()); + let mut definition = wait(); + definition["TimeoutSeconds"] = json!(1); + let arn = f.create("Timeout", definition).await; + let (_, started) = f + .call("StartExecution", json!({"stateMachineArn":arn})) + .await; + let execution = started["executionArn"].as_str().unwrap(); + let description = f.wait(execution).await; + let (_, history) = f + .call( + "GetExecutionHistory", + json!({"executionArn":execution,"reverseOrder":true}), + ) + .await; + assert_eq!(description["status"], "TIMED_OUT"); + assert!(description.get("error").is_none()); + assert!(description.get("cause").is_none()); + let details = &history["events"][0]["executionTimedOutEventDetails"]; + assert!(details.get("error").is_none()); + assert!(details.get("cause").is_none()); + assert_eq!(history["events"][0]["type"], "ExecutionTimedOut"); + f.close().await; +} + +#[tokio::test] +async fn test_should_list_only_empty_published_inventory_for_existing_scoped_machines() { + let mut fixture = Fixture::new(ApiLimits::default()); + let arn = fixture.create("UnpublishedInventory", succeed()).await; + let before = fixture + .call("DescribeStateMachine", json!({"stateMachineArn":arn})) + .await + .1; + for maximum in [None, Some(0), Some(1), Some(1000)] { + let mut input = json!({"stateMachineArn":arn}); + if let Some(maximum) = maximum { + input["maxResults"] = json!(maximum); + } + let (status, response) = fixture.call("ListStateMachineVersions", input).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response, json!({"stateMachineVersions":[]})); + } + for (input, code) in [ + ( + json!({"stateMachineArn":arn,"nextToken":"unissued"}), + "InvalidToken", + ), + ( + json!({"stateMachineArn":arn,"nextToken":""}), + "InvalidToken", + ), + ( + json!({"stateMachineArn":arn,"nextToken":"x".repeat(1025)}), + "InvalidToken", + ), + ( + json!({"stateMachineArn":arn,"nextToken":"é"}), + "InvalidToken", + ), + ( + json!({"stateMachineArn":arn,"maxResults":-1}), + "ValidationException", + ), + ( + json!({"stateMachineArn":arn,"maxResults":1001}), + "ValidationException", + ), + (json!({"stateMachineArn":format!("{arn}:1")}), "InvalidArn"), + ( + json!({"stateMachineArn":format!("{arn}:alias")}), + "InvalidArn", + ), + ( + json!({"stateMachineArn":"arn:aws:states:us-west-2:000000000000:stateMachine:Foreign"}), + "InvalidArn", + ), + ( + json!({"stateMachineArn":"arn:aws:states:us-east-1:000000000000:stateMachine:Missing"}), + "StateMachineDoesNotExist", + ), + ] { + let (status, response) = fixture.call("ListStateMachineVersions", input).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["__type"], code); + } + let (status, response) = fixture + .call( + "UpdateStateMachine", + json!({"stateMachineArn":arn,"publish":true}), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["__type"], "ValidationException"); + assert_eq!( + fixture + .call("DescribeStateMachine", json!({"stateMachineArn":arn})) + .await + .1, + before + ); + assert_eq!( + fixture + .call("DeleteStateMachine", json!({"stateMachineArn":arn})) + .await + .0, + StatusCode::OK + ); + let (status, response) = fixture + .call("ListStateMachineVersions", json!({"stateMachineArn":arn})) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(response["__type"], "StateMachineDoesNotExist"); + fixture.close().await; +} diff --git a/apps/rustack/src/stepfunctions_api/tests/callback.rs b/apps/rustack/src/stepfunctions_api/tests/callback.rs new file mode 100644 index 00000000..e13dd227 --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/tests/callback.rs @@ -0,0 +1,746 @@ +//! Real public callback requests over a registered observed ASL port. + +use rustack_stepfunctions_core::{ + AslError, CallbackCommand, CallbackError, CallbackInvocationMetadata, CallbackLimits, + CallbackRegistry, CallbackSupervisor, CallbackToken, ExecutionOutcome, LambdaTaskCommand, + LambdaTaskPort, Payload, RegistryCommand, RegistryResult, StateSupport, TaskCall, TaskFailure, + TaskResponder, +}; +use rustack_stepfunctions_model::StepFunctionsOperation as Op; +use tokio::time::{sleep, sleep_until, timeout_at}; +use uuid::Uuid; + +use super::*; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} +#[derive(Debug)] +struct Port(mpsc::Sender<(LambdaTaskCommand, TaskResponder)>); +impl LambdaTaskPort for Port { + fn start(&self, command: LambdaTaskCommand) -> Result { + let (call, responder) = TaskCall::channel(); + self.0 + .try_send((command, responder)) + .map_err(|_| TaskFailure::service_exception())?; + Ok(call) + } +} +struct CallbackFixture { + inner: Fixture, + registry: Registry, + callbacks: CallbackSupervisor, + calls: mpsc::Receiver<(LambdaTaskCommand, TaskResponder)>, +} +impl CallbackFixture { + fn new() -> Self { + Self::scoped(TenantScope::default()) + } + fn scoped(scope: TenantScope) -> Self { + let (callbacks, callback_owner) = + CallbackRegistry::new(scope.clone(), CallbackLimits::default()).unwrap(); + let (sender, calls) = mpsc::channel(8); + let mut runtime = ExecutionSupervisor::new( + RuntimeLimits::new(8, 1, 2).unwrap(), + HistoryLimits::default(), + 256 * 1024 * 1024, + Some(Arc::new(Port(sender))), + ) + .unwrap(); + runtime.install_callbacks(callbacks).unwrap(); + let (registry, owner) = Registry::new_with_state_support( + scope, + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + StateSupport::Structured, + ) + .unwrap(); + let inner = Fixture::from_registry( + registry.clone(), + owner, + TaskSupport::Lambda, + ApiLimits::default(), + ); + Self { + inner, + registry, + callbacks: callback_owner, + calls, + } + } + async fn start(&self, extra: Value) -> String { + let mut task = json!({"Type":"Task","Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken","TimeoutSeconds":60,"Parameters":{"FunctionName":"Worker","Payload":{"token.$":"$$.Task.Token","retry.$":"$$.State.RetryCount"}},"End":true}); + task.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + let arn = self + .inner + .create("Callback", json!({"StartAt":"Call","States":{"Call":task}})) + .await; + let (status, response) = self + .inner + .call("StartExecution", json!({"stateMachineArn":arn})) + .await; + assert_eq!(status, StatusCode::OK); + response["executionArn"].as_str().unwrap().into() + } + async fn next(&mut self) -> (LambdaTaskCommand, TaskResponder) { + timeout_at(deadline(), self.calls.recv()) + .await + .unwrap() + .unwrap() + } + async fn pending(&self, execution: &str) { + assert_eq!( + self.inner + .call("DescribeExecution", json!({"executionArn":execution})) + .await + .1["status"], + "RUNNING" + ); + } + async fn history(&self, execution: &str) -> Value { + self.inner + .call("GetExecutionHistory", json!({"executionArn":execution})) + .await + .1 + } + async fn submitted(&self, execution: &str) { + timeout_at(deadline(), async { + loop { + if self.history(execution).await["events"] + .as_array() + .unwrap() + .iter() + .any(|v| v["type"] == "TaskSubmitted") + { + break; + } + sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + } + async fn close(&mut self) { + self.inner.close().await; + self.callbacks.shutdown(deadline()).await.unwrap(); + } +} +fn token(command: &LambdaTaskCommand) -> String { + command.dispatch().parameters().value()["Payload"]["token"] + .as_str() + .unwrap() + .into() +} +fn report( + command: &mut LambdaTaskCommand, + responder: &mut TaskResponder, + result: Result<(), TaskFailure>, +) { + responder.started().unwrap(); + command + .take_callback_reporter() + .unwrap() + .unwrap() + .complete_at( + result.map(|()| CallbackInvocationMetadata::new(Uuid::new_v4(), "$LATEST").unwrap()), + Instant::now(), + ) + .unwrap(); +} +fn finish(mut command: LambdaTaskCommand, mut responder: TaskResponder) { + report(&mut command, &mut responder, Ok(())); + responder.finish(Ok(Payload::parse(b"null").unwrap())); +} +async fn success(f: &CallbackFixture, token: &str, output: Value) { + let (status, body) = f + .inner + .call( + "SendTaskSuccess", + json!({"taskToken":token,"output":output.to_string()}), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, Value::Null); +} +#[rstest::rstest] +#[case(true, json!({"answer":42}))] +#[case(false, json!([1,"value",null]))] +#[case(false, json!(true))] +#[case(false, Value::Null)] +#[tokio::test] +async fn test_should_route_public_success_before_or_after_invocation_without_double_encoding( + #[case] early: bool, + #[case] output: Value, +) { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + if early { + success(&f, &token, output.clone()).await; + f.pending(&execution).await; + } + finish(command, responder); + if !early { + f.submitted(&execution).await; + f.pending(&execution).await; + success(&f, &token, output.clone()).await; + } + let done = f.inner.wait(&execution).await; + assert_eq!(done["status"], "SUCCEEDED"); + assert_eq!( + serde_json::from_str::(done["output"].as_str().unwrap()).unwrap(), + output + ); + assert!(!f.history(&execution).await.to_string().contains(&token)); + for action in ["SendTaskSuccess", "SendTaskFailure", "SendTaskHeartbeat"] { + let mut input = json!({"taskToken":token}); + if action == "SendTaskSuccess" { + input["output"] = json!("null"); + } + let (status, body) = f.inner.call(action, input).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["__type"], "TaskTimedOut"); + assert!(!body.to_string().contains(&token)); + } + f.close().await; +} +#[tokio::test] +async fn test_should_preserve_real_invocation_failure_after_public_early_success() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (mut command, mut responder) = f.next().await; + success(&f, &token(&command), json!({"falseSuccess":true})).await; + report( + &mut command, + &mut responder, + Err(TaskFailure::new("HandlerFailure", Some("actual failure")).unwrap()), + ); + responder.finish(Err(TaskFailure::service_exception())); + let done = f.inner.wait(&execution).await; + assert_eq!(done["status"], "FAILED"); + assert_eq!(done["error"], "HandlerFailure"); + assert!(done.get("output").is_none()); + f.close().await; +} +#[rstest::rstest] +#[case(None, None)] +#[case(Some(""), Some(""))] +#[case(Some("ExternalFailure"), Some("quoted \"cause\" 中文"))] +#[tokio::test] +async fn test_should_route_public_failure_and_preserve_omitted_or_empty_fields( + #[case] error: Option<&str>, + #[case] cause: Option<&str>, +) { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + finish(command, responder); + f.submitted(&execution).await; + let mut input = json!({"taskToken":token}); + if let Some(error) = error { + input["error"] = json!(error); + } + if let Some(cause) = cause { + input["cause"] = json!(cause); + } + assert_eq!( + f.inner.call("SendTaskFailure", input).await, + (StatusCode::OK, Value::Null) + ); + let done = f.inner.wait(&execution).await; + assert_eq!(done["status"], "FAILED"); + assert_eq!(done.get("error"), error.map(|_| &done["error"])); + assert_eq!(done["error"], json!(error)); + assert_eq!(done["cause"], json!(cause)); + assert!(!f.history(&execution).await.to_string().contains(&token)); + f.close().await; +} +#[tokio::test] +async fn test_should_route_public_heartbeat_without_history_mutation_then_complete() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({"HeartbeatSeconds":30})).await; + let (command, responder) = f.next().await; + let token = token(&command); + finish(command, responder); + f.submitted(&execution).await; + let before = f.history(&execution).await; + for _ in 0..3 { + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await, + (StatusCode::OK, Value::Null) + ); + } + assert_eq!(f.history(&execution).await, before); + f.pending(&execution).await; + success(&f, &token, json!(42)).await; + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; +} +#[tokio::test] +async fn test_should_route_public_retry_with_new_token_and_reject_old_attempt() { + let mut f = CallbackFixture::new(); + let execution=f.start(json!({"Retry":[{"ErrorEquals":["ExternalFailure"],"IntervalSeconds":1,"MaxAttempts":1}]})).await; + let (command, responder) = f.next().await; + let old = token(&command); + finish(command, responder); + assert_eq!( + f.inner + .call( + "SendTaskFailure", + json!({"taskToken":old,"error":"ExternalFailure"}) + ) + .await + .0, + StatusCode::OK + ); + let (command, responder) = f.next().await; + let next = token(&command); + assert_ne!(old, next); + assert_eq!( + command.dispatch().parameters().value()["Payload"]["retry"], + 1 + ); + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":old})) + .await + .1["__type"], + "TaskTimedOut" + ); + success(&f, &next, json!({"retried":true})).await; + finish(command, responder); + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_closed_public_token_after_absolute_or_heartbeat_timeout( + #[case] heartbeat: bool, +) { + let mut f = CallbackFixture::new(); + let extra = if heartbeat { + json!({"HeartbeatSeconds":1}) + } else { + json!({"TimeoutSeconds":1}) + }; + let execution = f.start(extra).await; + let (command, responder) = f.next().await; + let token = token(&command); + finish(command, responder); + let done = f.inner.wait(&execution).await; + assert_eq!( + done["error"], + if heartbeat { + "States.HeartbeatTimeout" + } else { + "States.Timeout" + } + ); + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await + .1["__type"], + "TaskTimedOut" + ); + assert!( + f.history(&execution).await["events"] + .as_array() + .unwrap() + .iter() + .any(|e| e["type"] == "TaskTimedOut") + ); + f.close().await; +} +#[tokio::test] +async fn test_should_revoke_public_token_on_stop_and_join_physical_call() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + let (status, _) = f + .inner + .call("StopExecution", json!({"executionArn":execution})) + .await; + assert_eq!(status, StatusCode::OK); + let mut cancelled = responder.cancellation(); + timeout_at(deadline(), async { + while !*cancelled.borrow_and_update() { + cancelled.changed().await.unwrap(); + } + }) + .await + .unwrap(); + responder.finish(Err(TaskFailure::service_exception())); + drop(command); + assert_eq!(f.inner.wait(&execution).await["status"], "ABORTED"); + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await + .1["__type"], + "TaskTimedOut" + ); + f.close().await; +} +#[tokio::test] +async fn test_should_reject_invalid_public_output_or_credentials_without_consuming_valid_token() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + for (action, input, code) in [ + ( + "SendTaskSuccess", + json!({"taskToken":token,"output":"{"}), + "InvalidOutput", + ), + ( + "SendTaskSuccess", + json!({"taskToken":token,"output":"x".repeat(262_145)}), + "InvalidOutput", + ), + ( + "SendTaskFailure", + json!({"taskToken":token,"error":"x".repeat(257)}), + "ValidationException", + ), + ( + "SendTaskFailure", + json!({"taskToken":token,"cause":"x".repeat(32769)}), + "ValidationException", + ), + ( + "SendTaskHeartbeat", + json!({"taskToken":"malformed"}), + "InvalidToken", + ), + ( + "SendTaskHeartbeat", + json!({"taskToken":"f".repeat(64)}), + "InvalidToken", + ), + ] { + let (status, body) = f.inner.call(action, input).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["__type"], code); + assert!(!body.to_string().contains(&token)); + } + success(&f, &token, json!("still-live")).await; + finish(command, responder); + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; +} +#[tokio::test] +async fn test_should_bind_callback_epoch_and_trusted_http_scope() { + let mut f = CallbackFixture::new(); + let mut other_epoch = CallbackFixture::new(); + let mut foreign = + CallbackFixture::scoped(TenantScope::new("aws", "us-west-2", "111111111111").unwrap()); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + assert_eq!( + other_epoch + .inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await + .1["__type"], + "InvalidToken" + ); + assert_eq!( + foreign + .inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await + .1["__type"], + "InvalidToken" + ); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-west-2".into(), "111111111111".into()).unwrap(), + true, + None, + ) + .unwrap(); + let original = f.inner.service.clone(); + f.inner.service = StepFunctionsHttpService::new(f.inner.handle.clone(), config); + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await + .1["__type"], + "AccessDeniedException" + ); + f.inner.service = original; + success(&f, &token, json!(42)).await; + finish(command, responder); + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; + foreign.close().await; + other_epoch.close().await; +} +#[tokio::test] +async fn test_should_reject_unbound_or_closed_callback_api_and_inconsistent_compiler_composition() { + let mut unbound = Fixture::new(ApiLimits::default()); + for action in ["SendTaskSuccess", "SendTaskFailure", "SendTaskHeartbeat"] { + let mut input = json!({"taskToken":"a".repeat(64)}); + if action == "SendTaskSuccess" { + input["output"] = json!("null"); + } + assert_eq!( + unbound.call(action, input).await.1["__type"], + "UnsupportedOperation" + ); + } + unbound.close().await; + let mut f = CallbackFixture::new(); + assert!(matches!( + StepFunctionsApi::new( + TenantScope::default(), + TaskSupport::Disabled, + f.registry.clone(), + ApiLimits::default() + ), + Err(ApiError::Configuration) + )); + f.callbacks.close(); + assert_eq!( + f.inner + .call("SendTaskHeartbeat", json!({"taskToken":"a".repeat(64)})) + .await + .1["__type"], + "ServiceUnavailable" + ); + f.close().await; + assert!(matches!( + f.registry.submit_callback(CallbackCommand::heartbeat( + CallbackToken::try_from("a".repeat(64)).unwrap() + )), + Err(CallbackError::Unavailable) + )); +} +#[tokio::test] +async fn test_should_submit_callback_independently_of_metadata_admission() { + let mut f = CallbackFixture::new(); + let mut metadata = Vec::new(); + for _ in 0..256 { + match f.registry.submit(RegistryCommand::Machines) { + Ok(receipt) => metadata.push(receipt), + Err(_) => break, + } + } + assert!(!metadata.is_empty()); + assert!(f.registry.submit(RegistryCommand::Machines).is_err()); + let error = f + .registry + .submit_callback(CallbackCommand::heartbeat( + "a".repeat(64).try_into().unwrap(), + )) + .unwrap() + .wait(deadline()) + .await + .unwrap_err(); + assert_eq!( + super::super::error::callback(error).code, + Code::InvalidToken + ); + drop(metadata); + f.close().await; +} + +#[tokio::test] +async fn test_should_return_public_callback_throttling_without_consuming_the_live_token() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + let mut acknowledgments = Vec::new(); + for _ in 0..2048 { + match f.registry.submit_callback(CallbackCommand::heartbeat( + "a".repeat(64).try_into().unwrap(), + )) { + Ok(ack) => acknowledgments.push(ack), + Err(CallbackError::Capacity) => break, + Err(error) => panic!("unexpected admission error: {error:?}"), + } + } + assert!(!acknowledgments.is_empty()); + let (status, error) = f + .inner + .call("SendTaskSuccess", json!({"taskToken":token,"output":"42"})) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error["__type"], "ThrottlingException"); + drop(acknowledgments); + timeout_at(deadline(), async { + loop { + let (status, body) = f + .inner + .call("SendTaskHeartbeat", json!({"taskToken":token})) + .await; + if status == StatusCode::OK { + break; + } + assert_eq!(body["__type"], "ThrottlingException"); + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + success(&f, &token, json!(42)).await; + finish(command, responder); + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; +} + +async fn caller_context() -> RequestContext { + let (tx, mut rx) = mpsc::channel(1); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into()).unwrap(), + true, + None, + ) + .unwrap() + .with_budgets( + rustack_core::http::BodyBudget::control(), + Duration::from_secs(1), + ) + .unwrap(); + let service = StepFunctionsHttpService::new(Arc::new(CaptureContext(tx)), config); + service + .handle( + HttpRequest::builder() + .method("POST") + .uri("/") + .header("content-type", CONTENT_TYPE) + .header("x-amz-target", "AWSStepFunctions.ListStateMachines") + .body(Full::new(Bytes::from_static(b"{}"))) + .unwrap(), + ) + .await; + rx.recv().await.unwrap() +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_keep_accepted_public_callback_after_abandoned_or_late_ready_observer( + #[case] late: bool, +) { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + let context = caller_context().await; + let caller_deadline = context.deadline(); + let request = Request::decode( + Op::SendTaskSuccess, + &serde_json::to_vec(&json!({"taskToken":token,"output":"42"})).unwrap(), + ) + .unwrap(); + let mut future = f.inner.handle.handle(context, request); + let flag = Arc::new(WakeFlag::default()); + let waker = std::task::Waker::from(flag.clone()); + assert!( + future + .as_mut() + .poll(&mut std::task::Context::from_waker(&waker)) + .is_pending() + ); + if late { + timeout_at(caller_deadline, async { + while !flag.0.load(std::sync::atomic::Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + sleep_until(caller_deadline).await; + assert_eq!( + future.await.unwrap_err().code, + Code::RequestTimeoutException + ); + } else { + drop(future); + } + // API intake is independently joined before the actual Lambda report. + f.inner.api.shutdown(deadline()).await.unwrap(); + finish(command, responder); + let mut receipt = f + .registry + .submit(RegistryCommand::Execution(execution.parse().unwrap())) + .unwrap(); + let RegistryResult::Execution(snapshot) = receipt.wait(deadline()).await.unwrap() else { + panic!("execution"); + }; + let view = snapshot.handle().wait(deadline()).await.unwrap(); + assert!( + matches!(view.outcome().unwrap().as_ref(),ExecutionOutcome::Succeeded(output) if output.value()==&json!(42)) + ); + f.close().await; +} +#[tokio::test] +async fn test_should_reject_expired_public_request_before_consuming_live_token() { + let mut f = CallbackFixture::new(); + let execution = f.start(json!({})).await; + let (command, responder) = f.next().await; + let token = token(&command); + let context = caller_context().await; + sleep_until(context.deadline()).await; + let request = Request::decode( + Op::SendTaskFailure, + &serde_json::to_vec(&json!({"taskToken":token,"error":"must-not-run"})).unwrap(), + ) + .unwrap(); + assert_eq!( + f.inner + .handle + .handle(context, request) + .await + .unwrap_err() + .code, + Code::RequestTimeoutException + ); + success(&f, &token, json!(true)).await; + finish(command, responder); + assert_eq!(f.inner.wait(&execution).await["status"], "SUCCEEDED"); + f.close().await; +} +#[rstest::rstest] +#[case(CallbackError::InvalidToken, Code::InvalidToken, 400)] +#[case(CallbackError::TaskTimedOut, Code::TaskTimedOut, 400)] +#[case(CallbackError::Data(AslError::DataLimit), Code::InvalidOutput, 400)] +#[case(CallbackError::Validation, Code::ValidationException, 400)] +#[case(CallbackError::Capacity, Code::ThrottlingException, 400)] +#[case(CallbackError::Deadline, Code::RequestTimeoutException, 408)] +#[case(CallbackError::Unavailable, Code::ServiceUnavailable, 503)] +#[case(CallbackError::OwnerLost, Code::InternalFailure, 500)] +#[case(CallbackError::Progress, Code::InternalFailure, 500)] +#[case(CallbackError::AttemptExists, Code::InternalFailure, 500)] +#[test] +fn test_should_map_cached_callback_sources_to_static_action_errors( + #[case] cause: CallbackError, + #[case] code: Code, + #[case] status: u16, +) { + let error = super::super::error::callback(CallbackError::Cached(Arc::new(cause))); + assert_eq!(error.code, code); + assert_eq!(error.status_code.as_u16(), status); + assert!( + error + .source + .as_ref() + .unwrap() + .downcast_ref::() + .is_some() + ); +} diff --git a/apps/rustack/src/stepfunctions_api/worker.rs b/apps/rustack/src/stepfunctions_api/worker.rs new file mode 100644 index 00000000..f94e19fb --- /dev/null +++ b/apps/rustack/src/stepfunctions_api/worker.rs @@ -0,0 +1,184 @@ +//! Retained CPU preparation, registry commands and immutable response projection. + +use std::{future::Future, sync::Arc}; + +use rustack_stepfunctions_core::{AslCompiler, ExecutionError, RegistryCommand, RegistryResult}; +use rustack_stepfunctions_http::{Request, Response}; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode as Code, StepFunctionsOperation as Op, output, +}; +use tokio::{ + sync::oneshot, + task::JoinSet, + time::{Instant, timeout_at}, +}; + +use super::{ + Context, + error::{callback, check, error, registry, source}, + pages, + prepare::{self, Prepared}, + render, +}; + +pub(super) async fn run( + context: Arc, + request: Request, + deadline: Instant, +) -> Result { + let scope = context.scope.clone(); + let tasks = context.tasks; + let states = context.registry.state_support(); + let callbacks = context.registry.callback_support(); + let prepared = cpu(&context, deadline, move || { + prepare::prepare( + request, + &AslCompiler::new(scope.clone(), tasks) + .with_state_support(states) + .with_child_support(true) + .with_callback_support(callbacks), + &scope, + callbacks, + ) + }) + .await?; + check(deadline)?; + match prepared { + Prepared::Ready(response) => Ok(response), + Prepared::Callback(command, op) => { + let mut acknowledgment = context + .registry + .submit_callback(command) + .map_err(callback)?; + acknowledgment.wait(deadline).await.map_err(callback)?; + match op { + Op::SendTaskSuccess => Ok(output::SendTaskSuccessOutput::default().into()), + Op::SendTaskFailure => Ok(output::SendTaskFailureOutput::default().into()), + Op::SendTaskHeartbeat => Ok(output::SendTaskHeartbeatOutput::default().into()), + _ => Err(error(Code::InternalFailure)), + } + } + Prepared::Command(command, op) => { + let mut result = submit_registry(&context, command, op, deadline).await?; + if op == Op::StopExecution { + let RegistryResult::Execution(snapshot) = result else { + return Err(error(Code::InternalFailure)); + }; + result = RegistryResult::Execution(snapshot.wait(deadline).await.map_err(|e| { + source( + if matches!(e, ExecutionError::Deadline) { + Code::RequestTimeoutException + } else { + Code::InternalFailure + }, + e, + ) + })?); + } + cpu(&context, deadline, move || render::render(op, result)).await + } + Prepared::Page(binding, token, initial) => { + let cursor = if let Some(token) = token { + let (reply, receiver) = oneshot::channel(); + page_command( + &context, + deadline, + pages::Request::Read(token, binding, reply), + receiver, + ) + .await? + } else { + let result = submit_registry(&context, initial, binding.op, deadline).await?; + let captured = binding.clone(); + let data = cpu(&context, deadline, move || { + pages::Data::capture(&captured, result) + }) + .await?; + let (reply, receiver) = oneshot::channel(); + let snapshot = page_command( + &context, + deadline, + pages::Request::Reserve(data, reply), + receiver, + ) + .await?; + pages::first(binding, snapshot) + }; + let mut page = cpu(&context, deadline, move || pages::page(cursor)).await?; + if let Some(next) = page.next { + let (reply, receiver) = oneshot::channel(); + page_command( + &context, + deadline, + pages::Request::Save(page.token.clone(), next, reply), + receiver, + ) + .await?; + pages::attach(&mut page.response, page.token)?; + } + Ok(page.response) + } + } +} +async fn submit_registry( + context: &Context, + command: RegistryCommand, + op: Op, + deadline: Instant, +) -> Result { + check(deadline)?; + let mut receipt = context + .registry + .submit(command) + .map_err(|e| registry(e, op))?; + receipt.wait(deadline).await.map_err(|e| registry(e, op)) +} +async fn page_command( + context: &Context, + deadline: Instant, + request: pages::Request, + receiver: oneshot::Receiver>, +) -> Result { + let operation = async { + context + .pages + .send(pages::Command { deadline, request }) + .await + .map_err(|_| error(Code::ServiceUnavailable))?; + receiver + .await + .map_err(|_| error(Code::ServiceUnavailable))? + }; + observe(deadline, operation).await +} +async fn observe( + deadline: Instant, + future: impl Future>, +) -> Result { + timeout_at(deadline, future) + .await + .map_err(|_| error(Code::RequestTimeoutException))? +} +pub(super) async fn cpu( + context: &Context, + deadline: Instant, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { + check(deadline)?; + let permit = timeout_at(deadline, context.cpu.clone().acquire_owned()) + .await + .map_err(|_| error(Code::RequestTimeoutException))? + .map_err(|_| error(Code::ServiceUnavailable))?; + check(deadline)?; + let mut jobs = JoinSet::new(); + jobs.spawn_blocking(move || (work(), permit)); + match jobs.join_next().await { + Some(Ok((result, permit))) => { + drop(permit); + check(deadline)?; + result + } + Some(Err(e)) => Err(source(Code::InternalFailure, e)), + None => Err(error(Code::InternalFailure)), + } +} diff --git a/apps/rustack/src/stepfunctions_bridge.rs b/apps/rustack/src/stepfunctions_bridge.rs new file mode 100644 index 00000000..007c7ece --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge.rs @@ -0,0 +1,443 @@ +//! Retained Step Functions → actual Lambda provider composition. +//! +//! The port queues synchronously. Its independent manager retains each start, +//! response and cleanup receipt; dropping an ASL observer never drops the real +//! invocation future. No SDK, HTTP loopback or disabled-executor fallback is used. + +use std::{ + collections::HashMap, + fmt, + mem::replace, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use rustack_lambda_core::{ + executor::ExecutorBackend, invocation::InvocationHandle, provider::RustackLambda, + resolver::function_arn, +}; +use rustack_stepfunctions_core::{ + LambdaTaskCommand, LambdaTaskPort, QuarantinedTask, TaskCall, TaskFailure, TaskResponder, + TenantScope, +}; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, watch}, + task::{Id, JoinError, JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; + +mod error; +mod mapping; +#[cfg(test)] +mod tests; +mod worker; + +pub use error::BridgeError; + +/// Validated accepted-work and independent cleanup wait bounds. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct BridgeLimits { + accepted: usize, + cleanup: Duration, +} +impl BridgeLimits { + /// Validate1..=1,024 accepted calls and1..=60s cleanup observation. + /// + /// # Errors + /// Rejects zero/excessive capacities or cleanup durations. + /// + /// ``` + /// let limits=rustack_cli::stepfunctions_bridge::BridgeLimits::new(128,std::time::Duration::from_secs(10))?; + /// # Ok::<(),rustack_cli::stepfunctions_bridge::BridgeError>(()) + /// ``` + pub fn new(accepted: usize, cleanup: Duration) -> Result { + if !(1..=1024).contains(&accepted) + || !(Duration::from_secs(1)..=Duration::from_secs(60)).contains(&cleanup) + { + return Err(BridgeError::Configuration); + } + Ok(Self { accepted, cleanup }) + } +} +impl Default for BridgeLimits { + fn default() -> Self { + Self { + accepted: 128, + cleanup: Duration::from_secs(10), + } + } +} +struct Context { + provider: Arc, + scope: TenantScope, + limits: BridgeLimits, +} +impl fmt::Debug for Context { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LambdaBridgeContext") + .field("limits", &self.limits) + .finish_non_exhaustive() + } +} +#[derive(Debug)] +struct Shared { + context: Context, + accepting: AtomicBool, + unhealthy: AtomicBool, + capacity: Arc, + diagnostic: watch::Sender>>, +} +#[derive(Debug)] +struct Submission { + command: LambdaTaskCommand, + responder: TaskResponder, + permit: OwnedSemaphorePermit, +} +#[derive(Debug)] +struct Retained { + _permit: OwnedSemaphorePermit, + _resource: Option, +} +#[derive(Debug)] +struct Uncertain { + _handle: InvocationHandle, + _receipt: QuarantinedTask, +} +#[derive(Debug)] +struct WorkerExit { + uncertain: Option, + diagnostic: Option>, +} +#[derive(Debug)] +struct ManagerState { + queue: mpsc::Receiver, + stop: watch::Receiver, + jobs: JoinSet, + active: HashMap, + quarantine: Vec, +} +#[derive(Debug)] +struct ManagerOwner { + state: ManagerState, + shared: Arc, +} + +/// Synchronous bounded adapter, installed only for an explicitly selected backend. +#[derive(Debug)] +#[non_exhaustive] +pub struct StepFunctionsLambdaBridge { + queue: mpsc::Sender, + shared: Arc, +} +/// Independent join controller; a dropped waiter retains its exact manager handle. +#[derive(Debug)] +#[non_exhaustive] +pub struct LambdaBridgeSupervisor { + stop: watch::Sender, + manager: Option>>>, + shared: Arc, + joined: Option>>, +} +impl StepFunctionsLambdaBridge { + /// Read actual port readiness without acquiring task admission. + /// + /// ``` + /// # fn inspect(port:&rustack_cli::stepfunctions_bridge::StepFunctionsLambdaBridge){let ready=port.is_ready();} + /// ``` + #[must_use] + pub fn is_ready(&self) -> bool { + self.shared.accepting.load(Ordering::Acquire) + && !self.shared.unhealthy.load(Ordering::Acquire) + } + /// Start one retained manager before exposing the task port. + /// + /// The caller injects an actual provider and its trusted tenant scope. This + /// exports a composition adapter; it does not register a gateway service. + /// + /// # Errors + /// Returns Unavailable outside an active Tokio runtime. + /// + /// ```no_run + /// # fn build(provider:std::sync::Arc) -> Result<(),rustack_cli::stepfunctions_bridge::BridgeError> { + /// use rustack_cli::stepfunctions_bridge::{StepFunctionsLambdaBridge,BridgeLimits}; + /// let (port,owner)=StepFunctionsLambdaBridge::new(provider,rustack_stepfunctions_core::TenantScope::default(),BridgeLimits::default())?; + /// # Ok(()) } + /// ``` + pub fn new( + provider: Arc, + scope: TenantScope, + limits: BridgeLimits, + ) -> Result<(Arc, LambdaBridgeSupervisor), BridgeError> { + let config = provider.config(); + let expected = format!( + "arn:{}:lambda:{}:{}:function:", + scope.partition(), + scope.region(), + scope.account() + ); + if config.executor == ExecutorBackend::Disabled + || (config.executor == ExecutorBackend::Docker + && provider.executor().image_available().is_err()) + || !function_arn( + &config.default_region, + &config.account_id, + "BridgeScopeProbe", + ) + .starts_with(&expected) + { + return Err(BridgeError::Configuration); + } + let runtime = Handle::try_current().map_err(|_| BridgeError::Unavailable)?; + let (queue, receiver) = mpsc::channel(limits.accepted); + let (stop, cancelled) = watch::channel(false); + let (diagnostic, _) = watch::channel(None); + let shared = Arc::new(Shared { + context: Context { + provider, + scope, + limits, + }, + accepting: AtomicBool::new(true), + unhealthy: AtomicBool::new(false), + capacity: Arc::new(Semaphore::new(limits.accepted)), + diagnostic, + }); + let owner = ManagerOwner { + state: ManagerState { + queue: receiver, + stop: cancelled, + jobs: JoinSet::new(), + active: HashMap::new(), + quarantine: Vec::new(), + }, + shared: shared.clone(), + }; + let manager = runtime.spawn(owner.run()); + Ok(( + Arc::new(Self { + queue, + shared: shared.clone(), + }), + LambdaBridgeSupervisor { + stop, + manager: Some(manager), + shared, + joined: None, + }, + )) + } +} +impl LambdaTaskPort for StepFunctionsLambdaBridge { + fn start(&self, command: LambdaTaskCommand) -> Result { + let scope = command.machine().scope(); + let configured = &self.shared.context.scope; + if !self.shared.accepting.load(Ordering::Acquire) + || self.queue.is_closed() + || scope.partition() != configured.partition() + || scope.region() != configured.region() + || scope.account() != configured.account() + || Instant::now() >= Instant::from_std(command.dispatch().deadline()) + { + return Err(TaskFailure::service_exception()); + } + let permit = self + .shared + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| TaskFailure::throttled())?; + let (call, responder) = TaskCall::channel(); + self.queue + .try_send(Submission { + command, + responder, + permit, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => TaskFailure::throttled(), + mpsc::error::TrySendError::Closed(_) => TaskFailure::service_exception(), + })?; + Ok(call) + } +} +impl LambdaBridgeSupervisor { + /// Stop accepting new tasks and request cancellation of all owned workers. + /// + /// ``` + /// # fn close(owner:&rustack_cli::stepfunctions_bridge::LambdaBridgeSupervisor) {owner.close();} + /// ``` + pub fn close(&self) { + self.shared.accepting.store(false, Ordering::Release); + self.stop.send_replace(true); + } + /// Join the same manager through an absolute deadline, retaining it on expiry. + /// + /// Call before quiescing the underlying Lambda provider. + /// + /// # Errors + /// Returns Deadline, a supervised worker source, or retained Quarantined status. + /// + /// ```no_run + /// # async fn drain(owner:&mut rustack_cli::stepfunctions_bridge::LambdaBridgeSupervisor,deadline:tokio::time::Instant) { + /// let result=owner.shutdown(deadline).await; + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), Arc> { + self.close(); + if let Some(result) = &self.joined { + return result.clone(); + } + let Some(manager) = &mut self.manager else { + return Err(Arc::new(BridgeError::Unavailable)); + }; + let result = timeout_at(deadline, manager) + .await + .map_err(|_| Arc::new(BridgeError::Deadline))?; + self.manager = None; + let result = result.unwrap_or_else(|error| Err(Arc::new(BridgeError::Worker(error)))); + if let Err(error) = &result { + self.shared.diagnostic.send_replace(Some(error.clone())); + } + self.joined = Some(result.clone()); + result + } + /// Read the most recent redacted internal source; never a public ASL cause. + /// + /// ``` + /// # fn inspect(owner:&rustack_cli::stepfunctions_bridge::LambdaBridgeSupervisor) {let source=owner.diagnostic();} + /// ``` + #[must_use] + pub fn diagnostic(&self) -> Option> { + self.shared.diagnostic.borrow().clone() + } + /// Read accepted/uncertain calls still holding bridge admission capacity. + /// + /// ``` + /// # fn inspect(owner:&rustack_cli::stepfunctions_bridge::LambdaBridgeSupervisor) {let active=owner.owned_calls();} + /// ``` + #[must_use] + pub fn owned_calls(&self) -> usize { + self.shared + .context + .limits + .accepted + .saturating_sub(self.shared.capacity.available_permits()) + } +} +impl Drop for LambdaBridgeSupervisor { + fn drop(&mut self) { + self.close(); + // Intentional detached manager: its own Drop guard retains unresolved + // jobs/slots through public-controller loss and runtime cancellation. + if let Some(manager) = self.manager.take() { + drop(manager); + } + } +} +impl ManagerOwner { + async fn run(mut self) -> Result<(), Arc> { + loop { + if signal(&self.state.stop) { + self.shared.accepting.store(false, Ordering::Release); + self.state.queue.close(); + self.reject_queued(); + } + if self.state.queue.is_closed() + && self.state.queue.is_empty() + && self.state.jobs.is_empty() + { + break; + } + tokio::select! { + biased; + _=self.state.stop.changed(),if !signal(&self.state.stop)=>{}, + result=self.state.jobs.join_next_with_id(),if !self.state.jobs.is_empty()=>if let Some(result)=result {self.finish(result);}, + submission=self.state.queue.recv(),if !self.state.queue.is_closed() || !self.state.queue.is_empty()=>if let Some(submission)=submission { + let Submission {command,responder,permit}=submission; + let shared=self.shared.clone();let stop=self.state.stop.clone(); + let task=self.state.jobs.spawn(worker::invoke(shared,command,responder,stop));self.state.active.insert(task.id(),permit); + }, + } + } + self.shared.accepting.store(false, Ordering::Release); + if self.state.quarantine.is_empty() { + Ok(()) + } else { + Err(Arc::new(BridgeError::Quarantined)) + } + } + fn reject_queued(&mut self) { + while let Ok(submission) = self.state.queue.try_recv() { + submission + .responder + .finish(Err(TaskFailure::service_exception())); + } + } + fn finish(&mut self, result: Result<(Id, WorkerExit), JoinError>) { + match result { + Ok((id, exit)) => { + if let Some(diagnostic) = exit.diagnostic { + self.shared.diagnostic.send_replace(Some(diagnostic)); + } + if let Some(permit) = self.state.active.remove(&id) { + if let Some(resource) = exit.uncertain { + self.shared.unhealthy.store(true, Ordering::Release); + self.state.quarantine.push(Retained { + _permit: permit, + _resource: Some(resource), + }); + } + } + } + Err(error) => { + self.shared.unhealthy.store(true, Ordering::Release); + let id = error.id(); + self.shared + .diagnostic + .send_replace(Some(Arc::new(BridgeError::Worker(error)))); + if let Some(permit) = self.state.active.remove(&id) { + self.state.quarantine.push(Retained { + _permit: permit, + _resource: None, + }); + } + } + } + } +} +impl Drop for ManagerOwner { + fn drop(&mut self) { + self.shared.accepting.store(false, Ordering::Release); + self.state.queue.close(); + self.reject_queued(); + if self.state.jobs.is_empty() + && self.state.active.is_empty() + && self.state.quarantine.is_empty() + { + return; + } + // Retain the bounded ownership graph if normal shutdown cannot prove clean. + // This guard also covers an unpolled manager and runtime teardown. + let state = ManagerState { + queue: mpsc::channel(1).1, + stop: watch::channel(true).1, + jobs: JoinSet::new(), + active: HashMap::new(), + quarantine: Vec::new(), + }; + let retained = replace(&mut self.state, state); + tracing::error!( + calls = retained.active.len() + retained.quarantine.len(), + "Lambda task bridge retains uncertain cleanup" + ); + let _retained = Box::leak(Box::new((self.shared.clone(), retained))); + } +} +fn signal(receiver: &watch::Receiver) -> bool { + *receiver.borrow() || receiver.has_changed().is_err() +} diff --git a/apps/rustack/src/stepfunctions_bridge/error.rs b/apps/rustack/src/stepfunctions_bridge/error.rs new file mode 100644 index 00000000..4c34b9f0 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/error.rs @@ -0,0 +1,53 @@ +//! Redacted internal bridge errors; sources never become ASL public causes. + +use std::{fmt, sync::Arc}; + +use rustack_lambda_core::{error::LambdaServiceError, executor::ExecutorError}; +use rustack_stepfunctions_core::{AslError, CallbackError}; +use thiserror::Error; +use tokio::task::JoinError; + +/// Bridge lifecycle/internal errors with bounded static public descriptions. +#[derive(Error)] +#[non_exhaustive] +pub enum BridgeError { + /// Invalid operator bridge limits. + #[error("invalid Lambda task bridge configuration")] + Configuration, + /// No Tokio runtime is active for owned admission. + #[error("Lambda task bridge runtime unavailable")] + Unavailable, + /// Evaluated optimized parameters violate the supported schema. + #[error("invalid Lambda task parameters")] + Parameters, + /// An observer deadline expired; the manager remains owned. + #[error("Lambda task bridge observation deadline exceeded")] + Deadline, + /// Cleanup ownership remains uncertain and charged. + #[error("Lambda task bridge cleanup quarantined")] + Quarantined, + /// Provider failure, retained without copying its host/business text. + #[error("Lambda task provider failed")] + Service(#[source] Arc), + /// Execution/cleanup failure from an actual backend. + #[error("Lambda task execution failed")] + Executor(#[source] Arc), + /// Bounded JSON/invariant failure. + #[error("Lambda task data conversion failed")] + Evaluation( + #[from] + #[source] + AslError, + ), + /// Invalid or closed real callback invocation reporting control. + #[error("Lambda callback invocation report failed")] + Callback(#[source] Arc), + /// A supervised worker panicked or was unexpectedly cancelled. + #[error("Lambda task bridge worker failed")] + Worker(#[source] JoinError), +} +impl fmt::Debug for BridgeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} diff --git a/apps/rustack/src/stepfunctions_bridge/mapping.rs b/apps/rustack/src/stepfunctions_bridge/mapping.rs new file mode 100644 index 00000000..f113e11d --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/mapping.rs @@ -0,0 +1,471 @@ +//! Validated per-attempt plans and bounded direct/optimized result mapping. + +use std::{ + fmt, + io::{Error as IoError, Result as IoResult, Write}, + str, + sync::Arc, +}; + +use rustack_core::json::JsonError; +use rustack_lambda_core::{ + error::LambdaServiceError, + executor::{ExecutorError, InvokeResponse}, +}; +use rustack_stepfunctions_core::{ + AslError, CallbackError, CallbackInvocationMetadata, Integration, LambdaTaskCommand, + MAX_PAYLOAD_BYTES, Payload, StateView, TaskFailure, TenantScope, UuidSource, +}; +use serde_json::{Value, json}; +use tokio::time::Instant; + +use super::BridgeError; + +const CAUSE_LIMIT: usize = 32_768; +const TRUNCATION_MARKER: &str = " [truncated]"; + +struct CauseWriter { + prefix: Vec, + truncated: bool, +} +impl fmt::Debug for CauseWriter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LambdaCauseWriter()") + } +} +impl Write for CauseWriter { + fn write(&mut self, bytes: &[u8]) -> IoResult { + let count = CAUSE_LIMIT + .saturating_sub(self.prefix.len()) + .min(bytes.len()); + self.prefix.extend_from_slice( + bytes + .get(..count) + .ok_or_else(|| IoError::other("invalid Cause prefix"))?, + ); + self.truncated |= count < bytes.len(); + Ok(bytes.len()) + } + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +pub(super) struct CallPlan { + pub(super) function: String, + pub(super) qualifier: Option, + pub(super) payload: Payload, + pub(super) mode: ResponseMode, + pub(super) deadline: Instant, +} +#[derive(Debug, Clone, Copy)] +pub(super) enum ResponseMode { + Direct, + Optimized, + ObservedCallback, +} +impl fmt::Debug for CallPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LambdaCallPlan()") + } +} +#[derive(Debug)] +pub(super) struct Mapped { + pub(super) result: Result, + pub(super) diagnostic: Option, +} +impl CallPlan { + pub(super) fn compile(command: &LambdaTaskCommand) -> Result { + let call = command.dispatch(); + let StateView::Task { resource, .. } = command.machine().state(call.state())? else { + return Err(BridgeError::Parameters); + }; + let (function, qualifier, payload, optimized) = match resource { + Integration::DirectLambda(arn) => { + (arn.as_str().into(), None, call.parameters().clone(), false) + } + Integration::LambdaInvoke | Integration::LambdaCallback(_) => { + if matches!(resource, Integration::LambdaCallback(_)) + && !command.is_observed_callback() + { + return Err(BridgeError::Parameters); + } + let object = call + .parameters() + .value() + .as_object() + .ok_or(BridgeError::Parameters)?; + if object.keys().any(|key| { + !matches!( + key.as_str(), + "FunctionName" | "Payload" | "Qualifier" | "InvocationType" + ) + }) { + return Err(BridgeError::Parameters); + } + let function = object + .get("FunctionName") + .and_then(Value::as_str) + .filter(|v| !v.is_empty() && v.len() <= 256) + .ok_or(BridgeError::Parameters)? + .to_owned(); + validate_reference(&function, command.machine().scope())?; + let qualifier = object + .get("Qualifier") + .map(|v| { + v.as_str() + .filter(|q| !q.is_empty() && q.len() <= 128) + .map(str::to_owned) + .ok_or(BridgeError::Parameters) + }) + .transpose()?; + if object + .get("InvocationType") + .is_some_and(|v| v.as_str() != Some("RequestResponse")) + { + return Err(BridgeError::Parameters); + } + let payload = match object.get("Payload") { + None | Some(Value::Null) => Payload::parse(b"{}")?, + Some(Value::String(text)) => Payload::parse(text.as_bytes())?, + Some(value) => Payload::from_value(value)?, + }; + (function, qualifier, payload, true) + } + _ => return Err(BridgeError::Parameters), + }; + Ok(Self { + function, + qualifier, + payload, + mode: if command.is_observed_callback() { + if !optimized { + return Err(BridgeError::Parameters); + } + ResponseMode::ObservedCallback + } else if optimized { + ResponseMode::Optimized + } else { + ResponseMode::Direct + }, + deadline: Instant::from_std(call.deadline()), + }) + } +} +pub(super) struct ServiceTarget { + function: String, + qualifier: Option, + scope: TenantScope, +} +impl fmt::Debug for ServiceTarget { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LambdaServiceTarget()") + } +} +impl ServiceTarget { + pub(super) fn new(plan: &CallPlan, scope: &TenantScope) -> Self { + Self { + function: plan.function.clone(), + qualifier: plan.qualifier.clone(), + scope: scope.clone(), + } + } + fn qualified_arn(&self) -> Result { + validate_reference(&self.function, &self.scope)?; + let prefix = format!( + "arn:{}:lambda:{}:{}:function:", + self.scope.partition(), + self.scope.region(), + self.scope.account() + ); + let reference = self + .function + .strip_prefix(&prefix) + .unwrap_or(&self.function); + let mut parts = reference.split(':'); + let name = parts.next().ok_or(BridgeError::Parameters)?; + let embedded = parts.next(); + if parts.next().is_some() + || embedded + .zip(self.qualifier.as_deref()) + .is_some_and(|(left, right)| left != right) + { + return Err(BridgeError::Parameters); + } + let qualifier = embedded.or(self.qualifier.as_deref()).unwrap_or("$LATEST"); + if qualifier.is_empty() + || qualifier.len() > 128 + || !qualifier + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'$')) + { + return Err(BridgeError::Parameters); + } + let arn = format!("{prefix}{name}:{qualifier}"); + if arn.len() > 512 { + return Err(BridgeError::Parameters); + } + Ok(arn) + } +} + +fn validate_reference(reference: &str, scope: &TenantScope) -> Result<(), BridgeError> { + let valid_name = |name: &str| { + !name.is_empty() + && name.len() <= 64 + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-')) + }; + if reference.starts_with("arn:") { + let prefix = format!( + "arn:{}:lambda:{}:{}:function:", + scope.partition(), + scope.region(), + scope.account() + ); + let suffix = reference + .strip_prefix(&prefix) + .ok_or(BridgeError::Parameters)?; + let mut parts = suffix.split(':'); + let name = parts.next().ok_or(BridgeError::Parameters)?; + if !valid_name(name) + || parts.next().is_some_and(|q| { + q.is_empty() + || q.len() > 128 + || !q + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'$')) + }) + || parts.next().is_some() + { + return Err(BridgeError::Parameters); + } + } else if !valid_name(reference) { + return Err(BridgeError::Parameters); + } + Ok(()) +} +fn failure(name: &str, cause: &str) -> Result { + Ok(Mapped { + result: Err(TaskFailure::new(name, Some(cause))?), + diagnostic: None, + }) +} +pub(super) fn parameters(error: BridgeError) -> Result { + let mut mapped = failure("States.Runtime", "Invalid evaluated Lambda task parameters")?; + mapped.diagnostic = Some(error); + Ok(mapped) +} +pub(super) fn response( + response: &InvokeResponse, + optimized: bool, + request_id: &str, +) -> Result { + if response.status != 200 { + return failure( + "Lambda.ServiceException", + "Unexpected Lambda response status", + ); + } + if response.payload.len() > MAX_PAYLOAD_BYTES { + return failure( + "States.DataLimitExceeded", + "Lambda result exceeds the ASL payload limit", + ); + } + let payload = match Payload::parse(&response.payload) { + Ok(payload) => payload, + Err(error) => { + let name = if response.function_error.is_some() { + "Lambda.Unknown" + } else { + "States.Runtime" + }; + let mut mapped = failure(name, "Lambda returned invalid JSON")?; + mapped.diagnostic = Some(error.into()); + return Ok(mapped); + } + }; + if response.function_error.is_some() { + let error = payload + .value() + .get("errorType") + .and_then(Value::as_str) + .filter(|name| { + !name.is_empty() + && name.len() <= 256 + && !name.starts_with("States.") + && !name.chars().any(char::is_control) + }) + .unwrap_or("Lambda.Unknown"); + return failure(error, &bounded_cause(payload.value())?); + } + if !optimized { + return Ok(Mapped { + result: Ok(payload), + diagnostic: None, + }); + } + if response.executed_version.is_empty() + || response.executed_version.len() > 128 + || !response + .executed_version + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'$')) + { + return failure("Lambda.ServiceException", "Invalid Lambda version metadata"); + } + let envelope = json!({"Payload":payload.value(),"StatusCode":200,"ExecutedVersion":response.executed_version, + "SdkHttpMetadata":{"HttpStatusCode":200},"SdkResponseMetadata":{"RequestId":request_id}}); + match Payload::from_value(&envelope) { + Ok(payload) => Ok(Mapped { + result: Ok(payload), + diagnostic: None, + }), + Err(error) => { + let name = match &error { + AslError::DataLimit + | AslError::Json(JsonError::Bytes | JsonError::String | JsonError::Key) => { + "States.DataLimitExceeded" + } + AslError::Encoding(_) | AslError::Allocation(_) | AslError::Entropy(_) => { + "Lambda.ServiceException" + } + _ => "States.Runtime", + }; + let mut mapped = failure( + name, + "Lambda result envelope exceeds the ASL payload profile", + )?; + mapped.diagnostic = Some(error.into()); + Ok(mapped) + } + } +} +pub(super) fn observed_response( + invocation: &InvokeResponse, + metadata: Result, +) -> Result<(Mapped, Option), BridgeError> { + // FunctionError keeps the ordinary bounded failure path. Successful callback + // invocations never parse or retain the synchronous function business body. + if invocation.status != 200 || invocation.function_error.is_some() { + return response(invocation, false, "").map(|mapped| (mapped, None)); + } + let metadata = match metadata { + Ok(metadata) => metadata, + Err(error) => { + let mut mapped = failure( + "Lambda.ServiceException", + "Invalid Lambda invocation metadata", + )?; + mapped.diagnostic = Some(BridgeError::Callback(Arc::new(error))); + return Ok((mapped, None)); + } + }; + let output = metadata.payload()?; + Ok(( + Mapped { + result: Ok(output), + diagnostic: None, + }, + Some(metadata), + )) +} +fn bounded_cause(value: &Value) -> Result { + let mut writer = CauseWriter { + prefix: Vec::with_capacity(CAUSE_LIMIT), + truncated: false, + }; + serde_json::to_writer(&mut writer, value).map_err(AslError::Encoding)?; + if writer.truncated { + writer + .prefix + .truncate(CAUSE_LIMIT.saturating_sub(TRUNCATION_MARKER.len())); + } + let prefix = match str::from_utf8(&writer.prefix) { + Ok(prefix) => prefix, + Err(error) if writer.truncated && error.error_len().is_none() => str::from_utf8( + writer + .prefix + .get(..error.valid_up_to()) + .ok_or(BridgeError::Parameters)?, + ) + .map_err(|_| BridgeError::Parameters)?, + Err(_) => return Err(BridgeError::Parameters), + }; + let mut cause = prefix.to_owned(); + if writer.truncated { + cause.push_str(TRUNCATION_MARKER); + } + Ok(cause) +} +pub(super) fn service( + error: LambdaServiceError, + target: &ServiceTarget, + source: &mut S, +) -> Result { + let name = match &error { + #[cfg(unix)] + LambdaServiceError::ImageRestore { .. } | LambdaServiceError::ImageRestoreWorker { .. } => { + "Lambda.ServiceException" + } + LambdaServiceError::TooManyRequests => "Lambda.TooManyRequestsException", + LambdaServiceError::FunctionNotFound { .. } + | LambdaServiceError::VersionNotFound { .. } + | LambdaServiceError::AliasNotFound { .. } + | LambdaServiceError::EventSourceMappingNotFound { .. } + | LambdaServiceError::PolicyNotFound { .. } + | LambdaServiceError::EventInvokeConfigNotFound { .. } => { + "Lambda.ResourceNotFoundException" + } + LambdaServiceError::ResourceConflict { .. } => "Lambda.ResourceConflictException", + LambdaServiceError::PreconditionFailed => "Lambda.PreconditionFailedException", + LambdaServiceError::InvalidParameter { .. } + | LambdaServiceError::InvalidZipFile { .. } + | LambdaServiceError::InvalidArn { .. } => "Lambda.InvalidParameterValueException", + LambdaServiceError::ResourceNotReady { .. } => "Lambda.ResourceNotReadyException", + LambdaServiceError::RequestTooLarge { .. } => "Lambda.RequestTooLargeException", + LambdaServiceError::ArtifactIo { .. } + | LambdaServiceError::DockerNotAvailable + | LambdaServiceError::Internal { .. } => "Lambda.ServiceException", + }; + let mut mapped = if matches!( + &error, + LambdaServiceError::FunctionNotFound { .. } + | LambdaServiceError::VersionNotFound { .. } + | LambdaServiceError::AliasNotFound { .. } + ) { + let arn = target.qualified_arn()?; + let id = source.next_uuid()?; + // The RFC 4122/9562 variant has leading bits 10 in the ninth byte. + if id.get_version_num() != 4 || id.as_bytes().get(8).is_none_or(|byte| byte & 0xc0 != 0x80) + { + return Err(BridgeError::Parameters); + } + failure( + name, + &format!( + "Function not found: {arn} (Service: AWSLambda; Status Code: 404; Error Code: \ + ResourceNotFoundException; Request ID: {id}; Proxy: null)" + ), + )? + } else { + failure(name, "Lambda invocation admission failed")? + }; + mapped.diagnostic = Some(BridgeError::Service(Arc::new(error))); + Ok(mapped) +} +pub(super) fn executor(error: ExecutorError) -> Result { + let name = match &error { + ExecutorError::Timeout(_) => "Sandbox.Timedout", + ExecutorError::Unsupported(_) + | ExecutorError::InvalidCode(_) + | ExecutorError::InitFailed(_) => "Runtime.InvalidEntrypoint", + ExecutorError::RuntimeExited(_) => "Runtime.ExitError", + _ => "Lambda.ServiceException", + }; + let mut mapped = failure(name, "Lambda execution failed")?; + mapped.diagnostic = Some(BridgeError::Executor(Arc::new(error))); + Ok(mapped) +} diff --git a/apps/rustack/src/stepfunctions_bridge/tests.rs b/apps/rustack/src/stepfunctions_bridge/tests.rs new file mode 100644 index 00000000..ee322914 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/tests.rs @@ -0,0 +1,950 @@ +//! Controlled provider tests plus separately gated actual Native process fixtures. + +use std::{ + collections::{BTreeSet, HashMap}, + io::{Cursor, Write}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; +use bytes::Bytes; +use rustack_lambda_core::{ + config::LambdaConfig, + error::LambdaServiceError, + executor::{Executor, ExecutorBackend, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::InvocationContext, + provider::RustackLambda, + storage::FunctionStore, +}; +use rustack_lambda_model::{ + input::CreateFunctionInput, + types::{Environment, FunctionCode}, +}; +use rustack_stepfunctions_core::{ + AslCompiler, AslError, ClockSample, EvaluationContext, ExecutionDriver, ExecutionLimits, + ExecutionOutcome, ExecutionSupervisor, HistoryLimits, Payload, RuntimeLimits, SystemUuidSource, + TaskSupport, TenantScope, UuidSource, +}; +use serde_json::{Value, json}; +use tempfile::TempDir; +use tokio::{ + sync::Semaphore, + time::{Instant, timeout}, +}; +use uuid::Uuid; +use zip::{ZipWriter, write::SimpleFileOptions}; + +use super::*; + +#[derive(Debug)] +struct TestExecutor { + calls: AtomicUsize, + entered: Arc, + release: Arc, + cleanup: Arc, + fail_cleanup: bool, +} +#[async_trait] +impl Executor for TestExecutor { + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + self.entered.add_permits(1); + tokio::select! {biased;_=context.cancelled()=>{},permit=self.release.acquire()=>{if let Ok(permit)=permit {permit.forget();}}} + if let Ok(permit) = self.cleanup.acquire().await { + permit.forget(); + } + if self.fail_cleanup { + return Err(ExecutorError::CleanupFailed( + "controlled cleanup failure".into(), + )); + } + Ok(InvokeResponse::success(req.payload, "$LATEST".into())) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +fn executor() -> Arc { + Arc::new(TestExecutor { + calls: AtomicUsize::new(0), + entered: Arc::new(Semaphore::new(0)), + release: Arc::new(Semaphore::new(1)), + cleanup: Arc::new(Semaphore::new(1)), + fail_cleanup: false, + }) +} +fn zip(bytes: &[u8], marker: &str) -> Vec { + let mut zip = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default().unix_permissions(0o755); + zip.start_file("bootstrap", options).unwrap(); + zip.write_all(bytes).unwrap(); + zip.start_file("revision.txt", options).unwrap(); + zip.write_all(marker.as_bytes()).unwrap(); + zip.finish().unwrap().into_inner() +} +fn create(code: &[u8], env: &[(&str, &str)]) -> CreateFunctionInput { + let architecture = match std::env::consts::ARCH { + "aarch64" => "arm64", + _ => "x86_64", + }; + CreateFunctionInput { + function_name: "Worker".into(), + runtime: Some("provided.al2023".into()), + handler: Some("bootstrap".into()), + role: "arn:aws:iam::000000000000:role/test".into(), + timeout: Some(5), + architectures: Some(vec![architecture.into()]), + code: FunctionCode { + zip_file: Some(STANDARD.encode(code)), + ..FunctionCode::default() + }, + environment: Some(Environment { + variables: Some( + env.iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ), + }), + ..CreateFunctionInput::default() + } +} +fn provider(executor: Arc) -> (Arc, TempDir) { + let directory = tempfile::tempdir().unwrap(); + let config = LambdaConfig { + executor: ExecutorBackend::Native, + ..LambdaConfig::default() + }; + ( + Arc::new(RustackLambda::with_executor( + FunctionStore::new(directory.path()), + config, + executor, + )), + directory, + ) +} +fn definition(resource: &str, parameters: Option) -> Value { + let mut state = json!({"Type":"Task","Resource":resource,"End":true}); + if let Some(parameters) = parameters { + state["Parameters"] = parameters; + } + json!({"StartAt":"Call","States":{"Call":state}}) +} +fn direct() -> Value { + definition( + "arn:aws:lambda:us-east-1:000000000000:function:Worker", + None, + ) +} +fn driver(definition: &Value, input: &Value) -> ExecutionDriver { + let machine = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .compile(&serde_json::to_vec(definition).unwrap()) + .unwrap() + .machine() + .clone(); + let payload = Payload::from_value(input).unwrap(); + ExecutionDriver::new( + machine, + payload, + EvaluationContext::from_value( + &json!({"Execution":{"Input":input,"Id":"execution-fixture"}}), + ) + .unwrap(), + ClockSample::new( + Instant::now().into_std(), + std::time::SystemTime::now().into(), + ), + ExecutionLimits::new(Duration::from_secs(20), 1000).unwrap(), + ) + .unwrap() +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn executions(bridge: Arc) -> ExecutionSupervisor { + ExecutionSupervisor::new( + RuntimeLimits::new(8, 8, 2).unwrap(), + HistoryLimits::default(), + 256 * 1024 * 1024, + Some(bridge), + ) + .unwrap() +} +fn output(outcome: &ExecutionOutcome) -> &Value { + match outcome { + ExecutionOutcome::Succeeded(output) => output.value(), + other => panic!("expected success, got {other:?}"), + } +} +async fn clean( + owner: &mut ExecutionSupervisor, + bridge: &mut LambdaBridgeSupervisor, + provider: &RustackLambda, +) { + let workflow = owner.drain(deadline()).await; + let joined = bridge.shutdown(deadline()).await; + let provider = provider.quiesce(Duration::from_secs(10)).await; + assert!(workflow.is_ok(), "workflow cleanup {workflow:?}"); + assert!(joined.is_ok(), "bridge cleanup {joined:?}"); + assert!(provider.is_ok(), "provider cleanup {provider:?}"); +} +#[tokio::test] +async fn test_should_invoke_shared_provider_direct_and_optimized_without_http_loopback() { + for optimized in [false, true] { + let exec = executor(); + let (provider, _dir) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port); + let def = if optimized { + definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName":"Worker","Payload.$":"$"})), + ) + } else { + direct() + }; + let mut handle = owner.start(driver(&def, &json!({"number":7}))).unwrap(); + let result = handle.wait(deadline()).await; + clean(&mut owner, &mut bridge, &provider).await; + let result = result.unwrap(); + let body = output(result.outcome().unwrap()); + if optimized { + assert_eq!(body.get("Payload"), Some(&json!({"number":7}))); + assert_eq!(body.get("StatusCode"), Some(&json!(200))); + assert!( + body.pointer("/SdkResponseMetadata/RequestId") + .and_then(Value::as_str) + .is_some_and(|v| v.len() == 36) + ); + } else { + assert_eq!(body, &json!({"number":7})); + } + assert_eq!(exec.calls.load(Ordering::SeqCst), 1); + } +} +#[tokio::test] +async fn test_should_reject_dynamic_partial_and_cross_scope_references_before_execution() { + for function in [ + "999999999999:function:Worker", + "000000000000:function:Worker", + "999999999999:function:Worker:1", + "arn:aws:lambda:us-west-2:000000000000:function:Worker", + "arn:aws:lambda:us-east-1:999999999999:function:Worker", + ] { + let exec = executor(); + let (provider, _dir) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port); + let def = definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName.$":"$.function"})), + ); + let mut handle = owner + .start(driver(&def, &json!({"function":function}))) + .unwrap(); + let result = handle.wait(deadline()).await; + clean(&mut owner, &mut bridge, &provider).await; + assert!( + matches!(result.unwrap().outcome().unwrap().as_ref(),ExecutionOutcome::Failed(f) if f.error()==Some("States.Runtime")) + ); + assert_eq!(exec.calls.load(Ordering::SeqCst), 0); + } +} +#[test] +fn test_should_map_function_error_only_from_transport_flag_and_bound_causes() { + let payload = Bytes::from_static( + br#"{"errorType":"BusinessError","errorMessage":"boom","FunctionError":"business data"}"#, + ); + let response = InvokeResponse::success(payload.clone(), "$LATEST".into()); + let mapped = mapping::response(&response, false, "id").unwrap(); + assert!(mapped.result.is_ok()); + let mut response = InvokeResponse::success(payload, "$LATEST".into()); + response.function_error = Some("Unhandled".into()); + let mapped = mapping::response(&response, false, "id").unwrap(); + assert!(format!("{mapped:?}").find("boom").is_none()); + assert!( + matches!(mapped.result,Err(ref failure) if failure.details().error()==Some("BusinessError")) + ); +} + +mod native; + +#[test] +fn test_should_redact_call_plan_names_and_reject_invalid_limits() { + let plan = mapping::CallPlan { + function: "function-debug-marker".into(), + qualifier: Some("qualifier-debug-marker".into()), + payload: Payload::parse(b"{}").unwrap(), + mode: mapping::ResponseMode::Optimized, + deadline: deadline(), + }; + let debug = format!("{plan:?}"); + assert!(!debug.contains("function-debug-marker")); + assert!(!debug.contains("qualifier-debug-marker")); + for (accepted, seconds) in [(0, 1), (1025, 1), (1, 0), (1, 61)] { + assert!(BridgeLimits::new(accepted, Duration::from_secs(seconds)).is_err()); + } +} +#[test] +fn test_should_enforce_result_quota_utf8_json_and_envelope_depth() { + for bytes in [Bytes::from_static(b"not-json"), Bytes::from_static(&[0xff])] { + let mapped = mapping::response( + &InvokeResponse::success(bytes, "$LATEST".into()), + false, + "id", + ) + .unwrap(); + assert!(matches!(mapped.result,Err(f) if f.details().error()==Some("States.Runtime"))); + assert!(mapped.diagnostic.is_some()); + } + let exact = Bytes::from(serde_json::to_vec(&"x".repeat(262_142)).unwrap()); + assert_eq!(exact.len(), 262_144); + assert!( + mapping::response( + &InvokeResponse::success(exact.clone(), "$LATEST".into()), + false, + "id" + ) + .unwrap() + .result + .is_ok() + ); + let optimized = mapping::response( + &InvokeResponse::success(exact, "$LATEST".into()), + true, + "id", + ) + .unwrap(); + assert!( + matches!(optimized.result,Err(f) if f.details().error()==Some("States.DataLimitExceeded")) + ); + let oversized = Bytes::from(vec![b' '; 262_145]); + let mapped = mapping::response( + &InvokeResponse::success(oversized, "$LATEST".into()), + false, + "id", + ) + .unwrap(); + assert!( + matches!(mapped.result,Err(f) if f.details().error()==Some("States.DataLimitExceeded")) + ); + let mut nested = Value::Null; + for _ in 0..64 { + nested = json!([nested]); + } + let nested = Bytes::from(serde_json::to_vec(&nested).unwrap()); + let mapped = mapping::response( + &InvokeResponse::success(nested, "$LATEST".into()), + true, + "id", + ) + .unwrap(); + assert!(matches!(mapped.result,Err(f) if f.details().error()==Some("States.Runtime"))); +} +#[test] +fn test_should_bound_function_error_type_and_utf8_cause_without_leaking_debug_data() { + for error_type in [ + "States.Timeout".to_owned(), + String::new(), + "x".repeat(257), + "bad\nname".to_owned(), + "BusinessError".to_owned(), + ] { + let body = + serde_json::to_vec(&json!({"errorType":error_type,"errorMessage":"é".repeat(40_000)})) + .unwrap(); + let mut response = InvokeResponse::success(Bytes::from(body), "$LATEST".into()); + response.function_error = Some("Unhandled".into()); + let mapped = mapping::response(&response, false, "id").unwrap(); + let failure = mapped.result.unwrap_err(); + assert_eq!( + failure.details().error(), + Some(if error_type == "BusinessError" { + "BusinessError" + } else { + "Lambda.Unknown" + }) + ); + let cause = failure.details().cause().unwrap(); + assert!(cause.len() <= 32_768); + assert!(cause.ends_with(" [truncated]")); + assert!(!format!("{failure:?}").contains("errorMessage")); + } +} +#[tokio::test] +async fn test_should_preserve_cleanup_after_stop_and_resume_same_shutdown_waiter() { + let exec = Arc::new(TestExecutor { + calls: AtomicUsize::new(0), + entered: Arc::new(Semaphore::new(0)), + release: Arc::new(Semaphore::new(0)), + cleanup: Arc::new(Semaphore::new(0)), + fail_cleanup: false, + }); + let (provider, _directory) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::new(1, Duration::from_secs(1)).unwrap(), + ) + .unwrap(); + let mut owner = executions(port); + let mut handle = owner.start(driver(&direct(), &json!({}))).unwrap(); + timeout(Duration::from_secs(5), exec.entered.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + let stopped = handle + .stop(TaskFailure::new("Stopped", None).unwrap(), deadline()) + .await + .unwrap(); + assert!(matches!( + stopped.outcome().unwrap().as_ref(), + ExecutionOutcome::Aborted(_) + )); + assert!( + matches!(bridge.shutdown(Instant::now()+Duration::from_millis(20)).await,Err(e) if matches!(e.as_ref(),BridgeError::Deadline)) + ); + assert_eq!(bridge.owned_calls(), 1); + // Keep the same handle alive past the first bounded cleanup observation. + tokio::time::sleep(Duration::from_millis(1100)).await; + assert_eq!(bridge.owned_calls(), 1); + exec.cleanup.add_permits(1); + clean(&mut owner, &mut bridge, &provider).await; + assert_eq!(bridge.owned_calls(), 0); + assert_eq!( + handle.snapshot().cleanup(), + rustack_stepfunctions_core::CleanupState::Clean + ); +} +#[tokio::test] +async fn test_should_reject_evaluated_invalid_invocation_type_without_executing() { + let exec = executor(); + let (provider, _directory) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port); + let definition = definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName":"Worker","InvocationType.$":"$.kind"})), + ); + let mut handle = owner + .start(driver(&definition, &json!({"kind":"Event"}))) + .unwrap(); + let result = handle.wait(deadline()).await; + clean(&mut owner, &mut bridge, &provider).await; + assert!( + matches!(result.unwrap().outcome().unwrap().as_ref(),ExecutionOutcome::Failed(f) if f.error()==Some("States.Runtime")) + ); + assert_eq!(exec.calls.load(Ordering::SeqCst), 0); +} +#[tokio::test] +async fn test_should_refuse_mismatched_provider_scope_and_disabled_backend() { + let exec = executor(); + let (provider, _directory) = provider(exec); + let scope = TenantScope::new("aws", "us-west-2", "000000000000").unwrap(); + assert!(matches!( + StepFunctionsLambdaBridge::new(provider, scope, BridgeLimits::default()), + Err(BridgeError::Configuration) + )); + let provider = Arc::new(RustackLambda::new(LambdaConfig::default())); + assert!(matches!( + StepFunctionsLambdaBridge::new(provider, TenantScope::default(), BridgeLimits::default()), + Err(BridgeError::Configuration) + )); +} + +#[tokio::test] +async fn test_should_retain_uncertain_backend_cleanup_and_refuse_clean_shutdown() { + let exec = Arc::new(TestExecutor { + calls: AtomicUsize::new(0), + entered: Arc::new(Semaphore::new(0)), + release: Arc::new(Semaphore::new(1)), + cleanup: Arc::new(Semaphore::new(1)), + fail_cleanup: true, + }); + let (provider, _directory) = provider(exec); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::new(1, Duration::from_secs(1)).unwrap(), + ) + .unwrap(); + let mut owner = executions(port); + let mut handle = owner.start(driver(&direct(), &json!({}))).unwrap(); + let view = handle.wait(deadline()).await.unwrap(); + assert!( + matches!(view.outcome().unwrap().as_ref(),ExecutionOutcome::Failed(f) if f.error()==Some("Lambda.ServiceException")) + ); + assert!( + matches!(bridge.shutdown(deadline()).await,Err(e) if matches!(e.as_ref(),BridgeError::Quarantined)) + ); + assert_eq!(bridge.owned_calls(), 1); + assert!(matches!( + owner + .drain(Instant::now() + Duration::from_millis(20)) + .await, + Err(rustack_stepfunctions_core::ExecutionError::Deadline) + )); + assert!(!format!("{:?}", bridge.diagnostic()).contains("controlled cleanup")); +} +#[tokio::test] +async fn test_should_preserve_root_admission_permit_after_manager_abort() { + let exec = Arc::new(TestExecutor { + calls: AtomicUsize::new(0), + entered: Arc::new(Semaphore::new(0)), + release: Arc::new(Semaphore::new(0)), + cleanup: Arc::new(Semaphore::new(0)), + fail_cleanup: false, + }); + let (provider, _directory) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::new(1, Duration::from_secs(1)).unwrap(), + ) + .unwrap(); + let mut owner = executions(port); + let mut handle = owner.start(driver(&direct(), &json!({}))).unwrap(); + timeout(Duration::from_secs(5), exec.entered.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + bridge.manager.as_ref().unwrap().abort(); + assert!( + matches!(bridge.shutdown(deadline()).await,Err(e) if matches!(e.as_ref(),BridgeError::Worker(_))) + ); + assert_eq!(bridge.owned_calls(), 1); + exec.cleanup.add_permits(1); + let view = handle.wait(deadline()).await.unwrap(); + assert!(matches!( + view.outcome().unwrap().as_ref(), + ExecutionOutcome::Failed(_) + )); + owner.drain(deadline()).await.unwrap(); + provider.quiesce(Duration::from_secs(10)).await.unwrap(); + assert_eq!(bridge.owned_calls(), 1); +} + +#[tokio::test] +async fn test_should_keep_bridge_ready_after_settled_missing_function_diagnostic() { + let exec = executor(); + let (provider, _directory) = provider(exec.clone()); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port.clone()); + let mut handle = owner.start(driver(&direct(), &json!({}))).unwrap(); + let outcome = handle.wait(deadline()).await; + let settled = timeout(Duration::from_secs(5), async { + while bridge.owned_calls() != 0 { + tokio::task::yield_now().await; + } + }) + .await; + let ready = port.is_ready(); + let diagnostic = bridge.diagnostic(); + clean(&mut owner, &mut bridge, &provider).await; + assert!(settled.is_ok()); + assert!( + matches!(outcome.unwrap().outcome().unwrap().as_ref(),ExecutionOutcome::Failed(f) if f.error()==Some("Lambda.ResourceNotFoundException")) + ); + assert!(diagnostic.is_some()); + assert!(ready, "historical diagnostic is not owner failure"); + assert_eq!(exec.calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn test_should_ignore_callback_success_body_but_preserve_function_failures_and_actual_metadata() { + use rustack_stepfunctions_core::CallbackInvocationMetadata; + use uuid::Uuid; + + let id = Uuid::new_v4(); + for body in [ + Bytes::from_static(b"not-json"), + Bytes::from_static(&[0xff]), + Bytes::from(vec![b'x'; 262_145]), + ] { + let response = InvokeResponse::success(body, "42".into()); + assert!( + mapping::response(&response, true, &id.to_string()) + .unwrap() + .result + .is_err() + ); + let (mapped, metadata) = + mapping::observed_response(&response, CallbackInvocationMetadata::new(id, "42")) + .unwrap(); + let output = mapped.result.unwrap(); + assert!(output.value().get("Payload").is_none()); + assert_eq!(output.value()["ExecutedVersion"], "42"); + assert_eq!( + output.value()["SdkResponseMetadata"]["RequestId"], + id.to_string() + ); + assert_eq!(metadata.unwrap().request_id(), id); + } + let mut response = InvokeResponse::success( + Bytes::from_static(br#"{"errorType":"RealFailure"}"#), + "$LATEST".into(), + ); + response.function_error = Some("Unhandled".into()); + let (mapped, metadata) = + mapping::observed_response(&response, CallbackInvocationMetadata::new(id, "$LATEST")) + .unwrap(); + assert!(matches!(mapped.result,Err(f) if f.details().error()==Some("RealFailure"))); + assert!(metadata.is_none()); + response.function_error = None; + let (mapped, metadata) = + mapping::observed_response(&response, CallbackInvocationMetadata::new(id, "alias")) + .unwrap(); + assert!(matches!(mapped.result,Err(f) if f.details().error()==Some("Lambda.ServiceException"))); + assert!(mapped.diagnostic.is_some()); + assert!(metadata.is_none()); +} + +#[rstest::rstest] +#[case(Value::Null, json!({}))] +#[case(json!("null"), Value::Null)] +#[case(json!("42"), json!(42))] +#[case(json!("\"hello\""), json!("hello"))] +#[case(json!("[1,true]"), json!([1,true]))] +#[case(json!({"number":7}), json!({"number":7}))] +#[tokio::test] +async fn test_should_decode_optimized_serialized_payload_without_double_encoding( + #[case] payload: Value, + #[case] expected: Value, +) { + let exec = executor(); + let (provider, _directory) = provider(exec.clone()); + provider + .create_function(create(&zip(b"fixture", "v1"), &[])) + .await + .unwrap(); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port); + let def = definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName":"Worker","Payload":payload})), + ); + let mut handle = owner.start(driver(&def, &json!({}))).unwrap(); + let result = handle.wait(deadline()).await; + clean(&mut owner, &mut bridge, &provider).await; + assert_eq!( + output(result.unwrap().outcome().unwrap()).get("Payload"), + Some(&expected) + ); + assert_eq!(exec.calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn test_should_report_missing_optimized_function_as_started_service_failure() { + let exec = executor(); + let (provider, _directory) = provider(exec.clone()); + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let mut owner = executions(port); + let def = definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName":"Worker"})), + ); + let mut handle = owner.start(driver(&def, &json!({}))).unwrap(); + let result = handle.wait(deadline()).await; + clean(&mut owner, &mut bridge, &provider).await; + let result = result.unwrap(); + let page = result.history().page(1, 100, true).unwrap(); + assert_eq!( + page.events() + .iter() + .map(|e| e.r#type.as_str()) + .collect::>(), + [ + "ExecutionStarted", + "TaskStateEntered", + "TaskScheduled", + "TaskStarted", + "TaskFailed", + "ExecutionFailed" + ] + ); + assert_eq!(exec.calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn test_should_compact_function_error_cause_without_changing_its_values() { + let mut response = InvokeResponse::success( + Bytes::from_static(br#"{ "errorType" : "Oops", "errorMessage" : "line\nvalue" }"#), + "$LATEST".into(), + ); + response.function_error = Some("Unhandled".into()); + let mapped = mapping::response(&response, false, "id").unwrap(); + let failure = mapped.result.unwrap_err(); + assert_eq!( + failure.details().cause(), + Some(r#"{"errorMessage":"line\nvalue","errorType":"Oops"}"#) + ); +} + +#[test] +fn test_should_preserve_function_error_when_compact_numeric_cause_expands_past_payload_limit() { + let mut raw = String::from(r#"{"errorType":"Oops","items":["#); + raw.push_str(&vec!["1e9"; 40_000].join(",")); + raw.push_str("]}"); + assert!(raw.len() < 262_144); + let mut response = InvokeResponse::success(Bytes::from(raw), "$LATEST".into()); + response.function_error = Some("Unhandled".into()); + let failure = mapping::response(&response, false, "id") + .unwrap() + .result + .unwrap_err(); + assert_eq!(failure.details().error(), Some("Oops")); + let cause = failure.details().cause().unwrap(); + assert_eq!(cause.len(), 32_768); + assert!(cause.ends_with(" [truncated]")); + assert!(cause.starts_with(r#"{"errorType":"Oops","items":[1000000000.0"#)); +} + +#[derive(Debug)] +struct FixedServiceId(Uuid); +impl UuidSource for FixedServiceId { + fn next_uuid(&mut self) -> Result { + Ok(self.0) + } +} +#[derive(Debug)] +struct FailedServiceId; +impl UuidSource for FailedServiceId { + fn next_uuid(&mut self) -> Result { + Err(AslError::IntrinsicValue) + } +} +fn service_plan(function: &str, qualifier: Option<&str>) -> mapping::CallPlan { + mapping::CallPlan { + function: function.into(), + qualifier: qualifier.map(str::to_owned), + payload: Payload::parse(b"{}").unwrap(), + mode: mapping::ResponseMode::Optimized, + deadline: deadline(), + } +} + +#[rstest::rstest] +#[case("Worker", None, "$LATEST")] +#[case("Worker", Some("alias"), "alias")] +#[case( + "arn:aws-cn:lambda:cn-north-1:123456789012:function:Worker:2", + None, + "2" +)] +#[case( + "arn:aws-cn:lambda:cn-north-1:123456789012:function:Worker:alias", + Some("alias"), + "alias" +)] +#[case( + "arn:aws-cn:lambda:cn-north-1:123456789012:function:Worker", + None, + "$LATEST" +)] +fn test_should_render_missing_function_cause_from_validated_target_and_scope( + #[case] function: &str, + #[case] qualifier: Option<&str>, + #[case] expected: &str, +) { + let scope = TenantScope::new("aws-cn", "cn-north-1", "123456789012").unwrap(); + let target = mapping::ServiceTarget::new(&service_plan(function, qualifier), &scope); + let id = Uuid::parse_str("5a09f4e4-fbdc-49eb-9a92-9194264473cf").unwrap(); + let mapped = mapping::service( + LambdaServiceError::FunctionNotFound { + name: "private-provider-name\n/host/path".into(), + }, + &target, + &mut FixedServiceId(id), + ) + .unwrap(); + assert!(!format!("{target:?} {mapped:?}").contains("Worker")); + let failure = mapped.result.unwrap_err(); + assert_eq!( + failure.details().error(), + Some("Lambda.ResourceNotFoundException") + ); + assert_eq!( + failure.details().cause(), + Some( + format!( + "Function not found: \ + arn:aws-cn:lambda:cn-north-1:123456789012:function:Worker:{expected} (Service: \ + AWSLambda; Status Code: 404; Error Code: ResourceNotFoundException; Request ID: \ + {id}; Proxy: null)" + ) + .as_str() + ) + ); +} + +#[test] +fn test_should_preserve_missing_version_alias_and_unrelated_service_error_categories() { + let target = mapping::ServiceTarget::new( + &service_plan("Worker", Some("missing")), + &TenantScope::default(), + ); + for error in [ + LambdaServiceError::VersionNotFound { + function_name: "private".into(), + version: "123".into(), + }, + LambdaServiceError::AliasNotFound { + function_name: "private".into(), + alias: "private".into(), + }, + ] { + let failure = mapping::service(error, &target, &mut SystemUuidSource::default()) + .unwrap() + .result + .unwrap_err(); + assert!(failure.details().cause().unwrap().starts_with( + "Function not found: arn:aws:lambda:us-east-1:000000000000:function:Worker:missing " + )); + } + let mapped = mapping::service( + LambdaServiceError::TooManyRequests, + &target, + &mut FailedServiceId, + ) + .unwrap(); + assert!( + matches!(mapped.result,Err(f) if f.details().error()==Some("Lambda.TooManyRequestsException") && f.details().cause()==Some("Lambda invocation admission failed")) + ); +} + +#[test] +fn test_should_reject_invalid_service_targets_and_failed_or_invalid_identity_sources() { + let id = Uuid::parse_str("5a09f4e4-fbdc-49eb-9a92-9194264473cf").unwrap(); + for (function, qualifier) in [ + ( + "arn:aws:lambda:us-west-2:000000000000:function:Worker", + None, + ), + ("Worker", Some("bad\nqualifier")), + ( + "arn:aws:lambda:us-east-1:000000000000:function:Worker:1", + Some("2"), + ), + ] { + let target = mapping::ServiceTarget::new( + &service_plan(function, qualifier), + &TenantScope::default(), + ); + assert!( + mapping::service( + LambdaServiceError::FunctionNotFound { + name: "ignored".into() + }, + &target, + &mut FixedServiceId(id) + ) + .is_err() + ); + } + let target = + mapping::ServiceTarget::new(&service_plan("Worker", None), &TenantScope::default()); + assert!( + mapping::service( + LambdaServiceError::FunctionNotFound { + name: "ignored".into() + }, + &target, + &mut FailedServiceId + ) + .is_err() + ); + for id in [ + Uuid::nil(), + Uuid::from_u128(0x0000_0000_0000_4000_c000_0000_0000_0000), + ] { + assert!( + mapping::service( + LambdaServiceError::FunctionNotFound { + name: "ignored".into() + }, + &target, + &mut FixedServiceId(id) + ) + .is_err() + ); + } + let mut causes = BTreeSet::new(); + for _ in 0..2 { + let failure = mapping::service( + LambdaServiceError::FunctionNotFound { + name: "ignored".into(), + }, + &target, + &mut SystemUuidSource::default(), + ) + .unwrap() + .result + .unwrap_err(); + assert!(causes.insert(failure.details().cause().unwrap().to_owned())); + } +} diff --git a/apps/rustack/src/stepfunctions_bridge/tests/native.rs b/apps/rustack/src/stepfunctions_bridge/tests/native.rs new file mode 100644 index 00000000..284d3276 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/tests/native.rs @@ -0,0 +1,1287 @@ +//! Actual host process → Runtime API → ASL fixtures. Built by the Makefile gate. + +use std::{ + collections::BTreeSet, + env, + error::Error, + path::Path, + process::{self, ExitStatus, Stdio}, + time::{Duration, SystemTime}, +}; + +use http::{Request as HttpRequest, StatusCode}; +use http_body_util::{BodyExt, Full}; +use rustack_lambda_model::input::{ + CreateAliasInput, UpdateAliasInput, UpdateFunctionCodeInput, UpdateFunctionConfigurationInput, +}; +use rustack_stepfunctions_core::{ + ChildPattern, ChildStartParameters, ExecutionSnapshot, PreparedDefinition, Registry, + RegistryCommand, RegistryError, RegistryLimits, RegistryResult, RegistrySupervisor, + SnapshotLimits, StateMachineArn, StateSupport, Tags, +}; +use rustack_stepfunctions_http::{RequestScope, StepFunctionsHttpConfig, StepFunctionsHttpService}; +use tokio::{ + fs, + io::AsyncReadExt, + process::{Child, Command}, + time::{error::Elapsed, sleep, timeout, timeout_at}, +}; + +use super::*; +use crate::stepfunctions_api::{ApiLimits, StepFunctionsApi}; + +#[derive(Debug)] +struct NativeFixture { + provider: Arc, + owner: ExecutionSupervisor, + bridge: LambdaBridgeSupervisor, + port: Arc, + directory: Option, + bootstrap: Bytes, +} +impl NativeFixture { + async fn new(env: &[(&str, &str)], publish: bool) -> Self { + let path = env::var_os("RUSTACK_WORKFLOWS_BOOTSTRAP") + .expect("run make test-stepfunctions-bridge-native"); + let bootstrap = Bytes::from(fs::read(path).await.unwrap()); + let directory = tempfile::tempdir().unwrap(); + let config = LambdaConfig { + executor: ExecutorBackend::Native, + init_timeout: Duration::from_secs(5), + max_warm_instances: 2, + ..LambdaConfig::default() + }; + let provider = Arc::new(RustackLambda::with_store( + FunctionStore::new(directory.path()), + config, + )); + let bytes = bootstrap.clone(); + let code = tokio::task::spawn_blocking(move || zip(&bytes, "native-v1")) + .await + .unwrap(); + let marker = directory.path().join("invocation.json"); + let mut variables = env.to_vec(); + variables.push(("INVOCATION_FILE", marker.to_str().unwrap())); + let mut input = create(&code, &variables); + input.publish = Some(publish); + provider.create_function(input).await.unwrap(); + let (port, bridge) = StepFunctionsLambdaBridge::new( + provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + Self { + provider, + owner: executions(port.clone()), + port, + bridge, + directory: Some(directory), + bootstrap, + } + } + async fn ready_after_calls(&self) -> bool { + timeout_at(deadline(), async { + while self.bridge.owned_calls() != 0 { + sleep(Duration::from_millis(1)).await; + } + self.port.is_ready() + }) + .await + .unwrap_or(false) + } + async fn cleanup(&mut self) { + self.cleanup_at(Instant::now() + Duration::from_secs(15)) + .await; + } + async fn cleanup_at(&mut self, absolute: Instant) { + let workflow = self.owner.drain(absolute).await; + let bridge = self.bridge.shutdown(absolute).await; + let provider = self + .provider + .quiesce(absolute.saturating_duration_since(Instant::now())) + .await; + let processes = timeout_at(absolute, self.provider.executor().shutdown()).await; + if workflow.is_err() + || bridge.is_err() + || provider.is_err() + || !matches!(&processes, Ok(Ok(()))) + { + // Unproven process retirement must not remove its still-owned files. + if let Some(directory) = self.directory.take() { + let _retained = directory.keep(); + } + } + assert!(workflow.is_ok(), "workflow cleanup: {workflow:?}"); + assert!(bridge.is_ok(), "bridge cleanup: {bridge:?}"); + assert!(provider.is_ok(), "provider cleanup: {provider:?}"); + assert!( + matches!(processes, Ok(Ok(()))), + "native process cleanup: {processes:?}" + ); + } +} +fn workflow(optimized: bool) -> Value { + let parameters = json!({"n.$":"$.n","retryCount.$":"$$.State.RetryCount"}); + if optimized { + definition( + "arn:aws:states:::lambda:invoke", + Some(json!({"FunctionName":"Worker","Payload":parameters})), + ) + } else { + definition( + "arn:aws:lambda:us-east-1:000000000000:function:Worker", + Some(parameters), + ) + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_computation_and_return_actual_invocation_metadata() { + for optimized in [false, true] { + let mut fixture = NativeFixture::new(&[("WORKFLOW_MODE", "1")], false).await; + let mut handle = fixture + .owner + .start(driver(&workflow(optimized), &json!({"n":21}))) + .unwrap(); + let result = handle.wait(deadline()).await; + fixture.cleanup().await; + let view = result.unwrap(); + let response = output(view.outcome().unwrap()); + let body = if optimized { + response.get("Payload").unwrap() + } else { + response + }; + assert_eq!(body.get("computed"), Some(&json!(42))); + assert_eq!(body.get("codeMarker"), Some(&json!("native-v1"))); + assert_ne!( + body.get("pid").and_then(Value::as_u64).unwrap(), + u64::from(process::id()) + ); + let request_id = body.get("request_id").and_then(Value::as_str).unwrap(); + assert_eq!(request_id.len(), 36); + if optimized { + assert_eq!( + response + .pointer("/SdkResponseMetadata/RequestId") + .and_then(Value::as_str), + Some(request_id) + ); + assert_eq!(response.get("StatusCode"), Some(&json!(200))); + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_retry_function_error_and_recalculate_parameters() { + let mut fixture = NativeFixture::new( + &[("WORKFLOW_MODE", "1"), ("WORKFLOW_FAIL_ONCE", "1")], + false, + ) + .await; + let mut definition = workflow(false); + definition["States"]["Call"]["Retry"] = + json!([{"ErrorEquals":["WorkflowTransient"],"MaxAttempts":1,"IntervalSeconds":1}]); + let mut handle = fixture + .owner + .start(driver(&definition, &json!({"n":6}))) + .unwrap(); + let result = handle.wait(deadline()).await; + fixture.cleanup().await; + let view = result.unwrap(); + let response = output(view.outcome().unwrap()); + assert_eq!(response.get("computed"), Some(&json!(12))); + assert_eq!(response.get("retryCount"), Some(&json!(1))); + let history = view.history().page(1, 100, true).unwrap(); + let kinds = history + .events() + .iter() + .map(|e| e.r#type.as_str()) + .collect::>(); + assert_eq!( + kinds.iter().filter(|&&k| k == "TaskStateEntered").count(), + 1 + ); + assert_eq!( + kinds + .iter() + .filter(|&&k| k == "LambdaFunctionStarted") + .count(), + 2 + ); + assert_eq!( + kinds + .iter() + .filter(|&&k| k == "LambdaFunctionFailed") + .count(), + 1 + ); + assert_eq!(kinds.last().copied(), Some("ExecutionSucceeded")); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_resolve_alias_again_on_retry_and_capture_new_version() { + let mut fixture = + NativeFixture::new(&[("WORKFLOW_MODE", "1"), ("WORKFLOW_FAIL_ONCE", "1")], true).await; + let bytes = fixture.bootstrap.clone(); + let code = tokio::task::spawn_blocking(move || zip(&bytes, "native-v2")) + .await + .unwrap(); + fixture + .provider + .update_function_code( + "Worker", + UpdateFunctionCodeInput { + zip_file: Some(STANDARD.encode(code)), + publish: Some(true), + ..UpdateFunctionCodeInput::default() + }, + ) + .await + .unwrap(); + fixture + .provider + .create_alias( + "Worker", + CreateAliasInput { + name: "live".into(), + function_version: "1".into(), + ..CreateAliasInput::default() + }, + ) + .unwrap(); + let mut definition = workflow(true); + definition["States"]["Call"]["Parameters"]["Qualifier"] = json!("live"); + definition["States"]["Call"]["Retry"] = + json!([{"ErrorEquals":["WorkflowTransient"],"MaxAttempts":1,"IntervalSeconds":1}]); + let mut handle = fixture + .owner + .start(driver(&definition, &json!({"n":5}))) + .unwrap(); + let failed = timeout_at(deadline(), async { + loop { + let view = handle.snapshot(); + let history = view.history().page(1, 100, true).unwrap(); + if history + .events() + .iter() + .any(|e| e.r#type.as_str() == "TaskFailed") + { + break; + } + sleep(Duration::from_millis(5)).await; + } + }) + .await; + let updated = fixture.provider.update_alias( + "Worker", + "live", + &UpdateAliasInput { + function_version: Some("2".into()), + ..UpdateAliasInput::default() + }, + ); + let result = handle.wait(deadline()).await; + fixture.cleanup().await; + failed.unwrap(); + updated.unwrap(); + let view = result.unwrap(); + let response = output(view.outcome().unwrap()); + assert_eq!(response.get("ExecutedVersion"), Some(&json!("2"))); + assert_eq!( + response.pointer("/Payload/codeMarker"), + Some(&json!("native-v2")) + ); + assert_eq!(response.pointer("/Payload/computed"), Some(&json!(10))); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_catch_actual_function_error() { + let mut fixture = NativeFixture::new(&[("FAIL_MODE", "panic")], false).await; + let mut definition = direct(); + definition["States"]["Call"]["Catch"] = + json!([{"ErrorEquals":["TestError"],"ResultPath":"$.failure","Next":"Recovered"}]); + definition["States"]["Recovered"] = json!({"Type":"Succeed"}); + let mut handle = fixture + .owner + .start(driver(&definition, &json!({"keep":7}))) + .unwrap(); + let result = handle.wait(deadline()).await; + fixture.cleanup().await; + let view = result.unwrap(); + let response = output(view.outcome().unwrap()); + assert_eq!(response.get("keep"), Some(&json!(7))); + assert_eq!( + response.pointer("/failure/Error"), + Some(&json!("TestError")) + ); + assert!( + response + .pointer("/failure/Cause") + .and_then(Value::as_str) + .unwrap() + .contains("boom") + ); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_cancel_actual_process_on_stop_and_task_timeout() { + for stop in [false, true] { + let mut fixture = NativeFixture::new(&[("SLEEP_SECS", "5")], false).await; + let mut definition = direct(); + definition["States"]["Call"]["TimeoutSeconds"] = json!(1); + let mut handle = fixture + .owner + .start(driver(&definition, &json!({}))) + .unwrap(); + let marker = fixture + .directory + .as_ref() + .unwrap() + .path() + .join("invocation.json"); + let started = timeout_at(deadline(), async { + loop { + match fs::read_to_string(&marker).await { + Ok(body) => match serde_json::from_str::(&body) { + Ok(marker) => break Ok(marker), + Err(_) => sleep(Duration::from_millis(5)).await, + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + sleep(Duration::from_millis(5)).await; + } + Err(error) => break Err(error), + } + } + }) + .await; + let result = if stop { + handle + .stop(TaskFailure::new("Stopped", None).unwrap(), deadline()) + .await + } else { + handle.wait(deadline()).await + }; + let ready = fixture.ready_after_calls().await; + fixture.cleanup().await; + assert!(ready, "settled Task timeout/Stop poisoned bridge readiness"); + let marker = started.unwrap().unwrap(); + let pid = marker.get("pid").and_then(Value::as_u64).unwrap(); + assert!(pid > 1 && pid != u64::from(process::id())); + assert_eq!( + marker + .get("request_id") + .and_then(Value::as_str) + .unwrap() + .len(), + 36 + ); + verify_absent_probe(pid).await.unwrap(); + let view = result.unwrap(); + if stop { + assert!(matches!( + view.outcome().unwrap().as_ref(), + ExecutionOutcome::Aborted(_) + )); + } else { + assert!( + matches!(view.outcome().unwrap().as_ref(),ExecutionOutcome::Failed(f) if f.error()==Some("States.Timeout")) + ); + } + assert_eq!( + handle.snapshot().cleanup(), + rustack_stepfunctions_core::CleanupState::Clean + ); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_preserve_optimized_payload_json_types_and_default() { + let mut fixture = NativeFixture::new(&[], false).await; + let mut results = Vec::new(); + for (payload, expected) in [ + (None, json!({})), + (Some(Value::Null), json!({})), + (Some(json!(r#""json-string""#)), json!("json-string")), + (Some(json!([1, 2])), json!([1, 2])), + (Some(json!("null")), Value::Null), + ] { + let mut params = json!({"FunctionName":"Worker"}); + if let Some(payload) = &payload { + params["Payload"] = payload.clone(); + } + let definition = definition("arn:aws:states:::lambda:invoke", Some(params)); + let result = match fixture + .owner + .start(driver(&definition, &json!({"notForwarded":true}))) + { + Ok(mut handle) => handle.wait(deadline()).await, + Err(error) => Err(error), + }; + let failed = result.is_err(); + results.push((expected, result)); + if failed { + break; + } + } + fixture.cleanup().await; + assert_eq!(results.len(), 5); + for (expected, result) in results { + let view = result.unwrap(); + assert_eq!( + output(view.outcome().unwrap()).pointer("/Payload/echo"), + Some(&expected) + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_preserve_function_timeout_class_for_asl_catch() { + let mut fixture = NativeFixture::new(&[("SLEEP_SECS", "5")], false).await; + fixture + .provider + .update_function_configuration( + "Worker", + &UpdateFunctionConfigurationInput { + timeout: Some(1), + ..UpdateFunctionConfigurationInput::default() + }, + ) + .unwrap(); + let mut definition = direct(); + definition["States"]["Call"]["Catch"] = + json!([{"ErrorEquals":["Sandbox.Timedout"],"Next":"Recovered"}]); + definition["States"]["Recovered"] = json!({"Type":"Succeed"}); + let mut handle = fixture + .owner + .start(driver(&definition, &json!({}))) + .unwrap(); + let result = handle.wait(deadline()).await; + let ready = fixture.ready_after_calls().await; + fixture.cleanup().await; + assert!(ready, "settled function timeout poisoned bridge readiness"); + let view = result.unwrap(); + assert_eq!( + output(view.outcome().unwrap()).get("Error"), + Some(&json!("Sandbox.Timedout")) + ); +} + +async fn run_registry_case( + registry: &Registry, +) -> Result<(ExecutionSnapshot, ExecutionSnapshot, ExecutionSnapshot), Box> { + let compiler = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda); + let prepared = PreparedDefinition::new( + compiler + .compile(&serde_json::to_vec(&workflow(true))?)? + .machine() + .clone(), + ); + let mut create = registry.submit(RegistryCommand::Create( + "NativeWorkflow".parse()?, + prepared, + "arn:aws:iam::000000000000:role/Workflow".parse()?, + Tags::default(), + ))?; + let RegistryResult::Machine(machine) = create.wait(deadline()).await? else { + return Err(RegistryError::Unavailable.into()); + }; + let mut start = registry.submit(RegistryCommand::Start( + machine.arn().clone(), + Some("first".parse()?), + Payload::parse(br#"{"n":21}"#)?, + None, + ))?; + let RegistryResult::Execution(first) = start.wait(deadline()).await? else { + return Err(RegistryError::Unavailable.into()); + }; + first.handle().wait(deadline()).await?; + let mut updated = workflow(true); + updated["States"]["Call"]["Parameters"]["Payload"] = json!({"n":22,"retryCount":0}); + let prepared = PreparedDefinition::new( + compiler + .compile(&serde_json::to_vec(&updated)?)? + .machine() + .clone(), + ); + let mut update = registry.submit(RegistryCommand::Update( + machine.arn().clone(), + Some(prepared), + None, + ))?; + update.wait(deadline()).await?; + let mut start = registry.submit(RegistryCommand::Start( + machine.arn().clone(), + Some("second".parse()?), + Payload::parse(br#"{"n":21}"#)?, + None, + ))?; + let RegistryResult::Execution(second) = start.wait(deadline()).await? else { + return Err(RegistryError::Unavailable.into()); + }; + second.handle().wait(deadline()).await?; + let mut delete = registry.submit(RegistryCommand::Delete(machine.arn().clone()))?; + delete.wait(deadline()).await?; + let mut read = registry.submit(RegistryCommand::Execution(first.arn().clone()))?; + let RegistryResult::Execution(first_final) = read.wait(deadline()).await? else { + return Err(RegistryError::Unavailable.into()); + }; + let mut read = registry.submit(RegistryCommand::Execution(second.arn().clone()))?; + let RegistryResult::Execution(second_final) = read.wait(deadline()).await? else { + return Err(RegistryError::Unavailable.into()); + }; + Ok((first, first_final, second_final)) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_captured_registry_revisions_and_read_after_machine_deletion() { + let mut fixture = NativeFixture::new(&[("WORKFLOW_MODE", "1")], false).await; + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + fixture.provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let (registry, mut owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + executions(port), + ExecutionLimits::default(), + ) + .unwrap(); + let result = run_registry_case(®istry).await; + let absolute = Instant::now() + Duration::from_secs(15); + let workflow = owner.shutdown(absolute).await; + let calls = bridge.shutdown(absolute).await; + if workflow.is_err() || calls.is_err() { + if let Some(directory) = fixture.directory.take() { + let _retained = directory.keep(); + } + } + fixture.cleanup().await; + assert!(workflow.is_ok(), "registry cleanup: {workflow:?}"); + assert!(calls.is_ok(), "bridge cleanup: {calls:?}"); + let (first, captured, second) = result.unwrap(); + let archived = + serde_json::to_value(registry.export_snapshot(SnapshotLimits::default()).unwrap()).unwrap(); + assert_eq!(archived.get("machines"), Some(&json!([]))); + assert_eq!(archived.get("lambdaSupport"), Some(&json!(true))); + assert_eq!( + archived + .get("revisions") + .and_then(Value::as_array) + .map(Vec::len), + Some(2) + ); + for execution in [&captured, &second] { + let saved = archived + .get("executions") + .and_then(Value::as_array) + .unwrap() + .iter() + .find(|row| row.get("arn").and_then(Value::as_str) == Some(execution.arn().as_str())) + .unwrap(); + let saved_output: Value = serde_json::from_str( + saved + .pointer("/outcome/output") + .and_then(Value::as_str) + .unwrap(), + ) + .unwrap(); + assert_eq!(&saved_output, output(execution.view().outcome().unwrap())); + let history = saved.get("history").and_then(Value::as_array).unwrap(); + assert_eq!( + history.len(), + usize::try_from(execution.view().history().len()).unwrap() + ); + for (row, event) in history.iter().zip( + execution + .view() + .history() + .page(1, 100, true) + .unwrap() + .events(), + ) { + assert_eq!( + row.get("eventJson"), + Some(&json!(serde_json::to_string(event).unwrap())) + ); + assert_eq!( + row.get("timestamp"), + Some(&serde_json::to_value(event.timestamp).unwrap()) + ); + } + } + assert_eq!(first.revision().id(), captured.revision().id()); + assert_ne!(captured.revision().id(), second.revision().id()); + let first_result = output(captured.view().outcome().unwrap()); + let second_result = output(second.view().outcome().unwrap()); + assert_eq!(first_result.pointer("/Payload/computed"), Some(&json!(42))); + assert_eq!(second_result.pointer("/Payload/computed"), Some(&json!(44))); + for response in [first_result, second_result] { + let actual = response + .pointer("/Payload/request_id") + .and_then(Value::as_str) + .unwrap(); + assert_eq!(actual.len(), 36); + assert_eq!( + response + .pointer("/SdkResponseMetadata/RequestId") + .and_then(Value::as_str), + Some(actual) + ); + } + assert_ne!( + first_result.pointer("/Payload/request_id"), + second_result.pointer("/Payload/request_id") + ); +} + +async fn api_json( + service: &StepFunctionsHttpService, + action: &str, + input: Value, +) -> Result> { + let request = HttpRequest::builder() + .method("POST") + .uri("/") + .header("content-type", "application/x-amz-json-1.0") + .header("x-amz-target", format!("AWSStepFunctions.{action}")) + .body(Full::new(Bytes::from(serde_json::to_vec(&input)?)))?; + let response = service.handle(request).await; + let status = response.status(); + let body = response.into_body().collect().await?.to_bytes(); + let value = serde_json::from_slice(&body)?; + if status != StatusCode::OK { + return Err(RegistryError::Unavailable.into()); + } + Ok(value) +} +async fn run_http_registry_case( + service: &StepFunctionsHttpService, +) -> Result<(Value, Value, Value), Box> { + let created=api_json(service,"CreateStateMachine",json!({"name":"HttpNative","roleArn":"arn:aws:iam::000000000000:role/Workflow","definition":workflow(true).to_string()})).await?; + let started=api_json(service,"StartExecution",json!({"stateMachineArn":created["stateMachineArn"],"name":"computed","input":"{\"n\":21}"})).await?; + let absolute = deadline(); + let description = loop { + let description = api_json( + service, + "DescribeExecution", + json!({"executionArn":started["executionArn"]}), + ) + .await?; + if description["status"] != "RUNNING" { + break description; + } + if Instant::now() >= absolute { + return Err(RegistryError::Deadline.into()); + } + sleep(Duration::from_millis(10)).await; + }; + let history=api_json(service,"GetExecutionHistory",json!({"executionArn":started["executionArn"],"reverseOrder":true,"includeExecutionData":false,"maxResults":1})).await?; + api_json( + service, + "DeleteStateMachine", + json!({"stateMachineArn":created["stateMachineArn"]}), + ) + .await?; + let next=api_json(service,"GetExecutionHistory",json!({"executionArn":started["executionArn"],"reverseOrder":true,"includeExecutionData":false,"maxResults":1,"nextToken":history["nextToken"]})).await?; + Ok((description, history, next)) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_real_lambda_through_typed_http_api_and_paginated_history() { + let mut fixture = NativeFixture::new(&[("WORKFLOW_MODE", "1")], false).await; + let (port, mut bridge) = StepFunctionsLambdaBridge::new( + fixture.provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let (registry, mut owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + executions(port), + ExecutionLimits::default(), + ) + .unwrap(); + let (handler, mut api) = StepFunctionsApi::new( + TenantScope::default(), + TaskSupport::Lambda, + registry, + ApiLimits::default(), + ) + .unwrap(); + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into()).unwrap(), + true, + None, + ) + .unwrap(); + let service = StepFunctionsHttpService::new(handler, config); + let result = run_http_registry_case(&service).await; + let absolute = Instant::now() + Duration::from_secs(15); + let requests = api.shutdown(absolute).await; + let workflows = owner.shutdown(absolute).await; + let calls = bridge.shutdown(absolute).await; + if requests.is_err() || workflows.is_err() || calls.is_err() { + if let Some(directory) = fixture.directory.take() { + let _retained = directory.keep(); + } + } + fixture.cleanup_at(absolute).await; + assert!(requests.is_ok(), "API cleanup: {requests:?}"); + assert!(workflows.is_ok(), "registry cleanup: {workflows:?}"); + assert!(calls.is_ok(), "bridge cleanup: {calls:?}"); + let (description, history, next) = result.unwrap(); + assert_eq!(description["status"], "SUCCEEDED"); + let result: Value = serde_json::from_str(description["output"].as_str().unwrap()).unwrap(); + assert_eq!(result["Payload"]["computed"], 42); + assert_eq!( + result["Payload"]["request_id"], + result["SdkResponseMetadata"]["RequestId"] + ); + assert_eq!(result["Payload"]["request_id"].as_str().unwrap().len(), 36); + assert_eq!(history["events"][0]["type"], "ExecutionSucceeded"); + assert!( + history["events"][0]["executionSucceededEventDetails"] + .get("output") + .is_none() + ); + assert!( + next["events"][0]["id"].as_i64().unwrap() < history["events"][0]["id"].as_i64().unwrap() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_overlapping_parallel_functions_through_structured_owner() +-> Result<(), Box> { + let gate = tempfile::tempdir()?; + let release = gate.path().join("release"); + let release_text = release.to_str().ok_or("invalid gate path")?; + let consumed = gate.path().join("consumed"); + fs::create_dir(&consumed).await?; + let consumed_text = consumed.to_str().ok_or("invalid consumed marker path")?; + let mut fixture = NativeFixture::new( + &[ + ("WORKFLOW_MODE", "1"), + ("WORKFLOW_GATE_FILE", release_text), + ("WORKFLOW_INVOCATION_DIR", consumed_text), + ], + false, + ) + .await; + let result = async { + let definition = json!({ + "StartAt": "Parallel", + "States": {"Parallel": { + "Type": "Parallel", + "Branches": [workflow(false), workflow(false)], + "End": true + }} + }); + let compiled = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .with_state_support(StateSupport::Structured) + .compile(&serde_json::to_vec(&definition)?)?; + let input = Payload::from_value(&json!({"n": 21}))?; + let driver = ExecutionDriver::for_structured_owner( + compiled.machine().clone(), + input.clone(), + EvaluationContext::from(&input), + ClockSample::new(Instant::now().into_std(), SystemTime::now().into()), + ExecutionLimits::default(), + )?; + let mut handle = fixture.owner.start_structured(driver)?; + let markers = timeout_at(deadline(), consumed_invocations(&consumed)).await??; + fs::write(&release, b"release").await?; + let view = handle.wait(deadline()).await?; + Ok::<_, Box>((view, markers)) + } + .await; + let released = fs::write(&release, b"release").await; + fixture.cleanup().await; + released?; + let (view, markers) = result?; + let values = output(view.outcome().ok_or("outcome")?) + .as_array() + .ok_or("expected ordered array")?; + assert_eq!(values.len(), 2); + let returned: BTreeSet<_> = values + .iter() + .map(|value| { + Ok(( + value + .get("pid") + .and_then(Value::as_u64) + .ok_or("missing PID")?, + value + .get("request_id") + .and_then(Value::as_str) + .ok_or("missing request ID")? + .to_owned(), + )) + }) + .collect::>>()?; + assert_eq!(returned, markers); + for value in values { + assert_eq!(value.get("computed"), Some(&json!(42))); + assert_eq!(value.get("codeMarker"), Some(&json!("native-v1"))); + assert_ne!( + value.get("pid").and_then(Value::as_u64), + Some(u64::from(process::id())) + ); + } + assert_ne!( + values.first().and_then(|v| v.get("request_id")), + values.get(1).and_then(|v| v.get("request_id")) + ); + Ok(()) +} + +async fn consumed_invocations(directory: &Path) -> Result, Box> { + loop { + let mut entries = fs::read_dir(directory).await?; + let mut markers = BTreeSet::new(); + let mut count = 0; + while let Some(entry) = entries.next_entry().await? { + if entry + .path() + .extension() + .is_none_or(|extension| extension != "json") + { + continue; + } + count += 1; + if count > 2 || entry.metadata().await?.len() > 1024 { + return Err("unexpected consumed marker count or size".into()); + } + let marker: Value = serde_json::from_slice(&fs::read(entry.path()).await?)?; + let pid = marker + .get("pid") + .and_then(Value::as_u64) + .filter(|pid| *pid > 0) + .ok_or("invalid consumed PID")?; + let request_id = marker + .get("request_id") + .and_then(Value::as_str) + .filter(|id| id.len() == 36) + .ok_or("invalid consumed request ID")?; + markers.insert((pid, request_id.to_owned())); + } + if markers.len() == 2 { + let pids: BTreeSet<_> = markers.iter().map(|(pid, _)| *pid).collect(); + let requests: BTreeSet<_> = markers.iter().map(|(_, request)| request).collect(); + if pids.len() != 2 || requests.len() != 2 { + return Err("handlers did not consume distinct concurrent invocations".into()); + } + return Ok(markers); + } + sleep(Duration::from_millis(1)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_and_cancel_real_child_through_atomic_registry_port() +-> Result<(), Box> { + for cancel in [false, true] { + let directory = tempfile::tempdir()?; + let release = directory.path().join("release"); + let mut fixture = NativeFixture::new( + &[ + ("WORKFLOW_MODE", "1"), + ("WORKFLOW_GATE_FILE", release.to_str().ok_or("gate path")?), + ], + false, + ) + .await; + let runtime = ExecutionSupervisor::new( + RuntimeLimits::new(8, 1, 4)?, + HistoryLimits::default(), + 256 * 1024 * 1024, + Some(fixture.port.clone()), + )?; + let (registry, mut owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + )?; + let result = match timeout( + Duration::from_secs(30), + run_native_child_port(&fixture, ®istry, &release, cancel), + ) + .await + { + Ok(Ok(Some(pid))) => verify_absent_probe(pid).await, + Ok(Ok(None)) => Ok(()), + Ok(Err(error)) => Err(error), + Err(error) => Err(Box::::from(error)), + }; + let cleanup = owner + .shutdown(Instant::now() + Duration::from_secs(15)) + .await; + let released = fs::write(&release, b"release").await; + fixture.cleanup().await; + cleanup?; + released?; + result?; + } + Ok(()) +} +async fn child_port_machine( + registry: &Registry, + name: &str, + definition: &Value, +) -> Result> { + let compiled = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .compile(&serde_json::to_vec(definition)?)?; + let result = registry + .submit(RegistryCommand::Create( + name.parse()?, + PreparedDefinition::new(compiled.machine().clone()), + "arn:aws:iam::000000000000:role/Workflows".parse()?, + Tags::default(), + ))? + .wait(deadline()) + .await?; + match result { + RegistryResult::Machine(machine) => Ok(machine.arn().clone()), + _ => Err("machine".into()), + } +} +async fn run_native_child_port( + fixture: &NativeFixture, + registry: &Registry, + release: &Path, + cancel: bool, +) -> Result, Box> { + fixture.provider.update_function_configuration( + "Worker", + &UpdateFunctionConfigurationInput { + timeout: Some(900), + ..UpdateFunctionConfigurationInput::default() + }, + )?; + let parent = child_port_machine( + registry, + "PortParent", + &json!({"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":3600,"End":true}}}), + ) + .await?; + let target = child_port_machine(registry, "PortChild", &workflow(false)).await?; + let RegistryResult::Execution(parent) = registry + .submit(RegistryCommand::Start( + parent, + Some("parent".parse()?), + Payload::parse(b"{}")?, + None, + ))? + .wait(deadline()) + .await? + else { + return Err("parent".into()); + }; + let parameters = ChildStartParameters::parse( + &TenantScope::default(), + &Payload::from_value(&json!({"StateMachineArn":target.as_str(),"Input":{"n":21}}))?, + )?; + let mut submission = registry.child_port().start( + parent.child_parent(), + parameters, + ChildPattern::Sync2, + Instant::now() + Duration::from_secs(300), + )?; + submission.admitted(deadline()).await?; + let mut child = submission.into_execution()?; + let marker = child_port_marker(fixture).await?; + let pid = marker + .get("pid") + .and_then(Value::as_u64) + .filter(|pid| *pid > 1 && *pid != u64::from(process::id())) + .ok_or("handler PID")?; + let request = marker + .get("request_id") + .and_then(Value::as_str) + .filter(|id| id.len() == 36) + .ok_or("request ID")?; + if child.snapshot().handle().snapshot().outcome().is_some() { + return Err("child completed before consumed gate release".into()); + } + if cancel { + child.cancel(); + } else { + fs::write(release, b"release").await?; + } + child.wait(deadline()).await?; + child.wait_cleanup(deadline()).await?; + let result = child.result()?; + if cancel { + let failure = result.err().ok_or("cancelled child succeeded")?; + if failure.details().error() != Some("States.TaskFailed") { + return Err("child cancellation error class".into()); + } + let cause: Value = serde_json::from_str(failure.details().cause().ok_or("cause")?)?; + if cause.get("Status").and_then(Value::as_str) != Some("ABORTED") { + return Err("child was not aborted".into()); + } + return Ok(Some(pid)); + } + let result = result.map_err(|_| "child failed")?; + let output = result.value().get("Output").ok_or("Output")?; + if output.get("computed") != Some(&json!(42)) + || output.get("pid").and_then(Value::as_u64) != Some(pid) + || output.get("request_id").and_then(Value::as_str) != Some(request) + { + return Err("child result did not match consumed Native invocation".into()); + } + Ok(None) +} + +async fn child_port_marker(fixture: &NativeFixture) -> Result> { + let marker = fixture + .directory + .as_ref() + .ok_or("directory")? + .path() + .join("invocation.json"); + timeout_at(deadline(), async { + loop { + match fs::read(&marker).await { + Ok(bytes) => { + if bytes.len() > 1024 { + return Err("marker size".into()); + } + if let Ok(value) = serde_json::from_slice::(&bytes) { + return Ok::<_, Box>(value); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + sleep(Duration::from_millis(1)).await; + } + }) + .await? +} + +async fn verify_absent_probe(pid: u64) -> Result<(), Box> { + let child = Command::new("/bin/kill") + .arg("-0") + .arg(pid.to_string()) + .env_clear() + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn()?; + let (mut child, status) = reap_probe(child, deadline()).await?; + let stderr = child.stderr.take().ok_or("probe stderr")?; + let mut text = Vec::new(); + timeout_at(deadline(), stderr.take(1025).read_to_end(&mut text)).await??; + if status.success() + || text.len() > 1024 + || !String::from_utf8_lossy(&text).contains("No such process") + { + return Err("owned cancellation did not retire consumed PID before global cleanup".into()); + } + Ok(()) +} + +async fn reap_probe( + mut child: Child, + until: Instant, +) -> Result<(Child, ExitStatus), Box> { + let status = match timeout_at(until, child.wait()).await { + Ok(Ok(status)) => status, + observed => { + let saved: Box = match observed { + Ok(Err(error)) => Box::new(error), + Err(error) => Box::new(error), + Ok(Ok(_)) => return Err("probe wait invariant".into()), + }; + let killed = child.start_kill(); + match timeout_at(deadline(), child.wait()).await { + Ok(Ok(_)) => {} + failed => { + // Unproven helper retirement retains the exact process handle. + let _retained = Box::leak(Box::new(child)); + return Err(format!("PID probe retirement unproven: {failed:?}").into()); + } + } + killed?; + return Err(saved); + } + }; + if Instant::now() >= until { + return Err("PID probe observation expired after joined exit".into()); + } + Ok((child, status)) +} + +#[tokio::test] +#[ignore = "requires Unix process tools; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_kill_and_reap_timed_out_owned_probe_before_returning_error() +-> Result<(), Box> { + let child = Command::new("/bin/sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let pid = child.id().ok_or("owned probe PID")?; + let result = reap_probe(child, Instant::now() + Duration::from_millis(10)).await; + verify_absent_probe(u64::from(pid)).await?; + let error = result + .err() + .ok_or("slow probe did not report its observation timeout")?; + if error.downcast_ref::().is_none() { + return Err("probe failed without observing its execution timeout".into()); + } + Ok(()) +} + +mod callback; + +mod registered_callback; + +async fn run_restored_native_case( + source: &Registry, + source_owner: &mut RegistrySupervisor, + target: &Registry, + marker: &Path, +) -> Result<(), Box> { + let prepared = PreparedDefinition::new( + AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .compile(&serde_json::to_vec(&workflow(true))?)? + .machine() + .clone(), + ); + let mut create = source.submit(RegistryCommand::Create( + "NativeWorkflow".parse()?, + prepared, + "arn:aws:iam::000000000000:role/Workflow".parse()?, + Tags::default(), + ))?; + let RegistryResult::Machine(machine) = create.wait(deadline()).await? else { + return Err("missing source machine".into()); + }; + let mut receipt = source.submit(RegistryCommand::Start( + machine.arn().clone(), + Some("before".parse()?), + Payload::parse(br#"{"n":21}"#)?, + None, + ))?; + let RegistryResult::Execution(original) = receipt.wait(deadline()).await? else { + return Err("missing original execution".into()); + }; + let original_view = original.handle().wait(deadline()).await?; + source_owner.shutdown(deadline()).await?; + let before = fs::read(marker).await?; + let document = source.export_snapshot(SnapshotLimits::default())?; + let mut bytes = Vec::new(); + ciborium::into_writer(&document, &mut bytes)?; + let decoded = ciborium::from_reader(bytes.as_slice())?; + let prepared = target.prepare_snapshot(decoded, SnapshotLimits::default())?; + let mut receipt = target.restore_snapshot(prepared)?; + if !matches!(receipt.wait(deadline()).await?, RegistryResult::Restored) { + return Err("missing restore acknowledgement".into()); + } + let mut read = target.submit(RegistryCommand::Execution(original.arn().clone()))?; + let RegistryResult::Execution(saved) = read.wait(deadline()).await? else { + return Err("missing restored execution".into()); + }; + let unchanged = fs::read(marker).await?; + if before != unchanged + || output(saved.view().outcome().ok_or("missing old outcome")?).pointer("/Payload/computed") + != Some(&json!(42)) + || serde_json::to_value(original_view.history().page(1, 1000, true)?.events())? + != serde_json::to_value(saved.view().history().page(1, 1000, true)?.events())? + { + return Err("restore replayed work or changed output/history".into()); + } + let mut receipt = target.submit(RegistryCommand::Start( + machine.arn().clone(), + Some("after".parse()?), + Payload::parse(br#"{"n":22}"#)?, + None, + ))?; + let RegistryResult::Execution(next) = receipt.wait(deadline()).await? else { + return Err("missing new execution".into()); + }; + let completed = next.handle().wait(deadline()).await?; + if output(completed.outcome().ok_or("missing new outcome")?).pointer("/Payload/computed") + != Some(&json!(44)) + || before == fs::read(marker).await? + { + return Err("restored machine did not execute a new real Lambda call".into()); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_restore_without_replay_and_execute_new_lambda_requests() { + let mut fixture = NativeFixture::new(&[("WORKFLOW_MODE", "1")], false).await; + let (first_port, mut first_bridge) = StepFunctionsLambdaBridge::new( + fixture.provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let (second_port, mut second_bridge) = StepFunctionsLambdaBridge::new( + fixture.provider.clone(), + TenantScope::default(), + BridgeLimits::default(), + ) + .unwrap(); + let (source, mut source_owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + executions(first_port), + ExecutionLimits::default(), + ) + .unwrap(); + let (target, mut target_owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + executions(second_port), + ExecutionLimits::default(), + ) + .unwrap(); + let marker = fixture + .directory + .as_ref() + .unwrap() + .path() + .join("invocation.json"); + let result = run_restored_native_case(&source, &mut source_owner, &target, &marker).await; + // Join both independent registry/bridge owners even when any assertion path fails. + let absolute = Instant::now() + Duration::from_secs(15); + let first = source_owner.shutdown(absolute).await; + let second = target_owner.shutdown(absolute).await; + let first_calls = first_bridge.shutdown(absolute).await; + let second_calls = second_bridge.shutdown(absolute).await; + if first.is_err() || second.is_err() || first_calls.is_err() || second_calls.is_err() { + if let Some(directory) = fixture.directory.take() { + let _retained = directory.keep(); + } + } + fixture.cleanup().await; + assert!(first.is_ok(), "source cleanup: {first:?}"); + assert!(second.is_ok(), "target cleanup: {second:?}"); + assert!( + first_calls.is_ok(), + "source bridge cleanup: {first_calls:?}" + ); + assert!( + second_calls.is_ok(), + "target bridge cleanup: {second_calls:?}" + ); + assert!(result.is_ok(), "restore case: {result:?}"); +} diff --git a/apps/rustack/src/stepfunctions_bridge/tests/native/callback.rs b/apps/rustack/src/stepfunctions_bridge/tests/native/callback.rs new file mode 100644 index 00000000..88dee727 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/tests/native/callback.rs @@ -0,0 +1,356 @@ +//! Real Native invocation composition; ASL callback activation is separately gated. + +use rustack_stepfunctions_core::{ + CallbackAttempt, CallbackCommand, CallbackDispatch, CallbackInvocation, + CallbackInvocationMetadata, CallbackLimits, CallbackOutcome, CallbackRegistry, CallbackTask, + DriverStep, FrameCallId, FrameCapacity, FrameLimits, FrameScheduler, FrameUpdate, + LambdaTaskCommand, SystemUuidSource, TaskCall, TaskEvent, TaskFailure, TaskSettlement, +}; +use uuid::Uuid; + +use super::*; +use crate::stepfunctions_bridge::{mapping::Mapped, worker}; + +fn observed_dispatch() -> Result<(CallbackDispatch, FrameCallId), Box> { + let mut definition = workflow(true); + definition["States"]["Call"]["Resource"] = + json!("arn:aws:states:::lambda:invoke.waitForTaskToken"); + definition["States"]["Call"]["Parameters"]["Payload"]["token.$"] = json!("$$.Task.Token"); + let machine = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .with_callback_support(true) + .compile(&serde_json::to_vec(&definition)?)? + .machine() + .clone(); + let now = ClockSample::new(Instant::now().into_std(), SystemTime::now().into()); + let input = Payload::from_value(&json!({"n":21}))?; + let driver = ExecutionDriver::for_structured_owner( + machine, + input.clone(), + EvaluationContext::from_value(&json!({}))?, + now, + ExecutionLimits::default(), + )?; + let mut frames = FrameScheduler::new( + driver, + Arc::new(FrameCapacity::default()), + FrameLimits::default(), + )?; + for _ in 0..20 { + if let FrameUpdate::Transition(t) = frames.advance(now, &mut SystemUuidSource::default()) { + if let DriverStep::Callback(effect) = t.step() { + return Ok((effect.clone(), t.call().ok_or("call")?)); + } + } + } + Err("missing deferred callback".into()) +} + +fn command( + effect: &CallbackDispatch, + call: FrameCallId, + task: &mut CallbackTask, +) -> Result> { + let (dispatch, history) = + effect.prepare_with_source(task.token(), &mut SystemUuidSource::default())?; + let scheduled = effect.scheduled_event(&history, SystemTime::now().into())?; + if !history.redacted() || serde_json::to_string(&scheduled)?.contains(task.token().expose()) { + return Err("unsafe callback scheduling projection".into()); + } + Ok(LambdaTaskCommand::observed_callback( + effect.machine().clone(), + dispatch, + call, + task.take_invocation_reporter()?, + )?) +} + +async fn success(registry: &CallbackRegistry, task: &CallbackTask) -> Result<(), Box> { + registry + .submit( + &TenantScope::default(), + CallbackCommand::success(task.token().expose().parse()?, br#"{"external":42}"#)?, + )? + .wait(deadline()) + .await?; + Ok(()) +} +async fn result(call: &mut TaskCall) -> Result> { + loop { + match call.next().await? { + TaskEvent::Completed(result) => return result.map_err(|_| "invocation failed".into()), + TaskEvent::Started | TaskEvent::Submitted(_) => {} + _ => return Err("unexpected task event".into()), + } + } +} +async fn run( + fixture: &NativeFixture, + registry: &Registry, + callbacks: &CallbackRegistry, + release: &Path, + kind: u8, +) -> Result, Box> { + fixture.provider.update_function_configuration( + "Worker", + &UpdateFunctionConfigurationInput { + timeout: Some(900), + ..UpdateFunctionConfigurationInput::default() + }, + )?; + let arn = child_port_machine( + registry, + "CallbackParent", + &json!({"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":3600,"End":true}}}), + ) + .await?; + let RegistryResult::Execution(parent) = registry + .submit(RegistryCommand::Start( + arn, + Some("parent".parse()?), + Payload::parse(b"{}")?, + None, + ))? + .wait(deadline()) + .await? + else { + return Err("parent".into()); + }; + let (effect, call_id) = observed_dispatch()?; + let mut mint = callbacks.register(CallbackAttempt::new( + &parent, + call_id, + Instant::from_std(effect.deadline()), + effect.heartbeat(), + )?)?; + mint.admitted(deadline()).await?; + let mut task = mint.into_task()?; + let secret = task.token().expose().to_owned(); + let command = command(&effect, call_id, &mut task)?; + let mut call = fixture + .port + .start(command) + .map_err(|_| "bridge admission")?; + let marker = child_port_marker(fixture).await?; + let pid = marker.get("pid").and_then(Value::as_u64).ok_or("pid")?; + let request = marker + .get("request_id") + .and_then(Value::as_str) + .ok_or("request")?; + if !matches!(task.snapshot().as_ref(), CallbackOutcome::Invoking) + || task.invocation().is_some() + || call.settlement() != TaskSettlement::Pending + { + return Err("gated invocation completed".into()); + } + if kind == 3 { + task.revoke(); + call.cancel(); + timeout_at(deadline(), call.settled()).await??; + if !matches!( + task.wait(deadline()).await?.as_ref(), + CallbackOutcome::Revoked + ) { + return Err("callback cancellation".into()); + } + return Ok(Some(pid)); + } + if kind != 1 { + success(callbacks, &task).await?; + } + if !matches!(task.snapshot().as_ref(), CallbackOutcome::Invoking) { + return Err("early callback bypassed invocation".into()); + } + fs::write(release, b"release").await?; + let report = task.wait_invocation(deadline()).await?; + if kind == 2 { + if report.failure().and_then(|f| f.details().error()) != Some("TestError") + || !matches!( + task.wait(deadline()).await?.as_ref(), + CallbackOutcome::InvocationFailed(_) + ) + { + return Err("function failure hidden by callback".into()); + } + timeout_at(deadline(), call.settled()).await??; + return Ok(None); + } + finish( + callbacks, &mut task, &mut call, report, request, &secret, kind, + ) + .await?; + Ok(None) +} +async fn finish( + callbacks: &CallbackRegistry, + task: &mut CallbackTask, + call: &mut TaskCall, + report: Arc, + request: &str, + secret: &str, + kind: u8, +) -> Result<(), Box> { + let metadata = report.metadata().ok_or("actual metadata")?; + if metadata.request_id().to_string() != request || metadata.executed_version() != "$LATEST" { + return Err("invented invocation metadata".into()); + } + if kind == 1 { + // Waiting is published independently; repeat heartbeat as an actor barrier. + callbacks + .submit( + &TenantScope::default(), + CallbackCommand::heartbeat(task.token().expose().parse()?), + )? + .wait(deadline()) + .await?; + if !matches!(task.snapshot().as_ref(), CallbackOutcome::Waiting { .. }) { + return Err("not waiting after real response".into()); + } + success(callbacks, task).await?; + } + let outcome = task.wait(deadline()).await?; + if !matches!(outcome.as_ref(),CallbackOutcome::Succeeded(p) if p.value()==&json!({"external":42})) + { + return Err("synchronous body replaced callback output".into()); + } + let output = timeout_at(deadline(), result(call)).await??; + timeout_at(deadline(), call.settled()).await??; + if output.value().get("Payload").is_some() + || output.value()["SdkResponseMetadata"]["RequestId"] != request + || format!("{report:?}{output:?}").contains(secret) + { + return Err("unsafe callback metadata".into()); + } + Ok(()) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_observe_callbacks_before_and_after_real_return_prioritize_failure_and_reap_cancelled_process() +-> Result<(), Box> { + for kind in 0..4 { + let directory = tempfile::tempdir()?; + let release = directory.path().join("release"); + let mut vars = vec![ + ("WORKFLOW_MODE", "1"), + ( + "WORKFLOW_GATE_FILE", + release.to_str().ok_or("release path")?, + ), + ]; + if kind == 2 { + vars.push(("FAIL_MODE", "panic")); + } + let mut fixture = NativeFixture::new(&vars, false).await; + let (registry, mut owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + executions(fixture.port.clone()), + ExecutionLimits::default(), + )?; + let (callbacks, mut callback_owner) = + CallbackRegistry::new(TenantScope::default(), CallbackLimits::default())?; + let outcome = timeout_at( + deadline(), + run(&fixture, ®istry, &callbacks, &release, kind), + ) + .await; + let probe = match &outcome { + Ok(Ok(Some(pid))) => verify_absent_probe(*pid).await, + _ => Ok(()), + }; + let released = fs::write(&release, b"release").await; + let callback_join = callback_owner.shutdown(deadline()).await; + let joined = owner.shutdown(deadline()).await; + if callback_join.is_err() || joined.is_err() { + if let Some(dir) = fixture.directory.take() { + let _retained = dir.keep(); + } + } + fixture.cleanup().await; + released?; + callback_join?; + joined?; + probe?; + outcome??; + } + Ok(()) +} + +#[tokio::test] +async fn test_should_publish_return_before_bridge_settlement_and_preserve_report_on_quarantine() +-> Result<(), Box> { + let runtime = ExecutionSupervisor::new( + RuntimeLimits::default(), + HistoryLimits::default(), + 256 * 1024 * 1024, + None, + )?; + let (registry, mut owner) = Registry::new( + TenantScope::default(), + RegistryLimits::default(), + runtime, + ExecutionLimits::default(), + )?; + let (callbacks, mut callback_owner) = + CallbackRegistry::new(TenantScope::default(), CallbackLimits::default())?; + let outcome = async { + let arn = child_port_machine( + ®istry, + "ReportParent", + &json!({"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":3600,"End":true}}}), + ) + .await?; + let RegistryResult::Execution(parent) = registry + .submit(RegistryCommand::Start( + arn, + Some("parent".parse()?), + Payload::parse(b"{}")?, + None, + ))? + .wait(deadline()) + .await? + else { + return Err::<(), Box>("parent".into()); + }; + let (_, call_id) = observed_dispatch()?; + let mut mint = + callbacks.register(CallbackAttempt::new(&parent, call_id, deadline(), None)?)?; + mint.admitted(deadline()).await?; + let mut task = mint.into_task()?; + let mut reporter = Some(task.take_invocation_reporter()?); + let metadata = CallbackInvocationMetadata::new(Uuid::new_v4(), "$LATEST")?; + let mut mapped = Mapped { + result: Ok(metadata.payload()?), + diagnostic: None, + }; + let (mut call, mut responder) = TaskCall::channel(); + responder.started()?; + worker::report(&mut reporter, &mut mapped, Some(metadata), Instant::now()); + let report = task.wait_invocation(deadline()).await?; + if !report.succeeded() || call.settlement() != TaskSettlement::Pending { + return Err("report depended on settlement".into()); + } + success(&callbacks, &task).await?; + task.wait(deadline()).await?; + let retained = responder.quarantine(TaskFailure::service_exception()); + if call.settlement() != TaskSettlement::Quarantined + || !Arc::ptr_eq(&report, &task.wait_invocation(deadline()).await?) + { + return Err("cleanup rewrote invocation report".into()); + } + if timeout(Duration::from_millis(1), call.settled()) + .await + .is_ok() + { + return Err("quarantine fabricated clean".into()); + } + retained.settle(); + call.settled().await?; + Ok(()) + } + .await; + let a = callback_owner.shutdown(deadline()).await; + let b = owner.shutdown(deadline()).await; + a?; + b?; + outcome +} diff --git a/apps/rustack/src/stepfunctions_bridge/tests/native/registered_callback.rs b/apps/rustack/src/stepfunctions_bridge/tests/native/registered_callback.rs new file mode 100644 index 00000000..0422dfd5 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/tests/native/registered_callback.rs @@ -0,0 +1,277 @@ +//! Actual registered ASL callback owner → Native Lambda; API activation is separate. + +use rustack_stepfunctions_core::{ + CallbackCommand, CallbackLimits, CallbackRegistry, CallbackToken, ExecutionView, + LambdaTaskCommand, LambdaTaskPort, TaskCall, TaskFailure, +}; +use tokio::sync::mpsc; + +use super::*; + +/// Explicit bounded harness credential transport; never a service history/log sink. +#[derive(Debug)] +struct Capture { + inner: Arc, + tokens: mpsc::Sender, +} +impl LambdaTaskPort for Capture { + fn start(&self, command: LambdaTaskCommand) -> Result { + let token = command + .dispatch() + .parameters() + .value() + .pointer("/Payload/token") + .and_then(Value::as_str) + .ok_or_else(TaskFailure::service_exception)? + .parse() + .map_err(|_| TaskFailure::service_exception())?; + self.tokens + .try_send(token) + .map_err(|_| TaskFailure::service_exception())?; + self.inner.start(command) + } +} +async fn machine(registry: &Registry) -> Result> { + let source = json!({"StartAt":"Call","States":{"Call":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken","TimeoutSeconds":300,"Parameters":{"FunctionName":"Worker","Payload":{"n.$":"$.n","retryCount.$":"$$.State.RetryCount","token.$":"$$.Task.Token"}},"End":true}}}); + let compiled = AslCompiler::new(TenantScope::default(), TaskSupport::Lambda) + .with_callback_support(true) + .compile(&serde_json::to_vec(&source)?)?; + let RegistryResult::Machine(machine) = registry + .submit(RegistryCommand::Create( + "NativeCallback".parse()?, + PreparedDefinition::new(compiled.machine().clone()), + "arn:aws:iam::000000000000:role/Worker".parse()?, + Tags::default(), + ))? + .wait(deadline()) + .await? + else { + return Err("machine".into()); + }; + Ok(machine.arn().clone()) +} +async fn run( + fixture: &NativeFixture, + registry: &Registry, + callbacks: &CallbackRegistry, + tokens: &mut mpsc::Receiver, + release: &Path, + kind: u8, +) -> Result, Box> { + fixture.provider.update_function_configuration( + "Worker", + &UpdateFunctionConfigurationInput { + timeout: Some(900), + ..UpdateFunctionConfigurationInput::default() + }, + )?; + let machine = machine(registry).await?; + let RegistryResult::Execution(execution) = registry + .submit(RegistryCommand::Start( + machine, + Some("run".parse()?), + Payload::parse(br#"{"n":21}"#)?, + None, + ))? + .wait(deadline()) + .await? + else { + return Err("execution".into()); + }; + let token = timeout_at(deadline(), tokens.recv()) + .await? + .ok_or("token capture")?; + let marker = child_port_marker(fixture).await?; + let pid = marker + .get("pid") + .and_then(Value::as_u64) + .filter(|pid| *pid > 1 && *pid != u64::from(process::id())) + .ok_or("consumed PID")?; + let request = marker + .get("request_id") + .and_then(Value::as_str) + .ok_or("consumed request")?; + if execution.handle().snapshot().outcome().is_some() { + return Err("callback completed before consumed gate release".into()); + } + if kind == 3 { + execution + .handle() + .request_stop(TaskFailure::new("Stopped", None).map_err(|_| "stop failure")?); + let view = execution.handle().wait(deadline()).await?; + if !matches!( + view.outcome().map(Arc::as_ref), + Some(ExecutionOutcome::Aborted(_)) + ) { + return Err("callback stop outcome".into()); + } + execution.handle().wait_cleanup(deadline()).await?; + if callbacks + .submit( + &TenantScope::default(), + CallbackCommand::success(token.expose().parse()?, b"true")?, + )? + .wait(deadline()) + .await + .is_ok() + { + return Err("stopped token remained live".into()); + } + return Ok(Some(pid)); + } + if kind != 1 { + send(callbacks, &token).await?; + } + if execution.handle().snapshot().outcome().is_some() { + return Err("early callback bypassed actual Native return".into()); + } + fs::write(release, b"release").await?; + if kind == 1 { + wait_submitted(&execution).await?; + if execution.handle().snapshot().outcome().is_some() { + return Err("Lambda body completed callback Task".into()); + } + send(callbacks, &token).await?; + } + let view = execution.handle().wait(deadline()).await?; + execution.handle().wait_cleanup(deadline()).await?; + verify(&view, &token, request, kind)?; + Ok(None) +} +async fn send(callbacks: &CallbackRegistry, token: &CallbackToken) -> Result<(), Box> { + callbacks + .submit( + &TenantScope::default(), + CallbackCommand::success(token.expose().parse()?, br#"{"external":42}"#)?, + )? + .wait(deadline()) + .await?; + Ok(()) +} +async fn wait_submitted(execution: &ExecutionSnapshot) -> Result<(), Box> { + timeout_at(deadline(), async { + loop { + let view = execution.handle().snapshot(); + if view + .history() + .page(1, 1000, true)? + .events() + .iter() + .any(|event| event.task_submitted_event_details.is_some()) + { + return Ok::<_, Box>(()); + } + sleep(Duration::from_millis(1)).await; + } + }) + .await? +} +fn verify( + view: &ExecutionView, + token: &CallbackToken, + request: &str, + kind: u8, +) -> Result<(), Box> { + if kind == 2 { + if !matches!(view.outcome().map(Arc::as_ref),Some(ExecutionOutcome::Failed(f)) if f.error()==Some("TestError")) + { + return Err("Native failure hidden by provisional callback".into()); + } + } else if !matches!(view.outcome().map(Arc::as_ref),Some(ExecutionOutcome::Succeeded(p)) if p.value()==&json!({"external":42})) + { + return Err("callback output was replaced by function body".into()); + } + let page = view.history().page(1, 1000, true)?; + let events = page.events(); + if serde_json::to_string(events)?.contains(token.expose()) { + return Err("plaintext token in callback history".into()); + } + let scheduled = events + .iter() + .find_map(|e| e.task_scheduled_event_details.as_ref()) + .ok_or("scheduled history")?; + let parameters: Value = serde_json::from_str(&scheduled.parameters)?; + if parameters.pointer("/Payload/token") != Some(&Value::Null) { + return Err("callback history provenance missing".into()); + } + if kind != 2 { + let submitted = events + .iter() + .find_map(|e| e.task_submitted_event_details.as_ref()) + .ok_or("submitted history")?; + let output: Value = serde_json::from_str(submitted.output.as_deref().ok_or("metadata")?)?; + if output + .pointer("/SdkResponseMetadata/RequestId") + .and_then(Value::as_str) + != Some(request) + || output.get("Payload").is_some() + || output.get("ExecutedVersion").and_then(Value::as_str) != Some("$LATEST") + || submitted.output_details.as_ref().and_then(|d| d.truncated) != Some(true) + { + return Err("actual Native metadata or callback truncation mismatch".into()); + } + } + Ok(()) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the actual host bootstrap; run make test-stepfunctions-bridge-native"] +async fn native_e2e_should_execute_registered_asl_callbacks_and_join_consumed_native_cancellation() +-> Result<(), Box> { + for kind in 0..4 { + let gate = tempfile::tempdir()?; + let release = gate.path().join("release"); + let mut vars = vec![ + ("WORKFLOW_MODE", "1"), + ("WORKFLOW_GATE_FILE", release.to_str().ok_or("gate path")?), + ]; + if kind == 2 { + vars.push(("FAIL_MODE", "panic")); + } + let mut fixture = NativeFixture::new(&vars, false).await; + let (sender, mut tokens) = mpsc::channel(8); + let port = Arc::new(Capture { + inner: fixture.port.clone(), + tokens: sender, + }); + let (callbacks, mut callback_owner) = + CallbackRegistry::new(TenantScope::default(), CallbackLimits::default())?; + let mut runtime = ExecutionSupervisor::new( + RuntimeLimits::new(8, 1, 2)?, + HistoryLimits::default(), + 256 * 1024 * 1024, + Some(port), + )?; + runtime.install_callbacks(callbacks.clone())?; + let (registry, mut owner) = Registry::new_with_state_support( + TenantScope::default(), + RegistryLimits::default(), + runtime, + ExecutionLimits::new(Duration::from_secs(300), 10000)?, + StateSupport::Structured, + )?; + let result = timeout_at( + Instant::now() + Duration::from_secs(30), + run(&fixture, ®istry, &callbacks, &mut tokens, &release, kind), + ) + .await; + let probe = match &result { + Ok(Ok(Some(pid))) => verify_absent_probe(*pid).await, + _ => Ok(()), + }; + let released = fs::write(&release, b"release").await; + let workflow = owner.shutdown(deadline()).await; + let callback = callback_owner.shutdown(deadline()).await; + if workflow.is_err() || callback.is_err() { + if let Some(dir) = fixture.directory.take() { + let _retained = dir.keep(); + } + } + fixture.cleanup().await; + released?; + workflow?; + callback?; + probe?; + result??; + } + Ok(()) +} diff --git a/apps/rustack/src/stepfunctions_bridge/worker.rs b/apps/rustack/src/stepfunctions_bridge/worker.rs new file mode 100644 index 00000000..913fab26 --- /dev/null +++ b/apps/rustack/src/stepfunctions_bridge/worker.rs @@ -0,0 +1,360 @@ +//! Retained start/result/cleanup sequence for exactly one actual provider call. + +use std::{future::Future, pin::pin, sync::Arc}; + +use rustack_lambda_core::{ + error::LambdaServiceError, + executor::{ExecutorError, InvokeResponse}, + invocation::InvocationHandle, +}; +use rustack_stepfunctions_core::{ + CallbackError, CallbackInvocationMetadata, CallbackInvocationReporter, LambdaTaskCommand, + SystemUuidSource, TaskFailure, TaskResponder, +}; +use tokio::{sync::watch, task::JoinSet, time::Instant}; + +use super::{ + BridgeError, Shared, Uncertain, WorkerExit, + mapping::{self, CallPlan, Mapped, ResponseMode, ServiceTarget}, + signal, +}; + +async fn observe( + future: F, + core: &mut watch::Receiver, + manager: &mut watch::Receiver, + relay: &watch::Sender, +) -> F::Output { + let mut future = pin!(future); + loop { + let core_stopped = signal(core); + let manager_stopped = signal(manager); + if core_stopped || manager_stopped { + relay.send_replace(true); + } + tokio::select! { + biased; + _=core.changed(),if !core_stopped=>{}, + _=manager.changed(),if !manager_stopped=>{}, + result=&mut future=>return result, + } + } +} +fn checked(mapped: Result) -> Mapped { + mapped.unwrap_or_else(|error| Mapped { + result: Err(TaskFailure::service_exception()), + diagnostic: Some(error), + }) +} +#[tracing::instrument(name = "stepfunctions.lambda", skip_all)] +pub(super) async fn invoke( + shared: Arc, + mut command: LambdaTaskCommand, + mut responder: TaskResponder, + mut manager: watch::Receiver, +) -> WorkerExit { + let mut core = responder.cancellation(); + let (relay, cancellation) = watch::channel(signal(&core) || signal(&manager)); + let plan = CallPlan::compile(&command); + let mut reporter = match command.take_callback_reporter() { + Ok(reporter) => reporter, + Err(error) => { + responder.finish(Err(TaskFailure::service_exception())); + return WorkerExit { + uncertain: None, + diagnostic: Some(Arc::new(BridgeError::Callback(Arc::new(error)))), + }; + } + }; + let plan = match plan { + Ok(plan) => plan, + Err(error) => { + return finish_unsubmitted( + checked(mapping::parameters(error)), + &mut reporter, + responder, + ); + } + }; + let integration_started = matches!(plan.mode, ResponseMode::Optimized); + if integration_started && responder.started().is_err() { + relay.send_replace(true); + } + let start = shared.context.provider.start_invocation_owned( + &plan.function, + plan.qualifier.as_deref(), + plan.payload.original_bytes(), + Some(plan.deadline), + cancellation, + ); + let submission = observe(start, &mut core, &mut manager, &relay).await; + let mut submission = match submission { + Ok(submission) => submission, + Err(error) => { + let mapped = map_service( + error, + ServiceTarget::new(&plan, command.machine().scope()), + &mut core, + &mut manager, + &relay, + ) + .await; + return finish_unsubmitted(mapped, &mut reporter, responder); + } + }; + let admitted = observe(submission.admitted(), &mut core, &mut manager, &relay).await; + let failed_admission = admitted.is_err(); + let mut handle = submission.into_handle(); + if failed_admission { + handle.cancel(); + } + let (mut mapped, metadata, observed) = match admitted { + Ok(()) => { + if !integration_started && responder.started().is_err() { + relay.send_replace(true); + } + receive_response(&mut handle, plan.mode, &mut core, &mut manager, &relay).await + } + Err(error) => ( + map_service( + error, + ServiceTarget::new(&plan, command.machine().scope()), + &mut core, + &mut manager, + &relay, + ) + .await, + None, + Instant::now(), + ), + }; + report(&mut reporter, &mut mapped, metadata, observed); + let exit = cleanup( + &shared, + handle, + responder, + mapped, + &mut core, + &mut manager, + &relay, + ) + .await; + // Keep the external cancellation relay alive through the owned cleanup join. + drop(relay); + exit +} + +fn finish_unsubmitted( + mut mapped: Mapped, + reporter: &mut Option, + responder: TaskResponder, +) -> WorkerExit { + report(reporter, &mut mapped, None, Instant::now()); + responder.finish(mapped.result); + WorkerExit { + uncertain: None, + diagnostic: mapped.diagnostic.map(Arc::new), + } +} + +async fn map_service( + error: LambdaServiceError, + target: ServiceTarget, + core: &mut watch::Receiver, + manager: &mut watch::Receiver, + relay: &watch::Sender, +) -> Mapped { + let mut workers = JoinSet::new(); + workers + .spawn_blocking(move || mapping::service(error, &target, &mut SystemUuidSource::default())); + match observe(workers.join_next(), core, manager, relay).await { + Some(Ok(mapped)) => checked(mapped), + Some(Err(error)) => checked(Err(BridgeError::Worker(error))), + None => checked(Err(BridgeError::Unavailable)), + } +} + +async fn receive_response( + handle: &mut InvocationHandle, + mode: ResponseMode, + core: &mut watch::Receiver, + manager: &mut watch::Receiver, + relay: &watch::Sender, +) -> (Mapped, Option, Instant) { + match observe(handle.wait_response(), core, manager, relay).await { + Ok(response) => { + let observed = Instant::now(); + let request_id = handle.id(); + let (mapped, metadata) = map_response( + CallbackInvocationMetadata::new(request_id, &response.executed_version), + response, + mode, + request_id.to_string(), + core, + manager, + relay, + ) + .await; + (mapped, metadata, observed) + } + Err(error) => (checked(mapping::executor(error)), None, Instant::now()), + } +} + +async fn map_response( + metadata: Result, + response: InvokeResponse, + mode: ResponseMode, + request_id: String, + core: &mut watch::Receiver, + manager: &mut watch::Receiver, + relay: &watch::Sender, +) -> (Mapped, Option) { + let mut workers = JoinSet::new(); + workers.spawn_blocking(move || match mode { + ResponseMode::ObservedCallback => mapping::observed_response(&response, metadata), + ResponseMode::Direct | ResponseMode::Optimized => mapping::response( + &response, + matches!(mode, ResponseMode::Optimized), + &request_id, + ) + .map(|mapped| (mapped, None)), + }); + match observe(workers.join_next(), core, manager, relay).await { + Some(Ok(Ok(mapped))) => mapped, + Some(Ok(Err(error))) => (checked(Err(error)), None), + Some(Err(error)) => (checked(Err(BridgeError::Worker(error))), None), + None => (checked(Err(BridgeError::Unavailable)), None), + } +} + +pub(super) fn report( + reporter: &mut Option, + mapped: &mut Mapped, + metadata: Option, + observed: Instant, +) { + let Some(reporter) = reporter.as_mut() else { + return; + }; + let result = match &mapped.result { + Err(failure) => Err(failure.clone()), + Ok(_) => metadata.ok_or_else(TaskFailure::service_exception), + }; + if let Err(error) = reporter.complete_at(result, observed) { + mapped.result = Err(TaskFailure::service_exception()); + mapped.diagnostic = Some(BridgeError::Callback(Arc::new(error))); + } +} + +async fn cleanup( + shared: &Shared, + mut handle: InvocationHandle, + responder: TaskResponder, + mapped: Mapped, + core: &mut watch::Receiver, + manager: &mut watch::Receiver, + relay: &watch::Sender, +) -> WorkerExit { + if signal(core) || signal(manager) { + relay.send_replace(true); + handle.cancel(); + } + let cleanup = match Instant::now().checked_add(shared.context.limits.cleanup) { + Some(deadline) => observe(handle.wait_settlement(deadline), core, manager, relay).await, + None => Err(ExecutorError::CleanupFailed( + "Invalid cleanup deadline".into(), + )), + }; + match cleanup { + Ok(()) => { + responder.finish(mapped.result); + WorkerExit { + uncertain: None, + diagnostic: mapped.diagnostic.map(Arc::new), + } + } + Err(error) => { + handle.cancel(); + let failure = mapped + .result + .err() + .unwrap_or_else(TaskFailure::service_exception); + let receipt = responder.quarantine(failure); + let diagnostic = Arc::new(BridgeError::Executor(Arc::new(error))); + shared.diagnostic.send_replace(Some(diagnostic.clone())); + match observe(handle.wait_cleanup(), core, manager, relay).await { + Ok(()) => { + receipt.settle(); + WorkerExit { + uncertain: None, + diagnostic: Some(diagnostic), + } + } + Err(error) => WorkerExit { + uncertain: Some(Uncertain { + _handle: handle, + _receipt: receipt, + }), + diagnostic: Some(Arc::new(BridgeError::Executor(Arc::new(error)))), + }, + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{sync::mpsc, time::Duration}; + + use super::*; + + #[tokio::test] + async fn test_should_relay_caller_and_manager_cancellation_without_dropping_mapping_join() { + for manager_cancel in [false, true] { + let (caller, mut core) = watch::channel(false); + let (stop, mut manager) = watch::channel(false); + let (relay, mut relayed) = watch::channel(false); + let (entered, waiting) = tokio::sync::oneshot::channel(); + let (release, held) = mpsc::sync_channel(1); + let mut mapping = JoinSet::new(); + mapping.spawn_blocking(move || { + let _ = entered.send(()); + held.recv_timeout(Duration::from_secs(5)).unwrap(); + 42 + }); + let mut owners = JoinSet::new(); + owners.spawn(async move { + observe(mapping.join_next(), &mut core, &mut manager, &relay).await + }); + tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .unwrap() + .unwrap(); + if manager_cancel { + stop.send_replace(true); + } else { + caller.send_replace(true); + } + tokio::time::timeout(Duration::from_secs(5), relayed.wait_for(|v| *v)) + .await + .unwrap() + .unwrap(); + assert!( + owners.try_join_next().is_none(), + "mapping ownership ended before its CPU job" + ); + release.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), owners.join_next()) + .await + .unwrap() + .unwrap() + .unwrap() + .unwrap() + .unwrap(), + 42 + ); + } + } +} diff --git a/apps/rustack/src/workflows.rs b/apps/rustack/src/workflows.rs new file mode 100644 index 00000000..c434436f --- /dev/null +++ b/apps/rustack/src/workflows.rs @@ -0,0 +1,406 @@ +//! Actual gateway workflow composition and unique resumable shutdown ownership. + +use std::{error::Error, fmt, sync::Arc, time::Duration}; + +use anyhow::{Context, Result}; +use rustack_cli::stepfunctions_api::{ApiError, ApiLimits, ApiSupervisor, StepFunctionsApi}; +#[cfg(feature = "lambda")] +use rustack_cli::stepfunctions_bridge::{ + BridgeError, BridgeLimits, LambdaBridgeSupervisor, StepFunctionsLambdaBridge, +}; +use rustack_core::settings::RuntimeBudgets; +#[cfg(feature = "lambda")] +use rustack_lambda_core::{executor::ExecutorBackend, provider::RustackLambda}; +#[cfg(feature = "lambda")] +use rustack_stepfunctions_core::LambdaTaskPort; +use rustack_stepfunctions_core::{ + CallbackError, CallbackLimits, CallbackRegistry, CallbackSupervisor, ExecutionLimits, + ExecutionSupervisor, FrameCapacity, FrameLimits, HistoryLimits, Registry, RegistryCapacities, + RegistryError, RegistryLimits, RegistrySupervisor, RuntimeLimits, StateSupport, TaskSupport, + TenantScope, +}; +use serde_json::{Value, json}; +use tokio::time::Instant; + +use crate::snapshot::workflows::{ArchiveSupervisor, WorkflowArchive}; + +/// Shared health reads never acquire business admission or unique controllers. +#[derive(Debug, Clone)] +pub(crate) struct WorkflowStatus { + api: Arc, + registry: Registry, + budgets: RuntimeBudgets, + archive: WorkflowArchive, + callbacks: Option, + #[cfg(feature = "lambda")] + port: Option>, +} +impl WorkflowStatus { + pub(crate) fn is_ready(&self) -> bool { + let ready = self.archive.is_ready() + && self.api.is_ready() + && self.registry.status().ready + && self + .callbacks + .as_ref() + .is_none_or(|callbacks| callbacks.status().ready); + #[cfg(feature = "lambda")] + let ready = ready && self.port.as_ref().is_none_or(|port| port.is_ready()); + ready + } + pub(crate) fn capabilities(&self) -> Value { + #[cfg(feature = "lambda")] + let lambda = self.port.is_some(); + #[cfg(not(feature = "lambda"))] + let lambda = false; + let limits = json!({ + "apiAccepted": self.budgets.workflow_api_accepted, + "apiWorkers": self.budgets.workflow_api_workers, + "paginationBytes": self.budgets.workflow_pagination_bytes, + "paginationTokens": self.budgets.workflow_pagination_tokens, + "machines": self.budgets.workflow_registry_machines, + "versions": self.budgets.workflow_registry_versions, + "records": self.budgets.workflow_registry_records, + "commands": self.budgets.workflow_registry_commands, + "nameTombstones": self.budgets.workflow_name_tombstones, + "retentionSeconds": self.budgets.workflow_retention_seconds, + "openExecutions": self.budgets.workflow_open_executions, + "evaluations": self.budgets.workflow_evaluations, + "dispatches": self.budgets.workflow_dispatches, + "maxDurationSeconds": self.budgets.workflow_max_duration_seconds, + "transitions": self.budgets.workflow_transitions, + "frames": self.budgets.workflow_frames, + "framesPerExecution": self.budgets.workflow_frames_per_execution, + "resultBytes": self.budgets.workflow_result_bytes, + "resultBytesPerExecution": self.budgets.workflow_result_bytes_per_execution, + "historyEvents": 25000, "historyBytes": self.budgets.workflow_history_bytes, + "totalHistoryBytes": self.budgets.workflow_total_history_bytes, + "snapshotBytes": self.budgets.workflow_snapshot_bytes, + "lambdaAcceptedCalls": self.budgets.workflow_lambda_accepted_calls, + "lambdaCleanupSeconds": self.budgets.workflow_lambda_cleanup_seconds, + "drainSeconds": self.budgets.workflow_drain_seconds, + "callbackTokens": self.budgets.workflow_callback_tokens, + "callbackTombstones": self.budgets.workflow_callback_tombstones, + "callbackCommands": self.budgets.workflow_callback_commands, + "callbackWorkers": self.budgets.workflow_callback_workers, + "callbackRetentionSeconds": self.budgets.workflow_retention_seconds + }); + json!({ + "type": "STANDARD", "queryLanguage": "JSONPath", + "stateTypes": ["Pass", "Choice", "Wait", "Task", "Succeed", "Fail", "Parallel", "Map"], + "inlineMapMaxConcurrency": 40, + "lambdaRequestResponse": lambda, "nestedSync": true, "callbacks": self.registry.callback_support(), + "snapshot": "resources-and-history", "snapshotResume": false, "paginationEpoch": "fresh-on-restart", "dockerQualified": false, + "limits": limits + }) + } +} +#[derive(Debug, Clone, Copy)] +enum Phase { + Accepting, + Api(Instant), + ClosingApi, + Grace(Instant), + Registry, + Callbacks, + Bridge, + Done, +} +/// Kept by main, outside `Arc`; never protected by a mutex. +#[derive(Debug)] +pub(crate) struct WorkflowSupervisor { + api: ApiSupervisor, + archive: ArchiveSupervisor, + registry: RegistrySupervisor, + status: WorkflowStatus, + callbacks: Option, + #[cfg(feature = "lambda")] + bridge: Option, + phase: Phase, + failure: Option, +} +#[derive(Clone)] +struct WorkflowFailure(Arc); +impl fmt::Display for WorkflowFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("workflow cleanup failed") + } +} +impl fmt::Debug for WorkflowFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} +impl Error for WorkflowFailure { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.0.as_ref().as_ref()) + } +} +pub(crate) struct WorkflowRuntime { + pub(crate) api: Arc, + pub(crate) archive: WorkflowArchive, + pub(crate) status: WorkflowStatus, + pub(crate) owner: WorkflowSupervisor, +} +impl WorkflowRuntime { + pub(crate) fn new( + scope: TenantScope, + budgets: RuntimeBudgets, + #[cfg(feature = "lambda")] provider: Option>, + ) -> Result { + budgets.validate().context("validate workflow budgets")?; + #[cfg(feature = "lambda")] + let (port, bridge) = + match provider.filter(|p| p.config().executor != ExecutorBackend::Disabled) { + Some(provider) => { + let (port, owner) = StepFunctionsLambdaBridge::new( + provider, + scope.clone(), + BridgeLimits::new( + budgets.workflow_lambda_accepted_calls, + Duration::from_secs(budgets.workflow_lambda_cleanup_seconds), + )?, + )?; + (Some(port), Some(owner)) + } + None => (None, None), + }; + #[cfg(feature = "lambda")] + let task_port = port + .as_ref() + .map(|port| port.clone() as Arc); + #[cfg(not(feature = "lambda"))] + let task_port = None; + let tasks = if task_port.is_some() { + TaskSupport::Lambda + } else { + TaskSupport::Disabled + }; + let mut runtime = ExecutionSupervisor::new_with_frames( + RuntimeLimits::new( + budgets.workflow_open_executions, + budgets.workflow_dispatches, + budgets.workflow_evaluations, + )?, + HistoryLimits::new(25000, budgets.workflow_history_bytes)?, + budgets.workflow_total_history_bytes, + task_port, + Arc::new(FrameCapacity::new( + budgets.workflow_frames, + budgets.workflow_result_bytes, + )?), + FrameLimits::new( + budgets.workflow_frames_per_execution, + budgets.workflow_result_bytes_per_execution, + )?, + )?; + let (callbacks, callback_owner) = + callback_runtime(&scope, &budgets, &mut runtime, tasks)?.unzip(); + let (registry, owner) = Registry::new_with_state_support( + scope.clone(), + RegistryLimits::new( + RegistryCapacities::new( + budgets.workflow_registry_machines, + budgets.workflow_registry_versions, + budgets.workflow_registry_records, + budgets.workflow_name_tombstones, + )?, + Duration::from_secs(budgets.workflow_retention_seconds), + budgets.workflow_registry_commands, + )?, + runtime, + ExecutionLimits::new( + Duration::from_secs(budgets.workflow_max_duration_seconds), + budgets.workflow_transitions, + )?, + StateSupport::Structured, + )?; + let (api, api_owner) = api_runtime(scope, tasks, registry.clone(), &budgets)?; + let (archive, archive_owner) = WorkflowArchive::new( + registry.clone(), + budgets.workflow_snapshot_bytes, + Duration::from_secs(budgets.request_seconds), + )?; + let status = WorkflowStatus { + archive: archive.clone(), + api: api.clone(), + registry, + budgets, + callbacks, + #[cfg(feature = "lambda")] + port, + }; + Ok(Self { + api, + archive, + status: status.clone(), + owner: WorkflowSupervisor { + api: api_owner, + archive: archive_owner, + registry: owner, + status, + callbacks: callback_owner, + #[cfg(feature = "lambda")] + bridge, + phase: Phase::Accepting, + failure: None, + }, + }) + } +} +fn api_runtime( + scope: TenantScope, + tasks: TaskSupport, + registry: Registry, + budgets: &RuntimeBudgets, +) -> Result<(Arc, ApiSupervisor)> { + Ok(StepFunctionsApi::new( + scope, + tasks, + registry, + ApiLimits::new( + budgets.workflow_api_accepted, + budgets.workflow_api_workers, + budgets.workflow_pagination_tokens, + budgets.workflow_pagination_bytes, + )?, + )?) +} +fn callback_runtime( + scope: &TenantScope, + budgets: &RuntimeBudgets, + runtime: &mut ExecutionSupervisor, + tasks: TaskSupport, +) -> Result> { + if tasks != TaskSupport::Lambda { + return Ok(None); + } + let (callbacks, owner) = CallbackRegistry::new( + scope.clone(), + CallbackLimits::new( + budgets.workflow_callback_tokens, + budgets.workflow_callback_tombstones, + budgets.workflow_callback_commands, + budgets.workflow_callback_workers, + Duration::from_secs(budgets.workflow_retention_seconds), + )?, + )?; + runtime.install_callbacks(callbacks.clone())?; + Ok(Some((callbacks, owner))) +} + +impl WorkflowSupervisor { + /// Resume the same controllers; a dropped/expired observer never restarts grace. + pub(crate) async fn quiesce(&mut self, deadline: Instant) -> Result<()> { + if matches!(self.phase, Phase::Accepting) { + let cutoff = + Instant::now() + Duration::from_secs(self.status.budgets.workflow_drain_seconds); + self.api.close(); + self.phase = Phase::Api(cutoff); + } + loop { + match self.phase { + Phase::Accepting => anyhow::bail!("workflow shutdown phase invariant failed"), + Phase::Api(_) | Phase::ClosingApi => { + let cutoff = if let Phase::Api(cutoff) = self.phase { + Some(cutoff) + } else { + None + }; + match self.api.shutdown(deadline).await { + Ok(()) => {} + Err(ApiError::Deadline) => { + anyhow::bail!("workflow API shutdown observation expired") + } + Err(error) => self.fail(error.into()), + } + self.phase = cutoff.map_or(Phase::Registry, Phase::Grace); + } + Phase::Grace(cutoff) => { + match self.status.registry.wait_idle(cutoff.min(deadline)).await { + Err(RegistryError::Deadline) if Instant::now() < cutoff => { + anyhow::bail!("workflow grace observation expired") + } + Ok(()) | Err(RegistryError::Deadline) => {} + Err(error) => self.fail(error.into()), + } + self.phase = Phase::Registry; + } + Phase::Registry => { + match self.registry.shutdown(deadline).await { + Ok(()) => {} + Err(RegistryError::Deadline) => { + anyhow::bail!("workflow registry shutdown observation expired") + } + Err(error) => self.fail(error.into()), + } + self.phase = Phase::Callbacks; + } + Phase::Callbacks => { + if let Some(callbacks) = &mut self.callbacks { + match callbacks.shutdown(deadline).await { + Ok(()) => {} + Err(CallbackError::Deadline) => { + anyhow::bail!("workflow callback shutdown observation expired") + } + Err(error) => self.fail(error.into()), + } + } + self.phase = Phase::Bridge; + } + Phase::Bridge => { + #[cfg(feature = "lambda")] + if let Some(bridge) = &mut self.bridge { + if let Err(error) = bridge.shutdown(deadline).await { + if matches!(*error, BridgeError::Deadline) { + anyhow::bail!("workflow bridge shutdown observation expired"); + } + self.fail(anyhow::anyhow!(error)); + } + } + self.phase = Phase::Done; + } + Phase::Done => { + return match &self.failure { + Some(error) => Err(error.clone().into()), + None => Ok(()), + }; + } + } + } + } + /// Final cleanup skips grace and signals every owner on observer expiry. + pub(crate) async fn shutdown(&mut self, deadline: Instant) -> Result<()> { + self.api.close(); + self.phase = match self.phase { + Phase::Accepting | Phase::Api(_) => Phase::ClosingApi, + Phase::Grace(_) => Phase::Registry, + phase => phase, + }; + let archive = self.archive.shutdown(deadline).await; + // A lost archive observer may still own preparation or an accepted commit. + // Attempt the remaining controllers even when its final join fails. + let result = self.quiesce(deadline).await.and(archive); + if result.is_err() { + // No successful consistency cut is claimed; cancel retained owners + // before main attempts provider destruction under the same deadline. + self.registry.close(); + if let Some(callbacks) = &self.callbacks { + callbacks.close(); + } + #[cfg(feature = "lambda")] + if let Some(bridge) = &self.bridge { + bridge.close(); + } + } + result + } + fn fail(&mut self, error: anyhow::Error) { + if self.failure.is_none() { + self.failure = Some(WorkflowFailure(Arc::new(error))); + } + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/apps/rustack/src/workflows/tests.rs b/apps/rustack/src/workflows/tests.rs new file mode 100644 index 00000000..7be2e8fd --- /dev/null +++ b/apps/rustack/src/workflows/tests.rs @@ -0,0 +1,528 @@ +//! Real controller composition, observer loss and admission-independent health. + +use bytes::Bytes; +use http::{Request, StatusCode}; +use http_body_util::{BodyExt, Full}; +use rustack_stepfunctions_core::{ExecutionOutcome, RegistryCommand, RegistryResult}; +use rustack_stepfunctions_http::{RequestScope, StepFunctionsHttpConfig, StepFunctionsHttpService}; +use serde_json::json; + +use super::*; + +pub(crate) fn runtime(budgets: RuntimeBudgets) -> Result { + WorkflowRuntime::new( + TenantScope::default(), + budgets, + #[cfg(feature = "lambda")] + None, + ) +} +pub(crate) async fn request( + api: Arc, + operation: &str, + body: Value, +) -> Result<(StatusCode, Value)> { + let config = StepFunctionsHttpConfig::new( + RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into())?, + true, + None, + )?; + let service = StepFunctionsHttpService::new(api, config); + let request = Request::builder() + .method("POST") + .uri("/") + .header("x-amz-target", format!("AWSStepFunctions.{operation}")) + .header("content-type", "application/x-amz-json-1.0") + .body(Full::new(Bytes::from(serde_json::to_vec(&body)?)))?; + let response = service.handle(request).await; + let status = response.status(); + let body = response.into_body().collect().await?.to_bytes(); + Ok(( + status, + if body.is_empty() { + Value::Null + } else { + serde_json::from_slice(&body)? + }, + )) +} +pub(crate) async fn start( + runtime: &WorkflowRuntime, + seconds: u32, +) -> Result { + let definition = + json!({"StartAt":"Wait", "States":{"Wait":{"Type":"Wait","Seconds":seconds,"End":true}}}) + .to_string(); + let (status, created) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"Machine","roleArn":"arn:aws:iam::000000000000:role/Test","definition":definition})).await?; + assert_eq!(status, StatusCode::OK, "{created}"); + let (status, started) = request(runtime.api.clone(), "StartExecution", json!({"stateMachineArn":created["stateMachineArn"],"name":"Run","input":"{\"answer\":42}"})).await?; + assert_eq!(status, StatusCode::OK, "{started}"); + let arn = started["executionArn"] + .as_str() + .context("execution ARN")? + .parse()?; + let mut receipt = runtime + .status + .registry + .submit(RegistryCommand::Execution(arn))?; + let RegistryResult::Execution(snapshot) = receipt + .wait(Instant::now() + Duration::from_secs(5)) + .await? + else { + anyhow::bail!("unexpected capture") + }; + Ok(snapshot) +} +#[tokio::test] +async fn test_should_finish_accepted_execution_during_grace_before_registry_close() -> Result<()> { + let mut runtime = runtime(RuntimeBudgets::default())?; + let snapshot = start(&runtime, 1).await?; + assert!(runtime.status.is_ready()); + assert_eq!(runtime.status.registry.status().owned_executions, 1); + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(10)) + .await?; + assert!(!runtime.status.is_ready()); + let complete = snapshot + .wait(Instant::now() + Duration::from_secs(1)) + .await?; + assert!(matches!( + complete.view().outcome().map(Arc::as_ref), + Some(ExecutionOutcome::Succeeded(_)) + )); + assert_eq!( + complete.view().cleanup(), + rustack_stepfunctions_core::CleanupState::Clean + ); + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(1)) + .await?; + Ok(()) +} +#[tokio::test] +async fn test_should_resume_same_grace_after_cancelled_shutdown_observer() -> Result<()> { + let mut runtime = runtime(RuntimeBudgets::default())?; + let snapshot = start(&runtime, 3600).await?; + tokio::time::pause(); + let deadline = Instant::now() + Duration::from_secs(100); + assert!( + tokio::time::timeout(Duration::from_millis(1), runtime.owner.quiesce(deadline)) + .await + .is_err() + ); + let Phase::Grace(cutoff) = runtime.owner.phase else { + anyhow::bail!("grace not reached") + }; + assert!(!runtime.status.is_ready()); + tokio::time::advance(cutoff.saturating_duration_since(Instant::now())).await; + runtime.owner.quiesce(deadline).await?; + let complete = snapshot.wait(deadline).await?; + let Some(ExecutionOutcome::Aborted(failure)) = complete.view().outcome().map(Arc::as_ref) + else { + anyhow::bail!("execution was not aborted") + }; + assert_eq!(failure.details().error(), Some("Rustack.Shutdown")); + let page = complete.view().history().page(1, 100, true)?; + assert!(page.events().iter().any(|event| { + event + .execution_aborted_event_details + .as_ref() + .is_some_and(|details| details.error.as_deref() == Some("Rustack.Shutdown")) + })); + Ok(()) +} +#[tokio::test] +async fn test_should_preserve_grace_cutoff_after_expired_observer() -> Result<()> { + let mut runtime = runtime(RuntimeBudgets::default())?; + let snapshot = start(&runtime, 3600).await?; + tokio::time::pause(); + assert!( + runtime + .owner + .quiesce(Instant::now() + Duration::from_millis(10)) + .await + .is_err() + ); + let Phase::Grace(cutoff) = runtime.owner.phase else { + anyhow::bail!("grace not retained") + }; + tokio::time::advance(Duration::from_secs(6)).await; + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(10)) + .await?; + assert!(Instant::now() >= cutoff); + assert!(matches!( + snapshot + .wait(Instant::now() + Duration::from_secs(1)) + .await? + .view() + .outcome() + .map(Arc::as_ref), + Some(ExecutionOutcome::Aborted(_)) + )); + Ok(()) +} +#[tokio::test] +async fn test_should_read_health_when_registry_command_admission_is_full() -> Result<()> { + let mut budgets = RuntimeBudgets::default(); + budgets.workflow_registry_commands = 1; + let mut runtime = runtime(budgets)?; + let mut receipt = runtime.status.registry.submit(RegistryCommand::Machines)?; + assert!(matches!( + runtime.status.registry.submit(RegistryCommand::Machines), + Err(RegistryError::Capacity) + )); + assert!(runtime.status.is_ready()); + receipt + .wait(Instant::now() + Duration::from_secs(5)) + .await?; + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + Ok(()) +} +#[tokio::test] +async fn test_should_report_pure_profile_and_reject_lambda_without_actual_port() -> Result<()> { + let mut runtime = runtime(RuntimeBudgets::default())?; + let capabilities = runtime.status.capabilities(); + assert_eq!(capabilities["lambdaRequestResponse"], false); + assert_eq!(capabilities["snapshot"], "resources-and-history"); + assert_eq!(capabilities["dockerQualified"], false); + assert_eq!(capabilities["limits"]["apiWorkers"], 8); + assert_eq!(capabilities["nestedSync"], true); + assert_eq!(capabilities["callbacks"], false); + assert!(runtime.status.callbacks.is_none()); + let child = json!({"StartAt":"D","States":{"D":{"Type":"Pass","End":true}}}); + let (status, created) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"PureChild","definition":child.to_string(),"roleArn":"arn:aws:iam::000000000000:role/Test"})).await?; + assert_eq!(status, StatusCode::OK, "{created}"); + let parent = json!({"StartAt":"T","States":{"T":{"Type":"Task","Resource":"arn:aws:states:::states:startExecution.sync:2","Parameters":{"StateMachineArn":created["stateMachineArn"],"Input":{"answer":42}},"End":true}}}); + let (status, created) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"PureParent","definition":parent.to_string(),"roleArn":"arn:aws:iam::000000000000:role/Test"})).await?; + assert_eq!(status, StatusCode::OK, "{created}"); + let (status, started) = request( + runtime.api.clone(), + "StartExecution", + json!({"stateMachineArn":created["stateMachineArn"],"input":"{}"}), + ) + .await?; + assert_eq!(status, StatusCode::OK, "{started}"); + let arn = started["executionArn"] + .as_str() + .context("nested execution ARN")? + .parse()?; + let RegistryResult::Execution(snapshot) = runtime + .status + .registry + .submit(RegistryCommand::Execution(arn))? + .wait(Instant::now() + Duration::from_secs(5)) + .await? + else { + anyhow::bail!("nested execution missing"); + }; + let complete = snapshot + .wait(Instant::now() + Duration::from_secs(5)) + .await?; + assert!( + matches!(complete.view().outcome().map(Arc::as_ref), Some(ExecutionOutcome::Succeeded(output)) if output.value()["Output"] == json!({"answer":42})) + ); + let definition = json!({"StartAt":"Call","States":{"Call":{"Type":"Task","Resource":"arn:aws:lambda:us-east-1:000000000000:function:Fn","End":true}}}).to_string(); + let (status, body) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"Task","definition":definition,"roleArn":"arn:aws:iam::000000000000:role/Test"})).await?; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + Ok(()) +} +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_install_only_registered_selected_actual_lambda_provider() -> Result<()> { + use rustack_lambda_core::config::LambdaConfig; + + let config = + LambdaConfig::from_lookup(|key| (key == "LAMBDA_EXECUTOR").then(|| "native".to_owned()))?; + let provider = Arc::new(RustackLambda::new(config)); + let mut runtime = WorkflowRuntime::new( + TenantScope::default(), + RuntimeBudgets::default(), + Some(provider.clone()), + )?; + assert_eq!(runtime.status.capabilities()["lambdaRequestResponse"], true); + assert_eq!(runtime.status.capabilities()["callbacks"], true); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackTokens"], + 4096 + ); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackTombstones"], + 100_000 + ); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackCommands"], + 1024 + ); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackWorkers"], + 8 + ); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackRetentionSeconds"], + 86_400 + ); + assert!(runtime.status.is_ready()); + let callback = json!({"StartAt":"T","States":{"T":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken","Parameters":{"FunctionName":"Missing","Payload":{"token.$":"$$.Task.Token"}},"End":true}}}); + assert_eq!(request(runtime.api.clone(),"CreateStateMachine",json!({"name":"Callback","roleArn":"arn:aws:iam::000000000000:role/Test","definition":callback.to_string()})).await?.0,StatusCode::OK); + assert_eq!( + request( + runtime.api.clone(), + "SendTaskHeartbeat", + json!({"taskToken":"a".repeat(64)}) + ) + .await? + .1["__type"], + "InvalidToken" + ); + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + provider.quiesce(Duration::from_secs(5)).await?; + provider.shutdown().await; + assert!( + !runtime + .status + .callbacks + .as_ref() + .context("callback root")? + .status() + .ready + ); + Ok(()) +} + +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_reject_unqualified_manual_docker_provider() { + use rustack_lambda_core::config::LambdaConfig; + + let provider = Arc::new(RustackLambda::new(LambdaConfig { + executor: ExecutorBackend::Docker, + ..LambdaConfig::default() + })); + assert!( + WorkflowRuntime::new( + TenantScope::default(), + RuntimeBudgets::default(), + Some(provider) + ) + .is_err() + ); +} + +#[test] +fn test_should_retain_typed_cleanup_source_and_redact_debug() { + let failure = WorkflowFailure(Arc::new(anyhow::anyhow!("sensitive-marker"))); + assert!(!format!("{failure:?}").contains("sensitive-marker")); + assert!(failure.source().is_some()); +} + +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_observe_callback_failure_without_business_admission_and_resume_same_callback_join() +-> Result<()> { + use rustack_lambda_core::config::LambdaConfig; + + let config = + LambdaConfig::from_lookup(|key| (key == "LAMBDA_EXECUTOR").then(|| "native".to_owned()))?; + let provider = Arc::new(RustackLambda::new(config)); + let mut budgets = RuntimeBudgets::default(); + budgets.workflow_registry_commands = 1; + budgets.workflow_callback_tokens = 2; + budgets.workflow_callback_tombstones = 3; + budgets.workflow_callback_commands = 4; + budgets.workflow_callback_workers = 1; + let mut runtime = + WorkflowRuntime::new(TenantScope::default(), budgets, Some(provider.clone()))?; + let callbacks = runtime + .status + .callbacks + .clone() + .context("callback handle")?; + let receipt = runtime.status.registry.submit(RegistryCommand::Machines)?; + assert!(runtime.status.is_ready()); + runtime + .owner + .callbacks + .as_ref() + .context("callback controller")? + .close(); + assert!(!callbacks.status().ready); + assert!(!runtime.status.is_ready()); + assert_eq!(runtime.status.capabilities()["limits"]["callbackTokens"], 2); + assert_eq!( + runtime.status.capabilities()["limits"]["callbackTombstones"], + 3 + ); + assert_eq!( + request( + runtime.api.clone(), + "SendTaskHeartbeat", + json!({"taskToken":"a".repeat(64)}) + ) + .await? + .1["__type"], + "ServiceUnavailable" + ); + drop(receipt); + let deadline = Instant::now() + Duration::from_secs(5); + runtime.owner.api.shutdown(deadline).await?; + runtime.owner.registry.shutdown(deadline).await?; + runtime.owner.phase = Phase::Callbacks; + assert!(runtime.owner.quiesce(Instant::now()).await.is_err()); + assert!(matches!(runtime.owner.phase, Phase::Callbacks)); + runtime.owner.quiesce(deadline).await?; + assert!(matches!(runtime.owner.phase, Phase::Done)); + provider.quiesce(Duration::from_secs(5)).await?; + provider.shutdown().await; + Ok(()) +} + +#[cfg(feature = "lambda")] +#[tokio::test] +async fn test_should_signal_callback_root_when_final_shutdown_observation_expires() -> Result<()> { + use rustack_lambda_core::config::LambdaConfig; + + let config = + LambdaConfig::from_lookup(|key| (key == "LAMBDA_EXECUTOR").then(|| "native".to_owned()))?; + let provider = Arc::new(RustackLambda::new(config)); + let mut runtime = WorkflowRuntime::new( + TenantScope::default(), + RuntimeBudgets::default(), + Some(provider.clone()), + )?; + let callbacks = runtime + .status + .callbacks + .clone() + .context("callback handle")?; + assert!(runtime.owner.shutdown(Instant::now()).await.is_err()); + assert!(!callbacks.status().ready); + assert!(!runtime.status.is_ready()); + assert!(!matches!( + runtime.owner.phase, + Phase::Accepting | Phase::Done + )); + runtime + .owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await?; + assert!(matches!(runtime.owner.phase, Phase::Done)); + provider.quiesce(Duration::from_secs(5)).await?; + provider.shutdown().await; + Ok(()) +} + +#[tokio::test] +async fn test_should_skip_grace_on_final_cleanup_and_cancel_before_provider_destruction() +-> Result<()> { + let mut budgets = RuntimeBudgets::default(); + budgets.workflow_drain_seconds = 60; + let mut runtime = runtime(budgets)?; + let snapshot = start(&runtime, 3600).await?; + runtime + .owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await?; + assert!(matches!(runtime.owner.phase, Phase::Done)); + let complete = snapshot + .wait(Instant::now() + Duration::from_secs(1)) + .await?; + assert!( + matches!(complete.view().outcome().map(Arc::as_ref), Some(ExecutionOutcome::Aborted(failure)) if failure.details().error() == Some("Rustack.Shutdown")) + ); + Ok(()) +} + +#[tokio::test] +async fn test_should_compile_validate_update_and_execute_structured_profile_through_real_api() +-> Result<()> { + let mut budgets = RuntimeBudgets::default(); + budgets.workflow_frames = 16; + budgets.workflow_frames_per_execution = 8; + budgets.workflow_result_bytes = 2 * 1024 * 1024; + budgets.workflow_result_bytes_per_execution = 1024 * 1024; + let mut runtime = runtime(budgets)?; + let result = async { + let definition = json!({"StartAt":"M","States":{"M":{"Type":"Map","ItemProcessor":{"StartAt":"D","States":{"D":{"Type":"Pass","End":true}}},"End":true}}}).to_string(); + let (status, validated) = request(runtime.api.clone(), "ValidateStateMachineDefinition", json!({"definition":definition})).await?; + assert_eq!(status, StatusCode::OK, "{validated}"); + assert_eq!(validated["result"], "OK"); + let (status, created) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"Structured","roleArn":"arn:aws:iam::000000000000:role/Test","definition":definition})).await?; + assert_eq!(status, StatusCode::OK, "{created}"); + let definition = json!({"StartAt":"P","States":{"P":{"Type":"Parallel","Branches":[serde_json::from_str::(&definition)?,serde_json::from_str::(&definition)?],"End":true}}}).to_string(); + let (status, updated) = request(runtime.api.clone(), "UpdateStateMachine", json!({"stateMachineArn":created["stateMachineArn"],"definition":definition})).await?; + assert_eq!(status, StatusCode::OK, "{updated}"); + let (status, started) = request(runtime.api.clone(), "StartExecution", json!({"stateMachineArn":created["stateMachineArn"],"input":"[1,null,3]"})).await?; + assert_eq!(status, StatusCode::OK, "{started}"); + let arn = started["executionArn"].as_str().context("execution ARN")?.parse()?; + let RegistryResult::Execution(snapshot) = runtime.status.registry.submit(RegistryCommand::Execution(arn))?.wait(Instant::now()+Duration::from_secs(5)).await? else { anyhow::bail!("missing execution"); }; + let complete = snapshot.wait(Instant::now()+Duration::from_secs(5)).await?; + assert!(matches!(complete.view().outcome().map(Arc::as_ref),Some(ExecutionOutcome::Succeeded(value)) if value.value()==&json!([[1,null,3],[1,null,3]]))); + let capabilities = runtime.status.capabilities(); + assert_eq!(capabilities["stateTypes"].as_array().context("states")?.len(), 8); + assert_eq!(capabilities["inlineMapMaxConcurrency"], 40); + assert_eq!(capabilities["limits"]["frames"], 16); + assert_eq!(capabilities["limits"]["framesPerExecution"], 8); + assert_eq!(capabilities["limits"]["resultBytes"], 2*1024*1024); + assert_eq!(capabilities["limits"]["resultBytesPerExecution"], 1024*1024); + let invalid = json!({"StartAt":"M","States":{"M":{"Type":"Map","ItemProcessor":{"ProcessorConfig":{"Mode":"DISTRIBUTED"},"StartAt":"D","States":{"D":{"Type":"Succeed"}}},"End":true}}}).to_string(); + let (_, validation) = request(runtime.api.clone(), "ValidateStateMachineDefinition", json!({"definition":invalid})).await?; + assert_eq!(validation["result"], "FAIL"); + Ok(()) + }.await; + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(10)) + .await?; + result +} + +#[tokio::test] +async fn test_should_enforce_configured_frame_and_result_limits_in_public_execution_view() +-> Result<()> { + for (frames, bytes, expected) in [ + (1, 1024 * 1024, "Rustack.FrameLimitExceeded"), + (8, 4096, "Rustack.ResultLimitExceeded"), + ] { + let mut budgets = RuntimeBudgets::default(); + budgets.workflow_frames = frames; + budgets.workflow_frames_per_execution = frames; + budgets.workflow_result_bytes = bytes; + budgets.workflow_result_bytes_per_execution = bytes; + let mut runtime = runtime(budgets)?; + let result = async { + let definition = json!({"StartAt":"P","States":{"P":{"Type":"Parallel","Branches":[{"StartAt":"D","States":{"D":{"Type":"Pass","Result":"x".repeat(3000),"End":true}}}],"End":true}}}).to_string(); + let (status, created) = request(runtime.api.clone(), "CreateStateMachine", json!({"name":"Quota","roleArn":"arn:aws:iam::000000000000:role/Test","definition":definition})).await?; + assert_eq!(status, StatusCode::OK, "{created}"); + let (status, started) = request(runtime.api.clone(), "StartExecution", json!({"stateMachineArn":created["stateMachineArn"],"name":"Run"})).await?; + assert_eq!(status, StatusCode::OK, "{started}"); + let arn = started["executionArn"].as_str().context("ARN")?.parse()?; + let RegistryResult::Execution(snapshot) = runtime.status.registry.submit(RegistryCommand::Execution(arn))?.wait(Instant::now()+Duration::from_secs(5)).await? else { anyhow::bail!("missing execution"); }; + snapshot.wait(Instant::now()+Duration::from_secs(5)).await?; + let (status, described) = request(runtime.api.clone(), "DescribeExecution", json!({"executionArn":started["executionArn"]})).await?; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!(described["status"], "FAILED"); + assert_eq!(described["error"], expected); + Ok(()) + }.await; + runtime + .owner + .quiesce(Instant::now() + Duration::from_secs(5)) + .await?; + result?; + } + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup.rs b/apps/rustack/tests/docker_startup.rs new file mode 100644 index 00000000..c29f0d2d --- /dev/null +++ b/apps/rustack/tests/docker_startup.rs @@ -0,0 +1,698 @@ +//! Ordinary binary qualification; no injected readiness or replacement executor. +#![cfg(all(unix, feature = "lambda", feature = "stepfunctions"))] +#![forbid(unsafe_code)] + +#[path = "docker_startup/archive.rs"] +mod archive; +#[path = "docker_startup/archive_rejection.rs"] +mod archive_rejection; +#[path = "docker_startup/callback.rs"] +mod callback; +#[path = "docker_startup/container_server.rs"] +mod container_server; +#[path = "docker_startup/custom.rs"] +mod custom; +#[path = "docker_startup/faults.rs"] +mod faults; +#[path = "docker_startup/handler_sdk.rs"] +mod handler_sdk; +#[path = "docker_startup/iac.rs"] +mod iac; +#[path = "docker_startup/image_sdk.rs"] +mod image_sdk; +#[path = "docker_startup/io_faults.rs"] +mod io_faults; +#[path = "docker_startup/node.rs"] +mod node; +#[path = "docker_startup/performance.rs"] +mod performance; +#[path = "docker_startup/process.rs"] +mod process; +#[path = "docker_startup/provided.rs"] +mod provided; +#[path = "docker_startup/provided_archive.rs"] +mod provided_archive; +#[path = "docker_startup/recovery_gate.rs"] +mod recovery_gate; +#[path = "workflows_sdk/runner.rs"] +mod runner; +#[path = "docker_startup/sdk.rs"] +mod sdk; + +use std::{ + collections::BTreeMap, + env, + ffi::OsString, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use process::{Profile, Server}; +use rstest::rstest; +use sha2::{Digest, Sha256}; +use tokio::{ + fs, + io::AsyncReadExt, + time::{Instant, timeout}, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit Linux server/socket/image prerequisites; run make \ + test-workflows-server-docker"] +async fn test_should_execute_image_sdk_and_nested_asl_from_a_server_container() -> Result<()> { + container_server::exercise( + container_server::Mode::Sdk, + container_server::ImageProfile::Fixture, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit Linux server/socket/image prerequisites; run make \ + test-workflows-server-docker"] +async fn test_should_refuse_unreachable_server_container_gateway_and_join_startup_cleanup() +-> Result<()> { + container_server::exercise( + container_server::Mode::Unreachable, + container_server::ImageProfile::Fixture, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit official Node image prerequisites; run make test-lambda-node-docker"] +async fn test_should_execute_official_node_image_with_signed_sdks_and_asl() -> Result<()> { + node::exercise().await +} + +#[rstest] +#[case::disconnected(Profile::Disconnected)] +#[case::connected(Profile::Connected)] +#[case::missing_relay(Profile::MissingRelay)] +#[case::legacy(Profile::Legacy)] +#[case::wrong_endpoint(Profile::WrongEndpoint)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit two-daemon image prerequisites; run make test-lambda-startup-docker"] +async fn test_should_qualify_ordinary_binary_and_retain_negative_startup_cleanup( + #[case] profile: Profile, +) -> Result<()> { + let mut server = Server::spawn(profile).await?; + let result = timeout(Duration::from_secs(360), async { + if profile.rejects() { + server.failure().await?; + return Ok(()); + } + server.ready().await?; + sdk::exercise( + &server.endpoint, + &process::handler()?, + &process::architecture()?, + profile == Profile::Connected, + false, + ) + .await?; + ensure!( + process::get(&server.endpoint, "/_rustack/capabilities") + .await? + .1["ready"] + == true, + "ordinary FunctionError poisoned backend readiness" + ); + if profile == Profile::Disconnected { + boto(&server).await?; + let before = process::get(&server.endpoint, "/_rustack/runtime-probe") + .await? + .1; + let owner = + fs::read(server.path()?.join("data/.rustack-lambda-docker/owner.v1")).await?; + server.stop(true).await?; + server.audit().await?; + server.restart().await?; + server.ready().await?; + let after = process::get(&server.endpoint, "/_rustack/runtime-probe") + .await? + .1; + ensure!( + before["instanceId"] != after["instanceId"], + "restart retained stale gateway identity" + ); + ensure!( + fs::read(server.path()?.join("data/.rustack-lambda-docker/owner.v1")).await? + == owner, + "restart replaced immutable namespace identity" + ); + } + Ok::<(), anyhow::Error>(()) + }) + .await + .context("ordinary startup SDK observer expired") + .and_then(|result| result); + let cleanup = server.stop(!profile.rejects()).await; + let audit = server.audit().await; + if result.is_err() || cleanup.is_err() || audit.is_err() { + server.retain_on_error().await; + } + cleanup?; + audit?; + result +} +async fn boto(server: &Server) -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let arguments = [ + root.join("tests/workflows/docker_startup_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(process::handler()?), + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + ]; + let mut job = runner::Runner::spawn(&runner::interpreter()?, &arguments)?; + let barrier = job.await_pid().await; + let deadline = if barrier.is_ok() { + Instant::now() + Duration::from_secs(180) + } else { + Instant::now() + }; + let report = job.wait(deadline).await?; + let pid = barrier?; + ensure!( + pid == report.pid && !report.forced && report.status.success(), + "ordinary boto3 image exercise failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + let value: serde_json::Value = serde_json::from_slice(&report.stdout)?; + ensure!( + value["computed"] == 42 && value["warmCounter"] == 2 && value["functionError"] == true, + "missing boto3 evidence" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual production Docker startup; run make test-lambda-startup-docker"] +async fn test_should_restore_real_image_archive_and_preserve_generations_after_rejection() +-> Result<()> { + let mut server = Server::spawn(Profile::Archive).await?; + server + .root + .as_mut() + .context("image archive evidence guard")? + .disable_cleanup(true); + eprintln!("image archive evidence: {}", server.path()?.display()); + let result = timeout(Duration::from_secs(360), async { + server.ready().await?; + sdk::exercise( + &server.endpoint, + &process::handler()?, + &process::architecture()?, + false, + true, + ) + .await?; + let checkpoint = sdk::archive_checkpoint(&server.endpoint).await?; + server.stop(true).await?; + server.audit().await?; + let original = serde_json::to_value(archive::read(server.path()?).await?)?; + let snapshots = server.path()?.join("snapshots"); + let current = snapshots.join("images"); + let previous = snapshots.join(".images.previous"); + copy_tree(¤t, &previous).await?; + server.restart().await?; + server.ready().await?; + ensure!( + !fs::try_exists(&previous).await?, + "successful domain import did not retire previous generation" + ); + sdk::verify_archive(&server.endpoint, &checkpoint).await?; + server.stop(true).await?; + server.audit().await?; + ensure!( + serde_json::to_value(archive::read(server.path()?).await?)? == original, + "restored Lambda metadata/resource/revision identity changed" + ); + copy_tree(¤t, &previous).await?; + archive::corrupt_size(server.path()?).await?; + let before = fingerprint(&snapshots).await?; + server.restart().await?; + server.failure().await?; + ensure!( + fingerprint(&snapshots).await? == before, + "rejected immutable image proof modified current/previous archive generations" + ); + Ok::<(), anyhow::Error>(()) + }) + .await + .context("image archive observer expired") + .and_then(|result| result); + let cleanup = server.stop(false).await; + let audit = server.audit().await; + if result.is_err() || cleanup.is_err() || audit.is_err() { + server.retain_on_error().await; + } + cleanup?; + audit?; + result?; + server + .root + .as_mut() + .context("image archive evidence guard")? + .disable_cleanup(false); + Ok(()) +} +async fn copy_tree(source: &Path, target: &Path) -> Result<()> { + ensure!( + !fs::try_exists(target).await?, + "previous generation already exists" + ); + let mut queue = vec![(source.to_owned(), target.to_owned())]; + let mut visited = 0; + while let Some((source, target)) = queue.pop() { + visited += 1; + ensure!(visited <= 128, "fixture directory cap"); + fs::create_dir_all(&target).await?; + let mut entries = fs::read_dir(&source).await?; + while let Some(entry) = entries.next_entry().await? { + let kind = entry.file_type().await?; + if kind.is_dir() { + queue.push((entry.path(), target.join(entry.file_name()))); + } else { + ensure!( + kind.is_file(), + "fixture refuses symlink/special archive entry" + ); + fs::copy(entry.path(), target.join(entry.file_name())).await?; + } + } + } + Ok(()) +} +async fn fingerprint(root: &Path) -> Result>> { + let mut queue = vec![root.to_owned()]; + let mut result = BTreeMap::new(); + let mut total = 0_usize; + let mut visited = 0; + while let Some(directory) = queue.pop() { + visited += 1; + ensure!(visited <= 128, "fixture directory cap"); + let mut entries = fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let kind = entry.file_type().await?; + if kind.is_dir() { + queue.push(entry.path()); + continue; + } + ensure!( + kind.is_file() && result.len() < 512, + "fixture archive file cap/kind" + ); + let mut bytes = Vec::new(); + fs::File::open(entry.path()) + .await? + .take(16 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .await?; + total = total + .checked_add(bytes.len()) + .context("fixture size overflow")?; + ensure!( + bytes.len() <= 16 * 1024 * 1024 && total <= 64 * 1024 * 1024, + "fixture archive byte cap" + ); + result.insert( + entry.path().strip_prefix(root)?.to_owned(), + Sha256::digest(&bytes).to_vec(), + ); + } + } + Ok(result) +} + +#[test] +fn test_should_reject_earlier_startup_failure_as_negative_stage_evidence() { + assert!( + !Profile::WrongEndpoint.matches_failure("Docker startup probe failed: resource mismatch") + ); + assert!( + !Profile::MissingRelay + .matches_failure("Docker startup daemon failed: Docker response status 404 Not Found") + ); + assert!(!Profile::MissingRelay.matches_failure( + "Docker startup relay preparation failed: Docker response status 500 Internal Server Error" + )); + assert!(!Profile::ZipArchiveNoSource.matches_failure("Docker ownership profile mismatch")); + assert!(!Profile::ZipArchive.matches_failure("Docker startup resource probe failed")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit custom RIC images and bridge; run make test-lambda-custom-ric-docker"] +async fn test_should_execute_custom_ric_imageconfig_and_immutable_revisions_through_ordinary_service() +-> Result<()> { + let mut server = Server::spawn(Profile::Connected).await?; + // Keep alias intent even if the outer test panics or its future is dropped. + server + .root + .as_mut() + .context("custom fixture evidence")? + .disable_cleanup(true); + eprintln!("custom RIC evidence: {}", server.path()?.display()); + let result = custom::exercise(&mut server).await; + let cleanup = server.stop(true).await; + let audit = server.audit().await; + if result.is_err() || cleanup.is_err() || audit.is_err() { + server.retain_on_error().await; + } + result?; + cleanup?; + audit?; + server + .root + .as_mut() + .context("custom fixture evidence")? + .disable_cleanup(false); + Ok(()) +} + +#[rstest] +#[case::typed_faults(faults::Case::Faults)] +#[case::initializing_disconnect(faults::Case::Initializing)] +#[case::busy_disconnect(faults::Case::Busy)] +#[case::accepted_work_shutdown(faults::Case::Saturation)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires real Docker fault fixtures; run make test-lambda-faults-docker"] +async fn test_should_settle_real_image_faults_and_public_cancellation_admission( + #[case] case: faults::Case, +) -> Result<()> { + let mut server = Server::spawn(case.profile()).await?; + server + .root + .as_mut() + .context("fault evidence guard")? + .disable_cleanup(true); + eprintln!("fault evidence: {}", server.path()?.display()); + let result = faults::exercise(&mut server, case).await; + let cleanup = server.stop(case != faults::Case::Saturation).await; + let audit = server.audit().await; + let logs = faults::check_logs(&server).await; + if result.is_err() || cleanup.is_err() || audit.is_err() || logs.is_err() { + server.retain_on_error().await; + } + result?; + cleanup?; + audit?; + logs?; + server + .root + .as_mut() + .context("fault evidence guard")? + .disable_cleanup(false); + Ok(()) +} + +#[rstest] +#[case::allocating(io_faults::Case::Create)] +#[case::cleanup(io_faults::Case::Delete)] +#[case::daemon_loss(io_faults::Case::Loss)] +#[case::deployment_saturation(io_faults::Case::Saturation)] +#[case::create_io_timeout(io_faults::Case::CreateTimeout)] +#[case::delete_ack_timeout(io_faults::Case::DeleteTimeout)] +#[case::interrupted_pull(io_faults::Case::Pull)] +#[case::negative_images(io_faults::Case::Negative)] +#[case::runtime_http_bounds(io_faults::Case::RuntimeBounds)] +#[case::global_execution_limit(io_faults::Case::GlobalLimit)] +#[case::default_function_limit(io_faults::Case::DefaultLimit)] +#[case::independent_memory_limit(io_faults::Case::MemoryLimit)] +#[case::independent_environment_limit(io_faults::Case::EnvironmentLimit)] +#[case::zip_allocating(io_faults::Case::ZipCreate)] +#[case::zip_cleanup(io_faults::Case::ZipDelete)] +#[case::composite_pressure(io_faults::Case::CompositePressure)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit real-daemon IO proxy fixtures; run make test-lambda-io-faults-docker"] +async fn test_should_retain_actual_management_io_through_cancellation_and_deployment_faults( + #[case] case: io_faults::Case, +) -> Result<()> { + io_faults::exercise(case).await +} + +#[rstest] +#[case::deployments("deployments")] +#[case::faults("faults")] +#[case::initializing("initializing")] +#[case::busy("busy")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual provided runtime and original ZIP files; run make test-lambda-docker"] +async fn test_should_execute_ordinary_provided_zip_and_join_owned_retirement( + #[case] phase: &str, +) -> Result<()> { + provided::exercise(phase).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual ZIP/Image archive startup; run make test-lambda-provided-archive-docker"] +async fn test_should_restore_provided_zip_archives_without_replay_and_preserve_invalid_generations() +-> Result<()> { + provided_archive::exercise(false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires configured and image-only real startup; run make \ + test-lambda-provided-archive-docker"] +async fn test_should_refuse_provided_zip_archives_without_configured_runtime() -> Result<()> { + provided_archive::exercise(true).await +} + +#[tokio::test] +#[ignore = "requires actual root release image and explicit Docker resources"] +async fn test_should_execute_image_sdk_and_nested_asl_from_the_release_image() -> Result<()> { + container_server::exercise( + container_server::Mode::Sdk, + container_server::ImageProfile::Release, + ) + .await +} + +#[tokio::test] +#[ignore = "requires actual root release image and explicit Docker resources"] +async fn test_should_refuse_unreachable_release_image_gateway_and_join_startup_cleanup() +-> Result<()> { + container_server::exercise( + container_server::Mode::Unreachable, + container_server::ImageProfile::Release, + ) + .await +} + +#[tokio::test] +#[ignore = "requires explicit Docker socket, fixtures and qualified gateway"] +async fn test_should_execute_local_s3_and_dynamodb_from_an_explicit_image_sdk_context() -> Result<()> +{ + handler_sdk::exercise(false).await +} +#[tokio::test] +#[ignore = "requires explicit Docker socket, fixtures and qualified gateway"] +async fn test_should_reject_wrong_image_sdk_credentials_without_ambient_fallback() -> Result<()> { + handler_sdk::exercise(true).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit callback image, Docker socket and qualified gateway"] +async fn test_should_execute_real_image_callbacks_before_and_after_lambda_response() -> Result<()> { + callback::exercise(callback::Group::Causal).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit callback image, Docker socket and qualified gateway"] +async fn test_should_enforce_image_callback_retry_heartbeat_timeouts_and_stop() -> Result<()> { + callback::exercise(callback::Group::Timing).await +} + +#[tokio::test] +#[ignore = "requires explicit Pulumi image, Docker socket and qualified gateway"] +async fn test_should_manage_a_real_image_workflow_through_pulumi_lifecycle() -> Result<()> { + iac::exercise(iac::Mode::Lifecycle).await +} + +#[tokio::test] +#[ignore = "requires explicit Pulumi image, Docker socket and qualified gateway"] +async fn test_should_retire_the_pulumi_plugin_tree_after_controller_stop() -> Result<()> { + iac::exercise(iac::Mode::Stop).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "actual image timings; run make test-workflows-image-performance"] +async fn test_should_measure_one_hundred_warm_image_invocations() -> Result<()> { + performance::exercise("warm").await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "actual cold environment timings; run make test-workflows-image-performance"] +async fn test_should_measure_twenty_preloaded_cold_image_environments() -> Result<()> { + performance::exercise("cold").await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "actual log-flood/timeout cleanup; run make test-workflows-image-performance"] +async fn test_should_reuse_one_image_slot_after_one_hundred_log_flood_timeouts() -> Result<()> { + performance::exercise("cycles").await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires two actual binaries on one explicit daemon; run make \ + test-lambda-startup-docker"] +async fn test_should_preserve_live_warm_instance_during_other_owner_crash_recovery() -> Result<()> { + let mut gate = + recovery_gate::Gate::start(&PathBuf::from(env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")?))?; + let mut survivor = Server::spawn(Profile::Connected).await?; + let mut owned_crashed = None; + let result = timeout(Duration::from_secs(360), async { + owned_crashed = + Some(Server::spawn_with_socket(Profile::Connected, Some(&gate.socket)).await?); + let crashed = owned_crashed.as_mut().context("second owned binary")?; + survivor.ready().await?; + crashed.ready().await?; + ensure!( + survivor.observation_identity().await?.0 != crashed.observation_identity().await?.0, + "two live binaries shared installation identity" + ); + let mut warm = sdk::Witness::start(&survivor).await?; + let mut abandoned = sdk::Witness::start(crashed).await?; + let stable = sdk::owned_containers(&survivor).await?; + let old = sdk::owned_containers(crashed).await?; + ensure!( + stable.len() == 2 && old.len() == 2 && stable.is_disjoint(&old), + "live owner scopes differ" + ); + abandoned.advance().await?; + crashed.crash().await?; + warm.advance().await?; + gate.arm(); + crashed.restart().await?; + gate.blocked().await?; + let witness = async { + for _ in 0..3 { + warm.advance().await?; + let (status, value) = + process::get(&survivor.endpoint, "/_rustack/capabilities").await?; + ensure!( + status == 200 && value["ready"] == true, + "survivor lost readiness during recovery" + ); + } + Ok::<(), anyhow::Error>(()) + }; + witness.await?; + gate.release(); + crashed.ready().await?; + ensure!( + sdk::owned_containers(&survivor).await? == stable + && sdk::owned_containers(crashed).await?.is_disjoint(&old), + "recovery retired foreign containers or retained abandoned containers" + ); + warm.advance().await?; + Ok::<(), anyhow::Error>(()) + }) + .await + .context("two-instance crash recovery observer expired") + .and_then(|value| value); + gate.release(); + let first = if let Some(crashed) = &mut owned_crashed { + crashed.stop(true).await + } else { + Ok(()) + }; + let second = survivor.stop(true).await; + let first_audit = if let Some(crashed) = &owned_crashed { + crashed.audit().await + } else { + Ok(()) + }; + let second_audit = survivor.audit().await; + if result.is_err() + || first.is_err() + || second.is_err() + || first_audit.is_err() + || second_audit.is_err() + { + if let Some(crashed) = &mut owned_crashed { + crashed.retain_on_error().await; + } + survivor.retain_on_error().await; + } + let proxy = gate.close().await; + first?; + second?; + first_audit?; + second_audit?; + proxy?; + result +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires real saved images and affiliations; run make test-lambda-startup-docker"] +async fn test_should_restore_saved_image_after_tag_move_and_reject_absent_or_foreign_candidates() +-> Result<()> { + archive_rejection::exercise().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires real cleanup failure and snapshot recovery; run make \ + test-lambda-startup-docker"] +async fn test_should_preserve_previous_snapshot_when_docker_cleanup_cannot_be_confirmed() +-> Result<()> { + let mut gate = + recovery_gate::Gate::start(&PathBuf::from(env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")?))?; + let mut server = Server::spawn_with_socket(Profile::Archive, Some(&gate.socket)).await?; + let result = timeout(Duration::from_secs(480), async { + server.ready().await?; + let _committed = sdk::Witness::start(&server).await?; + server.stop(true).await?; + server.audit().await?; + let snapshots = server.path()?.join("snapshots"); + let saved = fingerprint(&snapshots).await?; + server.restart().await?; + server.ready().await?; + let unsaved = sdk::Witness::start(&server).await?; + let old = sdk::owned_containers(&server).await?; + ensure!( + old.len() == 2, + "cleanup-failure fixture lacked live containers" + ); + gate.disconnect().await?; + server.stop(false).await?; + ensure!( + fingerprint(&snapshots).await? == saved, + "unconfirmed cleanup published a snapshot generation" + ); + gate.resume().await?; + server.restart().await?; + server.ready().await?; + unsaved.missing().await?; + ensure!( + sdk::owned_containers(&server).await?.is_disjoint(&old), + "restart retained abandoned owned containers" + ); + server.stop(true).await?; + Ok::<(), anyhow::Error>(()) + }) + .await + .context("failed-cleanup snapshot observer expired") + .and_then(|value| value); + gate.release(); + let stopped = server.stop_after_transport_loss().await; + let audit = if result.is_ok() { + server.audit().await + } else { + Ok(()) + }; + if result.is_err() || stopped.is_err() || audit.is_err() { + server.retain_on_error().await; + } + let proxy = gate.close().await; + stopped?; + audit?; + proxy?; + result +} diff --git a/apps/rustack/tests/docker_startup/archive.rs b/apps/rustack/tests/docker_startup/archive.rs new file mode 100644 index 00000000..254d6a36 --- /dev/null +++ b/apps/rustack/tests/docker_startup/archive.rs @@ -0,0 +1,261 @@ +//! Independent bounded fixture codec for one real v2 Lambda metadata section. + +use std::{ + io::{Cursor, Read}, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use rustack_lambda_core::{ + artifact::{ImageMetadata, ResolvedImage}, + provider::LambdaSnapshot, +}; +use serde::{Serialize, de::DeserializeOwned}; +use sha2::{Digest, Sha256}; +use tokio::{fs, io::AsyncReadExt, task::spawn_blocking, time::timeout}; +use uuid::Uuid; + +const PLAIN_LIMIT: usize = 16 * 1024 * 1024; +const COMPRESSED_LIMIT: u64 = 4 * 1024 * 1024; +fn metadata(root: &Path) -> PathBuf { + root.join("snapshots/images/services/lambda/meta.ss.zst") +} +async fn bytes(path: &Path) -> Result> { + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(COMPRESSED_LIMIT + 1) + .read_to_end(&mut bytes) + .await?; + ensure!( + u64::try_from(bytes.len())? <= COMPRESSED_LIMIT, + "fixture compressed metadata cap" + ); + Ok(bytes) +} +fn u16_at(bytes: &[u8], at: usize) -> Result { + Ok(u16::from_le_bytes( + bytes + .get(at..at.checked_add(2).context("offset")?) + .context("u16")? + .try_into()?, + )) +} +fn u32_at(bytes: &[u8], at: usize) -> Result { + Ok(u32::from_le_bytes( + bytes + .get(at..at.checked_add(4).context("offset")?) + .context("u32")? + .try_into()?, + )) +} +fn u64_at(bytes: &[u8], at: usize) -> Result { + Ok(u64::from_le_bytes( + bytes + .get(at..at.checked_add(8).context("offset")?) + .context("u64")? + .try_into()?, + )) +} +pub(super) fn decode(bytes: &[u8]) -> Result { + let mut decoder = zstd::stream::read::Decoder::new(Cursor::new(bytes))?.take( + u64::try_from(PLAIN_LIMIT)? + .checked_add(1) + .context("plain cap")?, + ); + let mut plain = Vec::new(); + decoder.read_to_end(&mut plain)?; + decode_plain(&plain) +} +pub(super) fn decode_plain(plain: &[u8]) -> Result { + ensure!( + plain.len() <= PLAIN_LIMIT && plain.len() >= 76, + "fixture plain metadata cap/header" + ); + ensure!( + plain.get(..8) == Some(b"RSSNAP\0\x02") + && u16_at(plain, 8)? == 2 + && u16_at(plain, 10)? == 0, + "fixture v2 metadata kind" + ); + ensure!( + u32_at(plain, 12)? == 36 + && u32_at(plain, 16)? == 1 + && u64_at(plain, 20)? == u64::try_from(plain.len())? + && u64_at(plain, 28)? == 0, + "fixture metadata layout" + ); + ensure!( + u16_at(plain, 36)? == 1 + && u16_at(plain, 38)? == 0 + && u64_at(plain, 40)? == 64 + && u64_at(plain, 56)? == 1, + "fixture single state section" + ); + let end = plain.len().checked_sub(12).context("footer")?; + ensure!( + u64_at(plain, 48)? == u64::try_from(end.checked_sub(64).context("payload")?)? + && u64_at(plain, end.checked_add(4).context("footer length")?)? == u64::try_from(end)?, + "fixture section bounds" + ); + ensure!( + crc32fast::hash(plain.get(..end).context("CRC range")?) == u32_at(plain, end)?, + "fixture CRC" + ); + let payload = plain.get(64..end).context("state payload")?; + let mut reader = Cursor::new(payload); + let snapshot = ciborium::de::from_reader_with_recursion_limit(&mut reader, 128)?; + ensure!( + reader.position() == u64::try_from(payload.len())?, + "fixture trailing CBOR" + ); + Ok(snapshot) +} +pub(super) fn encode(snapshot: &T) -> Result> { + let mut state = Vec::new(); + ciborium::ser::into_writer(snapshot, &mut state)?; + let total = state.len().checked_add(76).context("archive length")?; + ensure!(total <= PLAIN_LIMIT, "fixture encoded cap"); + let mut plain = Vec::with_capacity(total); + plain.extend_from_slice(b"RSSNAP\0\x02"); + plain.extend_from_slice(&2u16.to_le_bytes()); + plain.extend_from_slice(&0u16.to_le_bytes()); + plain.extend_from_slice(&36u32.to_le_bytes()); + plain.extend_from_slice(&1u32.to_le_bytes()); + plain.extend_from_slice(&u64::try_from(total)?.to_le_bytes()); + plain.extend_from_slice(&0u64.to_le_bytes()); + plain.extend_from_slice(&1u16.to_le_bytes()); + plain.extend_from_slice(&0u16.to_le_bytes()); + plain.extend_from_slice(&64u64.to_le_bytes()); + plain.extend_from_slice(&u64::try_from(state.len())?.to_le_bytes()); + plain.extend_from_slice(&1u64.to_le_bytes()); + plain.extend_from_slice(&state); + let payload_len = u64::try_from(plain.len())?; + plain.extend_from_slice(&crc32fast::hash(&plain).to_le_bytes()); + plain.extend_from_slice(&payload_len.to_le_bytes()); + ensure!(plain.len() == total, "fixture encoded layout"); + let compressed = zstd::stream::encode_all(Cursor::new(plain), 1)?; + ensure!( + u64::try_from(compressed.len())? <= COMPRESSED_LIMIT, + "fixture compressed output cap" + ); + Ok(compressed) +} +pub(super) async fn read(root: &Path) -> Result { + let bytes = bytes(&metadata(root)).await?; + let mut job = spawn_blocking(move || decode(&bytes)); + timeout(Duration::from_secs(30), &mut job).await?? +} +pub(super) async fn corrupt_size(root: &Path) -> Result<()> { + let path = metadata(root); + let original = bytes(&path).await?; + let mut job = spawn_blocking(move || { + let mut snapshot = decode(&original)?; + let target = snapshot + .functions + .functions + .iter_mut() + .find(|function| { + function.latest.state == "Active" && function.latest.resolved_image.is_some() + }) + .context("actual saved usable image")?; + target.latest.code_size = target + .latest + .code_size + .checked_add(1) + .context("code size")?; + encode(&snapshot) + }); + let corrupted = timeout(Duration::from_secs(30), &mut job).await???; + fs::write(path, corrupted).await?; + Ok(()) +} + +pub(super) async fn corrupt_zip(root: &Path) -> Result<()> { + let path = metadata(root); + let original = bytes(&path).await?; + let mut job = spawn_blocking(move || { + let mut snapshot = decode(&original)?; + let version = &mut snapshot + .functions + .functions + .iter_mut() + .find(|function| function.latest.package_type == "Zip") + .context("actual saved provided ZIP")? + .latest; + let source = b"invalid ZIP archive source"; + version.zip_bytes_base64 = Some(STANDARD.encode(source)); + version.code_sha256 = STANDARD.encode(Sha256::digest(source)); + version.code_size = u64::try_from(source.len())?; + encode(&snapshot) + }); + let corrupted = timeout(Duration::from_secs(30), &mut job).await???; + fs::write(path, corrupted).await?; + Ok(()) +} + +pub(super) async fn missing_image(root: &Path) -> Result { + let path = metadata(root); + let original = bytes(&path).await?; + let mut job = spawn_blocking(move || { + let mut snapshot = decode(&original)?; + let version = &mut snapshot + .functions + .functions + .iter_mut() + .find(|function| function.latest.resolved_image.is_some()) + .context("saved image")? + .latest; + let mut metadata: ImageMetadata = version + .resolved_image + .take() + .context("saved resolved image")? + .into(); + let absent = format!( + "sha256:{}{}", + Uuid::new_v4().simple(), + Uuid::new_v4().simple() + ); + metadata.image_id = absent.parse()?; + metadata.selected_manifest = None; + metadata.registry_manifest = None; + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + None, + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + )?; + let resolved = ResolvedImage::try_from(metadata)?; + version.code_sha256 = resolved.code_sha256()?; + version.resolved_image = Some(resolved); + Ok::<_, anyhow::Error>((encode(&snapshot)?, absent)) + }); + let (corrupted, absent) = timeout(Duration::from_secs(30), &mut job).await???; + fs::write(path, corrupted).await?; + Ok(absent) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_should_refuse_truncated_wrong_kind_and_invalid_crc_fixture_frames() -> Result<()> { + let source = encode(&LambdaSnapshot::default())?; + assert!(decode(source.get(..source.len() / 2).context("truncated source")?).is_err()); + for at in [8usize, 36, 64] { + let mut plain = zstd::stream::decode_all(Cursor::new(&source))?; + *plain.get_mut(at).context("fixed corrupt offset")? ^= 1; + let corrupted = zstd::stream::encode_all(Cursor::new(plain), 1)?; + assert!(decode(&corrupted).is_err()); + } + Ok(()) + } + #[test] + fn test_should_refuse_fixture_decompression_above_the_plain_cap() -> Result<()> { + let compressed = zstd::stream::encode_all(Cursor::new(vec![0; PLAIN_LIMIT + 1]), 1)?; + assert!(decode(&compressed).is_err()); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/archive_rejection.rs b/apps/rustack/tests/docker_startup/archive_rejection.rs new file mode 100644 index 00000000..0837cfa5 --- /dev/null +++ b/apps/rustack/tests/docker_startup/archive_rejection.rs @@ -0,0 +1,136 @@ +//! Real archive restore with moved references and valid but foreign affiliations. + +use std::{env, path::Path, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::{ + artifact::ImageReference, + docker::{DockerError, DockerWire}, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + fs, + time::{Instant, timeout}, +}; +use uuid::Uuid; + +use super::{ + archive, copy_tree, + custom::Alias, + fingerprint, + process::{self, Profile, Server}, + sdk, +}; + +pub(super) async fn exercise() -> Result<()> { + let mut server = Server::spawn(Profile::Archive).await?; + let mut alias = None; + let result = timeout(Duration::from_secs(600), async { + server.ready().await?; + let a = env::var("RUSTACK_CUSTOM_IMAGE_A")?; + let b = env::var("RUSTACK_CUSTOM_IMAGE_B")?; + let ids = [server.track_image(&a).await?, server.track_image(&b).await?]; + ensure!( + ids.first() != ids.get(1), + "moving-tag sources are identical" + ); + alias = Some(Alias::new(&server, ids).await?); + let moving = alias.as_mut().context("retained source alias")?; + moving.select(0).await?; + let mut witness = sdk::Witness::start_image(&server, moving.name()).await?; + server.stop(true).await?; + server.audit().await?; + moving.select(1).await?; + server.restart().await?; + server.ready().await?; + witness.cold_revision("A").await?; + server.stop(true).await?; + server.audit().await?; + let snapshots = server.path()?.join("snapshots"); + let current = snapshots.join("images"); + let previous = snapshots.join(".images.previous"); + copy_tree(¤t, &previous).await?; + let meta = current.join("services/lambda/meta.ss.zst"); + let original_meta = fs::read(&meta).await?; + let absent = archive::missing_image(server.path()?).await?; + let at = Instant::now() + Duration::from_secs(10); + let mut wire = DockerWire::connect(process::socket()?.parse()?, at).await?; + let missing: ImageReference = absent.parse()?; + let observed = wire.port().inspect_image(&missing, at).await; + let joined = wire.shutdown().await; + joined?; + ensure!( + matches!(observed, Err(DockerError::Status(status)) if status.as_u16() == 404), + "absent-image fixture unexpectedly exists" + ); + let before = fingerprint(&snapshots).await?; + server.restart().await?; + server + .failure_with(Some("Lambda immutable image restore failed")) + .await?; + ensure!( + fingerprint(&snapshots).await? == before, + "missing image changed snapshot generations" + ); + server.stop(false).await?; + fs::write(&meta, original_meta).await?; + reject_affiliations(&mut server, &snapshots).await?; + server.audit().await?; + Ok::<(), anyhow::Error>(()) + }) + .await + .context("archive rejection observer expired") + .and_then(|value| value); + let cleanup = server.reap().await; + let alias_cleanup = if let Some(moving) = &mut alias { + moving.cleanup().await + } else { + Ok(()) + }; + let audit = server.audit().await; + if result.is_err() || cleanup.is_err() || alias_cleanup.is_err() || audit.is_err() { + server.retain_on_error().await; + } + result?; + cleanup?; + alias_cleanup?; + audit?; + Ok(()) +} + +async fn reject_affiliations(server: &mut Server, snapshots: &Path) -> Result<()> { + let affiliation = server.path()?.join("data/.rustack-lambda-docker/daemon.v1"); + let original = fs::read(&affiliation).await?; + ensure!(original.len() == 72, "daemon record fixture length"); + fs::write(server.path()?.join("affiliation-original.v1"), &original).await?; + for (range, role) in [(8..24, "owner"), (24..40, "engine")] { + let mut changed = original.clone(); + changed + .get_mut(range) + .context("affiliation UUID")? + .copy_from_slice(Uuid::new_v4().as_bytes()); + let hash = Sha256::digest(changed.get(..40).context("affiliation payload")?); + changed + .get_mut(40..) + .context("affiliation checksum")? + .copy_from_slice(&hash); + fs::write( + server.path()?.join(format!("affiliation-{role}-intent.v1")), + &changed, + ) + .await?; + fs::write(&affiliation, &changed).await?; + let before = fingerprint(snapshots).await?; + server.restart().await?; + server + .failure_with(Some("invalid Docker ownership state")) + .await?; + server.stop(false).await?; + ensure!( + fingerprint(snapshots).await? == before && fs::read(&affiliation).await? == changed, + "foreign affiliation was rewritten or snapshot changed" + ); + fs::write(&affiliation, &original).await?; + } + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/callback.rs b/apps/rustack/tests/docker_startup/callback.rs new file mode 100644 index 00000000..bdd9a57b --- /dev/null +++ b/apps/rustack/tests/docker_startup/callback.rs @@ -0,0 +1,178 @@ +//! Actual image callback consumers and confirmed-save credential observations. + +#[path = "callback/audit.rs"] +mod audit; + +use std::{collections::HashSet, env, ffi::OsString, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, anyhow, ensure}; +use serde::Deserialize; +use serde_json::Value; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + image_sdk, + process::{self, Profile, Server}, +}; + +#[derive(Debug, Clone, Copy)] +pub(super) enum Group { + Causal, + Timing, +} +impl Group { + fn name(self) -> &'static str { + match self { + Self::Causal => "causal", + Self::Timing => "timing", + } + } + fn cases(self) -> &'static [&'static str] { + match self { + Self::Causal => &["early", "late", "externalFailure", "invocationFailure"], + Self::Timing => &[ + "retry", + "heartbeat", + "absoluteTimeout", + "heartbeatTimeout", + "stop", + ], + } + } +} +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Versions { + lambda: String, + sfn: String, + dynamodb: String, +} +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Evidence { + schema_version: u8, + group: String, + cases: Vec, + versions: Versions, + functions: [String; 2], + executions: Vec, + worker_requests: Vec, + hashes: Vec, + history_candidates: usize, + history_bytes: usize, +} +fn evidence(group: Group, observed: &Value) -> Result { + ensure!( + serde_json::to_vec(observed)?.len() <= 16 * 1024, + "callback report byte cap" + ); + let value: Evidence = serde_json::from_value(observed.clone()) + .map_err(|_| anyhow!("invalid callback report shape"))?; + let (hashes, workers) = match group { + Group::Causal => (4, 1), + Group::Timing => (6, 4), + }; + ensure!( + value.schema_version == 1 + && value.group == group.name() + && value + .cases + .iter() + .map(String::as_str) + .eq(group.cases().iter().copied()) + && [ + &value.versions.lambda, + &value.versions.sfn, + &value.versions.dynamodb + ] + .into_iter() + .all(|version| version == "3.1105.0"), + "callback semantic/version inventory differs" + ); + ensure!( + value + .functions + .iter() + .all(|name| (1..=64).contains(&name.len()) + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b))) + && value.functions.iter().collect::>().len() == 2, + "invalid callback function collection" + ); + ensure!( + value.executions.len() == 5 + && value.executions.iter().all(|arn| arn.len() <= 256 + && arn.starts_with("arn:aws:states:us-east-1:000000000000:execution:") + && arn + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_:-".contains(&b))) + && value.executions.iter().collect::>().len() == 5, + "invalid retained callback execution collection" + ); + ensure!( + value.worker_requests.len() == workers + && value.worker_requests.iter().all(|id| id.len() == 36 + && Uuid::parse_str(id) + .is_ok_and(|uuid| uuid.get_version_num() == 4 && uuid.to_string() == *id)) + && value.worker_requests.iter().collect::>().len() == workers, + "invalid completed callback Event collection" + ); + ensure!( + value.hashes.len() == hashes + && value.hashes.iter().collect::>().len() == hashes + && value.hashes.iter().all(|hash| audit::valid_hash(hash)) + && value.history_candidates <= 65_536 + && value.history_bytes <= 16 * 1024 * 1024, + "invalid callback privacy evidence" + ); + Ok(value) +} + +pub(super) async fn exercise(group: Group) -> Result<()> { + image_sdk::exercise_with_audit( + Profile::Callback, + async |server: &mut Server| { + let image = env::var("RUSTACK_CALLBACK_IMAGE").context("explicit callback image")?; + server.track_image(&image).await?; + let capabilities = server.ready().await?; + ensure!( + capabilities["workflows"]["callbacks"] == true + && capabilities["workflows"]["lambdaRequestResponse"] == true + && capabilities["workflows"]["limits"]["dispatches"] == 1, + "image callback capability/profile mismatch" + ); + let (owner, engine) = server.observation_identity().await?; + Ok(vec![ + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/callback_image_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(image), + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("callback metadata socket")?, + OsString::from(owner), + OsString::from(engine), + OsString::from(group.name()), + ]) + }, + |observed| evidence(group, observed).map(|_| ()), + async |server: &Server, observed: &Value| { + let input = evidence(group, observed)?; + let root = server.path()?.to_owned(); + let deadline = Instant::now() + Duration::from_secs(30); + audit::run(&root, &input, deadline).await?; + eprintln!( + "actual image callback SDK versions: 3.1105.0; {}", + group.name() + ); + Ok(()) + }, + ) + .await +} diff --git a/apps/rustack/tests/docker_startup/callback/audit.rs b/apps/rustack/tests/docker_startup/callback/audit.rs new file mode 100644 index 00000000..08285168 --- /dev/null +++ b/apps/rustack/tests/docker_startup/callback/audit.rs @@ -0,0 +1,559 @@ +//! Bounded independent plaintext/hash observations over one stopped private generation. + +use std::{ + collections::HashSet, + fmt::{Formatter, Result as FmtResult, Write}, + future::Future, + io::{Cursor, Read, Result as IoResult}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, anyhow, ensure}; +use nix::libc::{O_NOFOLLOW, O_NONBLOCK}; +use rustack_lambda_core::provider::LambdaSnapshot; +use serde::{ + Deserialize, Deserializer, + de::{Error as DeError, SeqAccess, Visitor}, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + fs::{self, OpenOptions}, + io::AsyncReadExt, + task::{JoinHandle, spawn_blocking}, + time::{Instant, timeout_at}, +}; +use zstd::stream::read::Decoder; + +use super::{super::archive, Evidence}; + +const COMPRESSED: usize = 4 * 1024 * 1024; +const PLAIN: usize = 16 * 1024 * 1024; +const TOTAL_COMPRESSED: usize = 16 * 1024 * 1024; +const TOTAL_PLAIN: usize = 64 * 1024 * 1024; +const CANDIDATES: usize = 65_536; + +pub(super) fn valid_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn hex_digest(bytes: &[u8]) -> Result { + let mut encoded = String::with_capacity(64); + for byte in Sha256::digest(bytes) { + write!(&mut encoded, "{byte:02x}")?; + } + Ok(encoded) +} +#[derive(Debug)] +struct Scan { + hashes: HashSet, + candidates: usize, + plain: usize, + deadline: Instant, + failed: bool, +} +impl Scan { + fn new(hashes: &[String], candidates: usize, plain: usize, deadline: Instant) -> Result { + ensure!( + (1..=64).contains(&hashes.len()) + && hashes.iter().all(|h| valid_hash(h)) + && candidates <= CANDIDATES + && plain <= PLAIN, + "invalid callback scan input" + ); + Ok(Self { + hashes: hashes.iter().cloned().collect(), + candidates, + plain, + deadline, + failed: false, + }) + } + fn scan(&mut self, bytes: &[u8]) -> Result<()> { + ensure!(!self.failed, "callback scan already failed"); + let result = self.scan_inner(bytes); + if result.is_err() { + self.failed = true; + } + result + } + fn scan_inner(&mut self, bytes: &[u8]) -> Result<()> { + ensure!( + Instant::now() < self.deadline && bytes.len() <= PLAIN, + "callback scan source/deadline cap" + ); + self.plain = self + .plain + .checked_add(bytes.len()) + .context("callback plain bytes")?; + ensure!(self.plain <= TOTAL_PLAIN, "callback aggregate plain cap"); + for credential in [b"local-key".as_slice(), b"local-secret".as_slice()] { + ensure!( + !bytes.windows(credential.len()).any(|v| v == credential), + "injected SDK credential persisted" + ); + } + let mut run = 0usize; + for (at, byte) in bytes.iter().enumerate() { + run = if byte.is_ascii_digit() || (b'a'..=b'f').contains(byte) { + run.checked_add(1).context("callback hex run")? + } else { + 0 + }; + if run < 64 { + continue; + } + self.candidates = self + .candidates + .checked_add(1) + .context("callback candidates")?; + ensure!( + self.candidates <= CANDIDATES, + "callback aggregate candidate cap" + ); + let start = at.checked_sub(63).context("callback candidate offset")?; + let value = bytes.get(start..=at).context("callback candidate window")?; + let digest = hex_digest(value)?; + ensure!( + !self.hashes.contains(&digest), + "callback credential persisted" + ); + } + ensure!( + Instant::now() < self.deadline, + "late callback scan observation" + ); + Ok(()) + } +} + +fn fresh(deadline: Instant) -> Result<()> { + ensure!(Instant::now() < deadline, "callback observation expired"); + Ok(()) +} +async fn io(deadline: Instant, future: impl Future>) -> Result { + fresh(deadline)?; + let value = timeout_at(deadline, future) + .await + .context("callback IO deadline")??; + fresh(deadline)?; + Ok(value) +} +async fn files(root: &Path, deadline: Instant) -> Result> { + ensure!( + io(deadline, fs::symlink_metadata(root)) + .await? + .file_type() + .is_dir(), + "invalid callback generation root" + ); + let root = io(deadline, fs::canonicalize(root)).await?; + let mut stack = vec![(root.clone(), 0usize)]; + let mut entries = 0usize; + let mut found = Vec::new(); + while let Some((directory, depth)) = stack.pop() { + fresh(deadline)?; + ensure!(depth <= 8, "callback archive depth cap"); + let mut directory = io(deadline, fs::read_dir(directory)).await?; + while let Some(entry) = io(deadline, directory.next_entry()).await? { + entries = entries.checked_add(1).context("callback archive nodes")?; + ensure!(entries <= 128, "callback archive entry cap"); + let kind = io(deadline, entry.file_type()).await?; + let path = entry.path(); + ensure!( + path.as_os_str().len() <= 4096 + && !kind.is_symlink() + && io(deadline, fs::canonicalize(&path)) + .await? + .starts_with(&root), + "unsafe callback archive path" + ); + if kind.is_dir() { + stack.push(( + path, + depth.checked_add(1).context("callback archive depth")?, + )); + } else { + ensure!( + kind.is_file() && found.len() < 64, + "callback archive file kind/count" + ); + found.push(path); + } + } + } + Ok(found) +} +async fn read(path: &Path, cap: usize, deadline: Instant) -> Result> { + let file = io( + deadline, + OpenOptions::new() + .read(true) + .custom_flags(O_NOFOLLOW | O_NONBLOCK) + .open(path), + ) + .await?; + let metadata = io(deadline, file.metadata()).await?; + ensure!( + metadata.is_file() && metadata.len() <= u64::try_from(cap)?, + "callback observed file cap/kind" + ); + let limit = u64::try_from(cap)? + .checked_add(1) + .context("callback read cap")?; + let mut bytes = Vec::new(); + io(deadline, file.take(limit).read_to_end(&mut bytes)).await?; + ensure!(bytes.len() <= cap, "callback observed read cap"); + Ok(bytes) +} +async fn owned_cpu(deadline: Instant, mut job: JoinHandle>) -> Result { + let result = if let Ok(result) = timeout_at(deadline, &mut job).await { + result.map_err(|_| anyhow!("callback observer worker failed"))? + } else { + // Preserve ownership of the original bounded CPU job after expiry. + let _late = job.await; + return Err(anyhow!("callback observer worker expired")); + }; + fresh(deadline)?; + result +} +fn plain(bytes: &[u8]) -> Result> { + let mut decoder = Decoder::new(Cursor::new(bytes))?.take( + u64::try_from(PLAIN)? + .checked_add(1) + .context("callback decoder cap")?, + ); + let mut out = Vec::new(); + decoder.read_to_end(&mut out)?; + ensure!(out.len() <= PLAIN, "callback decompressed cap"); + Ok(out) +} + +// Deliberately project only durable execution identities/outcomes; definitions and +// arbitrary business payloads cannot satisfy this independent presence oracle. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Workflows { + schema_version: u32, + #[serde(deserialize_with = "executions")] + executions: Vec, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Execution { + arn: String, + stopped_at: Option, + outcome: Outcome, +} +#[derive(Debug, Deserialize)] +#[serde(tag = "status", rename_all = "SCREAMING_SNAKE_CASE")] +enum Outcome { + Running, + Succeeded, + Failed, + TimedOut, + Aborted, +} +fn executions<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + struct Rows; + impl<'de> Visitor<'de> for Rows { + type Value = Vec; + fn expecting(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter.write_str("bounded callback execution rows") + } + fn visit_seq>(self, mut rows: A) -> Result { + let mut values = Vec::new(); + while let Some(row) = rows.next_element::()? { + if values.len() >= 64 + || row.arn.len() > 256 + || row.stopped_at.as_ref().is_some_and(|v| v.len() > 64) + { + return Err(A::Error::custom("callback execution projection cap")); + } + values.push(row); + } + Ok(values) + } + } + deserializer.deserialize_seq(Rows) +} +fn workflows(data: &[u8], expected: &[String]) -> Result<()> { + let snapshot: Workflows = + archive::decode_plain(data).map_err(|_| anyhow!("invalid saved callback executions"))?; + ensure!(snapshot.schema_version == 1, "callback metadata schema"); + let mut actual = HashSet::new(); + for row in snapshot.executions { + ensure!( + actual.insert(row.arn.clone()), + "duplicate callback execution row" + ); + if expected.contains(&row.arn) { + ensure!( + row.stopped_at.is_some() && !matches!(row.outcome, Outcome::Running), + "callback execution not terminal" + ); + } + } + ensure!( + expected.iter().all(|arn| actual.contains(arn)), + "actual terminal callback executions absent from archive" + ); + Ok(()) +} +#[derive(Debug, Clone, Copy)] +enum Kind { + Other, + Lambda, + Workflows, +} +fn inspect( + mut scan: Scan, + bytes: Vec, + compressed: bool, + kind: Kind, + expected: &[String], +) -> Result { + let data = if compressed { plain(&bytes)? } else { bytes }; + scan.scan(&data)?; + match kind { + Kind::Other => (), + Kind::Lambda => { + let snapshot: LambdaSnapshot = archive::decode_plain(&data) + .map_err(|_| anyhow!("invalid saved callback functions"))?; + let names: HashSet<_> = snapshot + .functions + .functions + .iter() + .map(|f| f.name.as_str()) + .collect(); + ensure!( + names == expected.iter().map(String::as_str).collect(), + "actual callback functions absent from archive" + ); + } + Kind::Workflows => workflows(&data, expected)?, + } + fresh(scan.deadline)?; + Ok(scan) +} +pub(super) async fn run(root: &Path, input: &Evidence, deadline: Instant) -> Result<()> { + let selected = root.join("snapshots/images"); + ensure!( + io(deadline, fs::symlink_metadata(&selected)) + .await? + .file_type() + .is_dir(), + "linked callback generation" + ); + let generation = io(deadline, fs::canonicalize(&selected)).await?; + let mut scan = Scan::new( + &input.hashes, + input.history_candidates, + input.history_bytes, + deadline, + )?; + let log = read(&root.join("server.log"), 1024 * 1024, deadline).await?; + fresh(deadline)?; + scan = owned_cpu( + deadline, + spawn_blocking(move || inspect(scan, log, false, Kind::Other, &[])), + ) + .await?; + let mut compressed = 0usize; + let mut lambda = false; + let mut registry = false; + for path in files(&generation, deadline).await? { + let bytes = read(&path, COMPRESSED, deadline).await?; + compressed = compressed + .checked_add(bytes.len()) + .context("callback compressed bytes")?; + ensure!( + compressed <= TOTAL_COMPRESSED, + "callback aggregate compressed cap" + ); + let relative = path.strip_prefix(&generation)?; + let (kind, expected) = if relative == Path::new("services/lambda/meta.ss.zst") { + lambda = true; + (Kind::Lambda, input.functions.to_vec()) + } else if relative == Path::new("services/stepfunctions/meta.ss.zst") { + registry = true; + (Kind::Workflows, input.executions.clone()) + } else { + (Kind::Other, Vec::new()) + }; + let compressed = path.extension().is_some_and(|v| v == "zst"); + fresh(deadline)?; + scan = owned_cpu( + deadline, + spawn_blocking(move || inspect(scan, bytes, compressed, kind, &expected)), + ) + .await?; + } + ensure!( + lambda && registry, + "callback saved service metadata missing" + ); + fresh(deadline)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use nix::{sys::stat::Mode, unistd::mkfifo}; + use serde_json::Value; + use tokio::net::UnixListener; + + use super::*; + + fn digest(bytes: &[u8]) -> Result { + hex_digest(bytes) + } + fn scan(hashes: &[String]) -> Result { + Scan::new(hashes, 0, 0, Instant::now() + Duration::from_secs(10)) + } + #[test] + fn test_should_detect_every_embedded_and_chunk_split_token() -> Result<()> { + let token = b"0123456789abcdef".repeat(4); + for offset in 0..64 { + let mut bytes = vec![b'a'; offset]; + bytes.extend_from_slice(&token); + bytes.extend_from_slice(b"ffff"); + assert!(scan(&[digest(&token)?])?.scan(&bytes).is_err()); + } + let encoded = zstd::stream::encode_all(Cursor::new(&token), 1)?; + let decoded = plain(&encoded)?; + assert!(scan(&[digest(&token)?])?.scan(&decoded).is_err()); + Ok(()) + } + #[test] + fn test_should_charge_overlapping_candidates_across_all_sources_and_fail_closed() -> Result<()> + { + let mut state = scan(&[digest(b"not a token")?])?; + state.scan(&vec![b'a'; CANDIDATES + 63])?; + assert_eq!(state.candidates, CANDIDATES); + assert!(state.scan(&[b'c'; 64]).is_err()); + assert!(state.scan(b"safe").is_err()); + assert!(Scan::new(&["invalid".into()], 0, 0, Instant::now()).is_err()); + assert!(Scan::new(&[digest(b"safe")?], CANDIDATES + 1, 0, Instant::now()).is_err()); + Ok(()) + } + #[test] + fn test_should_refuse_decoder_expansion_and_late_or_oversized_observations() -> Result<()> { + let encoded = zstd::stream::encode_all(Cursor::new(vec![0; PLAIN + 1]), 1)?; + assert!(plain(&encoded).is_err()); + assert!( + scan(&[digest(b"safe")?])? + .scan(&vec![0; PLAIN + 1]) + .is_err() + ); + assert!( + Scan::new(&[digest(b"safe")?], 0, 0, Instant::now())? + .scan(b"safe") + .is_err() + ); + Ok(()) + } + #[tokio::test] + async fn test_should_reject_linked_special_deep_and_excessive_archive_entries() -> Result<()> { + let root = tempfile::tempdir()?; + fs::symlink("/outside", root.path().join("link")).await?; + assert!( + files(root.path(), Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + fs::remove_file(root.path().join("link")).await?; + let socket = UnixListener::bind(root.path().join("socket"))?; + assert!( + files(root.path(), Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + drop(socket); + fs::remove_file(root.path().join("socket")).await?; + let mut deep = root.path().to_owned(); + for _ in 0..9 { + deep.push("nested"); + } + fs::create_dir_all(&deep).await?; + assert!( + files(root.path(), Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + fs::remove_dir_all(root.path().join("nested")).await?; + for index in 0..65 { + fs::write(root.path().join(index.to_string()), b"").await?; + } + assert!( + files(root.path(), Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + Ok(()) + } + #[test] + fn test_should_reject_both_private_sdk_values_with_redacted_errors() -> Result<()> { + for value in [b"local-key".as_slice(), b"local-secret".as_slice()] { + let error = scan(&[digest(b"safe")?])? + .scan(value) + .err() + .context("credential leak accepted")?; + assert_eq!(error.to_string(), "injected SDK credential persisted"); + } + Ok(()) + } + #[test] + fn test_should_require_framed_actual_terminal_execution_rows() -> Result<()> { + let arn = "arn:aws:states:us-east-1:000000000000:execution:machine:case"; + let expected = vec![arn.to_owned()]; + let frame = |value: &Value| -> Result> { plain(&archive::encode(value)?) }; + let empty = serde_json::json!({"schemaVersion":1,"executions":[],"unrelated":arn}); + assert!(workflows(&frame(&empty)?, &expected).is_err()); + let mut actual = serde_json::json!({"schemaVersion":1,"executions":[{ + "arn":arn,"stoppedAt":"2026-10-30T00:00:00Z","outcome":{"status":"SUCCEEDED","output":"42"} + }]}); + let valid = frame(&actual)?; + workflows(&valid, &expected)?; + assert!(workflows(arn.as_bytes(), &expected).is_err()); + let mut bad_crc = valid.clone(); + *bad_crc.get_mut(64).context("CRC corruption")? ^= 1; + assert!(workflows(&bad_crc, &expected).is_err()); + actual["executions"][0]["outcome"]["status"] = "RUNNING".into(); + assert!(workflows(&frame(&actual)?, &expected).is_err()); + actual["executions"][0]["outcome"]["status"] = "FAILED".into(); + actual["executions"][0]["stoppedAt"] = Value::Null; + assert!(workflows(&frame(&actual)?, &expected).is_err()); + Ok(()) + } + #[tokio::test] + async fn test_should_reject_replaced_fifo_before_any_blocking_read() -> Result<()> { + let root = tempfile::tempdir()?; + let path = root.path().join("server.log"); + fs::write(&path, b"safe").await?; + let deadline = Instant::now() + Duration::from_secs(2); + assert_eq!( + files(root.path(), deadline).await?, + vec![fs::canonicalize(&path).await?] + ); + fs::remove_file(&path).await?; + mkfifo(&path, Mode::from_bits_truncate(0o600))?; + assert!(read(&path, 1024, deadline).await.is_err()); + assert!(Instant::now() < deadline); + Ok(()) + } + #[tokio::test] + async fn test_should_reject_completed_worker_observed_at_cutoff() -> Result<()> { + let job = spawn_blocking(|| Ok(42)); + while !job.is_finished() { + tokio::task::yield_now().await; + } + let deadline = Instant::now(); + assert!(owned_cpu(deadline, job).await.is_err()); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/container_server.rs b/apps/rustack/tests/docker_startup/container_server.rs new file mode 100644 index 00000000..0e288b57 --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server.rs @@ -0,0 +1,338 @@ +//! Actual Linux server container; normal production readiness and retirement. +#![warn( + clippy::pedantic, + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +#[path = "container_server/cli.rs"] +mod cli; +#[path = "container_server/image_profile.rs"] +mod image_profile; +#[path = "container_server/logs.rs"] +mod logs; +#[path = "container_server/owner.rs"] +mod owner; +#[path = "container_server/policy.rs"] +mod policy; +#[path = "container_server/records.rs"] +mod records; + +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsString, + path::PathBuf, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +pub(super) use image_profile::ImageProfile; +use owner::Owner; +use serde::Deserialize; +use tokio::time::Instant; + +use super::{ + image_sdk::join_with_pid_before, + runner::{Runner, interpreter}, +}; + +#[derive(Debug, Clone, Copy)] +pub(super) enum Mode { + Sdk, + Unreachable, +} +impl Mode { + fn gateway_port(self) -> u16 { + match self { + Self::Sdk => 4566, + Self::Unreachable => 9, + } + } + fn name(self) -> &'static str { + match self { + Self::Sdk => "sdk-nested", + Self::Unreachable => "gateway-rejection", + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Versions { + s3: String, + dynamodb: String, + node: String, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Evidence { + schema_version: u8, + cases: Vec, + versions: Versions, + computed: Vec, + counters: Vec, + warm: BTreeMap, + parent_execution: String, + child_execution: String, +} + +pub(super) async fn exercise(mode: Mode, profile: ImageProfile) -> Result<()> { + let cutoff = Instant::now() + Duration::from_secs(600); + let mut owner = Owner::new(mode, profile, cutoff).await?; + let observed = async { + owner.prepare(cutoff).await?; + match mode { + Mode::Sdk => { + owner.ready(cutoff).await?; + sdk(&owner, cutoff).await?; + } + Mode::Unreachable => owner.rejection(cutoff).await?, + } + Ok::<(), anyhow::Error>(()) + } + .await; + // No outer timeout drops a live CLI wait or the root's cleanup authority. + let cleaned = owner.cleanup().await; + observed?; + cleaned?; + if profile == ImageProfile::Release { + let receipt = owner.publication_receipt(mode.name()).await?; + println!( + "WORKFLOWS_RELEASE_RESOURCE {}", + serde_json::to_string(&receipt)? + ); + } + owner.release(); + let topology = if profile == ImageProfile::Release { + "release-image server-container" + } else { + "server-container" + }; + println!( + "actual {topology} linux/{}: {}", + owner.inputs.architecture, + mode.name() + ); + Ok(()) +} + +async fn sdk(owner: &Owner, cutoff: Instant) -> Result<()> { + let identity = owner.identity()?; + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/server_container_sdk.py"); + let architecture = if owner.inputs.architecture == "arm64" { + "arm64" + } else { + "x86_64" + }; + let socket = owner + .inputs + .client + .strip_prefix("unix://") + .context("client socket prefix")?; + let args: Vec = [ + script.into_os_string(), + owner.endpoint()?.into(), + owner.inputs.sdk_image.clone().into(), + architecture.into(), + socket.into(), + identity.owner.to_string().into(), + identity.engine.to_string().into(), + format!("http://{}:4566/", owner.expected.name).into(), + owner.source(&owner.inputs.sdk_image)?.into(), + owner.source(&owner.inputs.relay)?.into(), + owner.expected.network.clone().into(), + ] + .into(); + let child = Runner::spawn(&interpreter()?, &args)?; + let (entered, report) = join_with_pid_before(child, cutoff).await?; + ensure!( + report.pid == entered.context("server SDK child did not publish its execution PID")? + && !report.forced, + "server SDK process failed its execution barrier" + ); + if !report.status.success() { + anyhow::bail!( + "server SDK consumer failed: {} ({})", + failure_stage(&report.stderr), + failure_origin(&report.stderr) + ); + } + let observed: Evidence = cli::decode(&report.stdout)?; + validate(&observed)?; + let actual: BTreeSet<_> = owner.runtime_ids(cutoff).await?.into_iter().collect(); + let reported: BTreeSet<_> = observed.warm.values().cloned().collect(); + ensure!( + actual == reported && actual.len() == 2, + "warm server runtime pair did not survive until shutdown" + ); + Ok(()) +} + +fn failure_stage(stderr: &[u8]) -> &'static str { + let Ok(text) = std::str::from_utf8(stderr) else { + return "invalid diagnostic"; + }; + let Some(stage) = text + .lines() + .next() + .and_then(|line| line.strip_prefix("server-container SDK oracle failed at ")) + else { + return "unclassified diagnostic"; + }; + match stage { + "input" => "input", + "create-data" => "create-data", + "create-function" => "create-function", + "direct-first" => "direct-first", + "direct-second" => "direct-second", + "nested" => "nested", + "delete-data" => "delete-data", + _ => "invalid diagnostic stage", + } +} + +fn failure_origin(stderr: &[u8]) -> String { + let origin = std::str::from_utf8(stderr) + .ok() + .filter(|text| text.len() <= 512 && text.lines().count() == 2) + .and_then(|text| text.lines().nth(1)) + .and_then(|line| line.strip_prefix("origin ")) + .and_then(|value| value.split_once(':')); + if let Some((name, line)) = origin { + let allowed = [ + "server_container_sdk.py", + "server_container_oracle.py", + "handler_sdk_image.py", + "lambda_fault_support.py", + "stepfunctions_history.py", + "stepfunctions_nested.py", + ]; + if allowed.contains(&name) + && line + .parse::() + .is_ok_and(|number| (1..=10000).contains(&number) && number.to_string() == line) + { + return format!("{name}:{line}"); + } + } + "unknown source coordinate".to_owned() +} + +fn validate(evidence: &Evidence) -> Result<()> { + ensure!( + evidence.schema_version == 1 + && evidence.cases.iter().map(String::as_str).eq([ + "direct-sdk-cold", + "direct-sdk-warm", + "nested-sync2-sdk", + "internal-dns-origin", + "no-server-mounts", + "warm-for-shutdown" + ]) + && evidence.versions.s3 == "3.1105.0" + && evidence.versions.dynamodb == "3.1105.0" + && evidence.versions.node == "24.22.1" + && evidence.computed == [42, 46, 50] + && evidence.counters == [1, 2, 3] + && evidence.warm.len() == 2 + && evidence.warm.contains_key("function") + && evidence.warm.contains_key("relay"), + "server SDK catalog or literal outputs differ" + ); + let ids = evidence + .warm + .values() + .map(|id| cli::container_id(id)) + .collect::>>()?; + ensure!( + ids.len() == 2, + "server SDK reported duplicate environment IDs" + ); + execution(&evidence.parent_execution, "ServerParent")?; + execution(&evidence.child_execution, "ServerChild")?; + ensure!( + evidence.parent_execution != evidence.child_execution, + "nested execution identity reused" + ); + Ok(()) +} + +fn execution(value: &str, expected: &str) -> Result<()> { + ensure!(value.len() <= 256, "execution ARN byte bound"); + let tail = value + .strip_prefix("arn:aws:states:us-east-1:000000000000:execution:") + .context("execution ARN scope")?; + let (name, run) = tail.split_once(':').context("execution ARN resource")?; + ensure!( + name.starts_with(expected) + && [name, run].iter().all(|part| (1..=80).contains(&part.len()) + && part + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b))), + "execution ARN name grammar" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_reject_invalid_daemon_socket_and_gid_before_cli_dispatch() -> Result<()> { + for source in [ + "", + "/", + "relative/docker.sock", + "/run//docker.sock", + "/run/../docker.sock", + "/run/./docker.sock", + "/run/docker.sock/", + "/run/a,b", + "/run/a\n", + ] { + assert!(policy::socket_source(source).is_err()); + } + policy::socket_source("/run/user/1000/docker.sock")?; + for gid in ["", "-1", "01", "2147483648", "4294967296", "991\n"] { + assert!(policy::socket_gid(gid).is_err()); + } + assert_eq!(policy::socket_gid("0")?, 0); + assert_eq!(policy::socket_gid("991")?, 991); + Ok(()) + } + + #[test] + fn test_should_never_project_untrusted_sdk_diagnostics() { + assert_eq!(failure_origin(b"server-container SDK oracle failed at direct-first\norigin server_container_oracle.py:50\n"), "server_container_oracle.py:50"); + for origin in [ + "local-secret:50", + "server_container_oracle.py:local-secret", + "server_container_oracle.py:050", + "server_container_oracle.py:0", + "server_container_oracle.py:10001", + "server_container_oracle.py:50\nlocal-secret", + ] { + let diagnostic = + format!("server-container SDK oracle failed at nested\norigin {origin}\n"); + assert_eq!( + failure_origin(diagnostic.as_bytes()), + "unknown source coordinate" + ); + } + assert_eq!( + failure_stage(b"server-container SDK oracle failed at nested\n"), + "nested" + ); + for bytes in [ + b"raw-private-diagnostic".as_slice(), + b"server-container SDK oracle failed at arbitrary\n", + b"\xff", + ] { + assert!(!["raw-private-diagnostic", "arbitrary"].contains(&failure_stage(bytes))); + } + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/cli.rs b/apps/rustack/tests/docker_startup/container_server/cli.rs new file mode 100644 index 00000000..0e58d9df --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/cli.rs @@ -0,0 +1,151 @@ +//! Direct bounded CLI observers; every child is joined before interpretation. + +use std::{ + ffi::OsString, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use rustack_core::json::{JsonLimits, from_slice}; +use serde::de::DeserializeOwned; +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::super::runner::{Report, Runner}; + +#[derive(Debug)] +pub(super) struct Cli { + program: PathBuf, + socket: String, + pub engine: Uuid, +} + +impl Cli { + pub async fn connect(program: PathBuf, socket: String, cutoff: Instant) -> Result { + let bytes = command(&program, &socket, &info_arguments(), cutoff).await?; + let engine = uuid(&decode::(&bytes.stdout)?)?; + Ok(Self { + program, + socket, + engine, + }) + } + + pub async fn run(&self, args: &[String], cutoff: Instant) -> Result> { + Ok(self.output(args, cutoff).await?.stdout) + } + + pub async fn output(&self, args: &[String], cutoff: Instant) -> Result { + let at = command_deadline(args, cutoff); + self.require_engine(at).await?; + let bytes = command(&self.program, &self.socket, args, at).await?; + self.require_engine(at).await?; + ensure!(Instant::now() < at, "late Docker CLI observation"); + Ok(bytes) + } + + pub async fn json(&self, args: &[String], cutoff: Instant) -> Result { + decode(&self.run(args, cutoff).await?) + } + + async fn require_engine(&self, cutoff: Instant) -> Result<()> { + let bytes = command(&self.program, &self.socket, &info_arguments(), cutoff).await?; + ensure!( + uuid(&decode::(&bytes.stdout)?)? == self.engine, + "fixture Docker daemon identity changed" + ); + Ok(()) + } +} + +pub(super) fn decode(raw: &[u8]) -> Result { + // Docker configuration paths/env entries require4KiB strings; the whole + // response, depth, keys and aggregate members retain independent caps. + Ok(from_slice( + raw, + JsonLimits::new(32768, 16, 2048, 4096, 256)?, + )?) +} + +pub(super) fn uuid(text: &str) -> Result { + ensure!( + text.len() == 36 + && text + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b) || b == b'-'), + "invalid fixture UUID grammar" + ); + let id = Uuid::parse_str(text)?; + ensure!( + !id.is_nil() && id.to_string() == text, + "noncanonical fixture UUID" + ); + Ok(id) +} + +pub(super) fn container_id(text: &str) -> Result { + ensure!( + text.len() == 64 + && text + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "invalid full container ID" + ); + Ok(text.to_owned()) +} + +pub(super) fn arguments(parts: &[&str]) -> Vec { + parts.iter().map(|part| (*part).to_owned()).collect() +} + +fn info_arguments() -> Vec { + arguments(&["info", "--format", "{{json .ID}}"]) +} + +fn command_deadline(args: &[String], cutoff: Instant) -> Instant { + let seconds = if args.first().is_some_and(|arg| arg == "wait") { + 70 + } else { + 60 + }; + cutoff.min(Instant::now() + Duration::from_secs(seconds)) +} + +async fn command(program: &Path, socket: &str, args: &[String], cutoff: Instant) -> Result { + let cutoff = command_deadline(args, cutoff); + ensure!( + Instant::now() < cutoff, + "Docker CLI deadline expired before dispatch" + ); + ensure!( + (1..=96).contains(&args.len()) + && args + .iter() + .all(|arg| arg.len() <= 4096 && !arg.chars().any(char::is_control)), + "Docker CLI argv boundary" + ); + let mut invocation = vec![OsString::from("--host"), OsString::from(socket)]; + invocation.extend(args.iter().map(OsString::from)); + let child = Runner::spawn(program, &invocation)?; + let report = child + .wait(cutoff) + .await + .context("join exact Docker CLI child")?; + // stderr may contain daemon-selected text. Preserve only its digest in + // diagnostics, never a response body or a private env-file value. + ensure!( + !report.forced && report.status.success(), + "Docker CLI failed (exit {:?}, forced {}, stderr SHA256/base64 {})", + report.status.code(), + report.forced, + STANDARD.encode(Sha256::digest(&report.stderr)) + ); + ensure!( + Instant::now() < cutoff, + "Docker CLI result expired after join" + ); + Ok(report) +} diff --git a/apps/rustack/tests/docker_startup/container_server/image_profile.rs b/apps/rustack/tests/docker_startup/container_server/image_profile.rs new file mode 100644 index 00000000..859795bd --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/image_profile.rs @@ -0,0 +1,164 @@ +//! Exact image contracts; release health remains enabled and independently observed. + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::models::{ + ContainerInspectResponse, HealthConfig, HealthStatusEnum, +}; +use serde::Deserialize; +use tokio::time::Instant; + +use super::cli::{Cli, arguments}; + +const CONTRACT: &str = r#"{"user":{{json .Config.User}},"workingDir":{{json .Config.WorkingDir}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"health":{{json .Config.Healthcheck}}}"#; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImageProfile { + Fixture, + Release, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Contract { + user: String, + working_dir: String, + entrypoint: Vec, + cmd: Option>, + health: Option, +} + +impl ImageProfile { + pub(super) fn variable(self) -> &'static str { + match self { + Self::Fixture => "RUSTACK_SERVER_IMAGE", + Self::Release => "RUSTACK_RELEASE_IMAGE", + } + } + pub(super) fn configuration(self) -> &'static str { + match self { + Self::Fixture => "/etc/rustack/config.yaml", + Self::Release => "/data/qualification.yaml", + } + } + pub(super) async fn source(self, cli: &Cli, id: &str, cutoff: Instant) -> Result<()> { + let contract: Contract = cli + .json( + &arguments(&["image", "inspect", "--format", CONTRACT, id]), + cutoff, + ) + .await?; + ensure!( + contract.user == "65532:65532" + && contract.working_dir == "/data" + && contract + .entrypoint + .iter() + .map(String::as_str) + .eq(["/rustack"]) + && contract.cmd.as_ref().is_none_or(Vec::is_empty), + "server image execution contract differs" + ); + self.health(contract.health.as_ref()) + } + pub(super) fn health(self, health: Option<&HealthConfig>) -> Result<()> { + let health = health.context("server image healthcheck missing")?; + let command = health + .test + .as_ref() + .context("server image health command missing")?; + match self { + Self::Fixture => ensure!( + command.iter().map(String::as_str).eq(["NONE"]) + && [ + health.interval, + health.timeout, + health.retries, + health.start_period, + health.start_interval + ] + .iter() + .all(|v| v.is_none_or(|v| v == 0)), + "fixture image health differs" + ), + Self::Release => ensure!( + command + .iter() + .map(String::as_str) + .eq(["CMD", "/rustack", "--health-check"]) + && health.interval == Some(2_000_000_000) + && health.timeout == Some(4_000_000_000) + && health.start_period == Some(1_000_000_000) + && health.retries == Some(3) + && health.start_interval.is_none_or(|v| v == 0), + "release image health differs" + ), + } + Ok(()) + } + pub(super) fn healthy(self, view: &ContainerInspectResponse) -> bool { + self == Self::Fixture + || view + .state + .as_ref() + .and_then(|s| s.health.as_ref()) + .and_then(|h| h.status) + == Some(HealthStatusEnum::HEALTHY) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{super::cli::decode, *}; + + #[test] + fn test_should_require_exact_release_exec_health_without_shell_or_disable() -> Result<()> { + let value = json!({"Test":["CMD","/rustack","--health-check"],"Interval":2_000_000_000_i64, + "Timeout":4_000_000_000_i64,"StartPeriod":1_000_000_000_i64,"Retries":3}); + ImageProfile::Release + .health(Some(&decode::(&serde_json::to_vec(&value)?)?))?; + for (field, replacement) in [ + ("Test", json!(["NONE"])), + ("Test", json!(["CMD-SHELL", "/rustack --health-check"])), + ("Interval", json!(0)), + ("Timeout", json!(1)), + ("StartPeriod", json!(0)), + ("Retries", json!(1)), + ("StartInterval", json!(1)), + ] { + let mut changed = value.clone(); + changed + .as_object_mut() + .context("health fixture")? + .insert(field.into(), replacement); + assert!( + ImageProfile::Release + .health(Some(&decode::(&serde_json::to_vec( + &changed + )?)?)) + .is_err() + ); + } + assert!(ImageProfile::Release.health(None).is_err()); + Ok(()) + } + + #[test] + fn test_should_keep_fixture_none_and_require_actual_release_healthy_state() -> Result<()> { + let none: HealthConfig = decode(br#"{"Test":["NONE"]}"#)?; + ImageProfile::Fixture.health(Some(&none))?; + assert!(ImageProfile::Release.health(Some(&none)).is_err()); + for state in ["starting", "unhealthy", "none", ""] { + let view: ContainerInspectResponse = decode(&serde_json::to_vec( + &json!({"State":{"Health":{"Status":state}}}), + )?)?; + assert!(ImageProfile::Fixture.healthy(&view)); + assert!(!ImageProfile::Release.healthy(&view)); + } + let view: ContainerInspectResponse = + decode(br#"{"State":{"Health":{"Status":"healthy"}}}"#)?; + assert!(ImageProfile::Release.healthy(&view)); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/logs.rs b/apps/rustack/tests/docker_startup/container_server/logs.rs new file mode 100644 index 00000000..952ecdfb --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/logs.rs @@ -0,0 +1,89 @@ +//! D97's raw plus decoded-JSON-token privacy profile, bounded for server logs. + +use anyhow::{Context, Result, ensure}; +use rustack_core::json::{JsonLimits, from_slice}; + +pub(super) fn check(bytes: &[u8]) -> Result<()> { + ensure!(bytes.len() <= 65536, "server diagnostic byte cap"); + std::str::from_utf8(bytes).context("invalid server diagnostic encoding")?; + private(bytes)?; + let mut start = None; + let mut escaped = false; + let mut tokens = 0usize; + let mut decoded = 0usize; + for (offset, byte) in bytes.iter().enumerate() { + let Some(begin) = start else { + if *byte == b'"' { + start = Some(offset); + } + continue; + }; + if *byte == b'"' && !escaped { + tokens = tokens + .checked_add(1) + .context("server diagnostic token count")?; + ensure!(tokens <= 4096, "server diagnostic token cap"); + let raw = bytes + .get(begin..=offset) + .context("server diagnostic token range")?; + let value: String = from_slice(raw, JsonLimits::new(65536, 1, 1, 65536, 256)?)?; + decoded = decoded + .checked_add(value.len()) + .context("server decoded diagnostic bytes")?; + ensure!(decoded <= 65536, "server decoded diagnostic byte cap"); + private(value.as_bytes())?; + start = None; + } + escaped = *byte == b'\\' && !escaped; + } + ensure!(start.is_none(), "unclosed server diagnostic JSON token"); + Ok(()) +} + +fn private(bytes: &[u8]) -> Result<()> { + ensure!( + ![b"local-key".as_slice(), b"local-secret"] + .iter() + .any(|needle| bytes.windows(needle.len()).any(|window| window == *needle)), + "server diagnostic credential leak" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_reject_raw_and_json_escaped_keys_and_credentials_without_echo() -> Result<()> { + for bytes in [ + b"local-key".as_slice(), + b"local-secret", + br#"{"\u006cocal-key":"ignored"}"#, + br#"{"credential":"\u006cocal-\u0073ecret"}"#, + br#"mixed> {"credential":"\u006cocal-key"}"#, + ] { + let error = check(bytes) + .err() + .context("escaped diagnostic leak accepted")?; + assert_eq!(error.to_string(), "server diagnostic credential leak"); + } + Ok(()) + } + #[test] + fn test_should_accept_safe_structured_and_mixed_diagnostics_with_escaped_quotes() -> Result<()> + { + check(br#"{"message":"quoted \"safe\" content","details":{"ok":true}}"#)?; + check(br#"plain> "trailing\\" "safe""#)?; + check(b"plain safe diagnostic\n")?; + Ok(()) + } + #[test] + fn test_should_fail_closed_on_malformed_or_over_budget_diagnostics() { + for bytes in [b"log \"unterminated".as_slice(), b"\xff", b"\"\x00\""] { + assert!(check(bytes).is_err()); + } + assert!(check(&vec![b'a'; 65537]).is_err()); + assert!(check("\"x\" ".repeat(4097).as_bytes()).is_err()); + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/owner.rs b/apps/rustack/tests/docker_startup/container_server/owner.rs new file mode 100644 index 00000000..bcee279a --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/owner.rs @@ -0,0 +1,498 @@ +//! One retained owner above every CLI/SDK observer and all daemon effects. + +#[path = "owner/cleanup.rs"] +mod cleanup; +#[path = "owner/configuration.rs"] +mod configuration; +#[path = "owner/data.rs"] +mod data; + +use std::{ + fs::Permissions, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::models::{ContainerInspectResponse, NetworkInspect, Volume}; +use serde::Deserialize; +use serde_json::json; +use tempfile::TempDir; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, + time::{Instant, sleep, timeout_at}, +}; + +use self::data::{DataOperation, DataOutput, commit_identity}; +use super::{ + ImageProfile, Mode, + cli::{self, Cli, arguments}, + policy::{self, Expected, Inputs}, + records::Identity, +}; + +const INSPECT: &str = r#"{"Id":{{json .Id}},"Name":{{json .Name}},"Image":{{json .Image}},"State":{{json .State}},"Config":{"Image":{{json .Config.Image}},"User":{{json .Config.User}},"WorkingDir":{{json .Config.WorkingDir}},"Entrypoint":{{json .Config.Entrypoint}},"Cmd":{{json .Config.Cmd}},"Labels":{{json .Config.Labels}},"Healthcheck":{{json .Config.Healthcheck}}},"HostConfig":{{json .HostConfig}},"NetworkSettings":{{json .NetworkSettings}},"Mounts":{{json .Mounts}}}"#; +const SOURCE: &str = + r#"{"id":{{json .Id}},"os":{{json .Os}},"architecture":{{json .Architecture}}}"#; + +#[derive(Debug)] +enum Resources { + Intent, + Volume(Volume), + Server { volume: Volume, id: String }, +} +#[derive(Debug, Clone, Copy)] +enum ExpectedExit { + Success, + EndpointRejected(i64), +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Source { + id: String, + os: String, + architecture: String, +} +#[derive(Debug)] +pub(super) struct Owner { + root: TempDir, + profile: ImageProfile, + path: PathBuf, + pub cli: Cli, + pub inputs: Inputs, + pub expected: Expected, + resources: Resources, + uncertain: bool, + identity: Option, + endpoint: Option, + expected_exit: ExpectedExit, + sources: Vec<(String, String)>, +} + +impl Owner { + pub async fn new(mode: Mode, profile: ImageProfile, cutoff: Instant) -> Result { + let inputs = Inputs::load(profile)?; + let root = tempfile::tempdir()?; + timeout_at( + cutoff, + fs::set_permissions(root.path(), Permissions::from_mode(0o700)), + ) + .await??; + let path = timeout_at(cutoff, fs::canonicalize(root.path())).await??; + let visible = path.to_str().context("private evidence path encoding")?; + ensure!( + visible.len() <= 4096 && !visible.chars().any(char::is_control), + "private evidence path bound" + ); + let cli = Cli::connect(inputs.program.clone(), inputs.client.clone(), cutoff).await?; + let [network]: [NetworkInspect; 1] = cli + .json(&arguments(&["network", "inspect", &inputs.network]), cutoff) + .await?; + let network = policy::network(&network, &inputs.network)?; + let mut sources = Vec::new(); + for reference in [&inputs.image, &inputs.sdk_image, &inputs.relay] { + let value: Source = cli + .json( + &arguments(&["image", "inspect", "--format", SOURCE, reference]), + cutoff, + ) + .await?; + ensure!( + value.os == "linux" && value.architecture == inputs.architecture, + "server fixture source platform differs" + ); + sources.push((reference.clone(), policy::source_id(&value.id)?)); + } + let source = sources.first().context("server source capture")?.1.clone(); + profile.source(&cli, &source, cutoff).await?; + let nonce = uuid::Uuid::new_v4().simple().to_string(); + let expected = Expected { + name: format!("rustack-server-{nonce}"), + volume: format!("rustack-server-data-{nonce}"), + nonce, + engine: cli.engine, + source, + network, + socket: inputs.source_socket.clone(), + gid: inputs.gid.clone(), + }; + let mut owner = Self { + root, + profile, + path, + cli, + inputs, + expected, + resources: Resources::Intent, + uncertain: false, + identity: None, + endpoint: None, + expected_exit: ExpectedExit::Success, + sources, + }; + owner.write_environment(mode, cutoff).await?; + owner.write("intent.json", &serde_json::to_vec(&json!({ + "schemaVersion":1,"engine":owner.expected.engine.to_string(),"nonce":owner.expected.nonce, + "server":owner.expected.name,"volume":owner.expected.volume,"source":owner.expected.source, + "network":owner.expected.network,"daemonSocket":owner.expected.socket,"socketGid":owner.expected.gid, + "sources":owner.sources + }))?, cutoff).await?; + owner.root.disable_cleanup(true); + eprintln!("server-container evidence: {}", owner.path.display()); + Ok(owner) + } + + pub async fn prepare(&mut self, cutoff: Instant) -> Result<()> { + let mut args = arguments(&["volume", "create", "--driver", "local"]); + for (key, value) in self.expected.labels("server-data") { + args.extend(["--label".into(), format!("{key}={value}")]); + } + args.push(self.expected.volume.clone()); + self.uncertain = true; + let reply = self.cli.run(&args, cutoff).await?; + ensure!( + reply == format!("{}\n", self.expected.volume).as_bytes(), + "volume creation reply differs" + ); + let data = self.inspect_volume(cutoff).await?; + policy::volume(&data, &self.expected)?; + self.resources = Resources::Volume(data.clone()); + self.write("volume.json", &serde_json::to_vec(&data)?, cutoff) + .await?; + self.uncertain = false; + let args = self.create_arguments()?; + self.uncertain = true; + let reply = self.cli.run(&args, cutoff).await?; + let text = std::str::from_utf8(&reply)? + .strip_suffix('\n') + .context("container creation newline")?; + let id = cli::container_id(text)?; + self.resources = Resources::Server { + volume: data, + id: id.clone(), + }; + self.write("server.json", &serde_json::to_vec(&json!({"id":id,"engine":self.expected.engine.to_string(), + "nonce":self.expected.nonce,"source":self.expected.source,"volume":self.expected.volume}))?, cutoff).await?; + self.inspect_server(cutoff).await?; + self.uncertain = false; + if self.profile == ImageProfile::Release { + self.install_public_configuration(cutoff).await?; + } + self.uncertain = true; + let started = self.cli.run(&arguments(&["start", &id]), cutoff).await?; + ensure!( + started == format!("{id}\n").as_bytes(), + "server start reply differs" + ); + self.uncertain = false; + Ok(()) + } + + fn create_arguments(&self) -> Result> { + let environment = self.path.join("server.env"); + let environment = environment + .to_str() + .context("private environment path encoding")?; + let mut args = arguments(&[ + "create", + "--pull=never", + "--name", + &self.expected.name, + "--platform", + &format!("linux/{}", self.inputs.architecture), + "--user", + "65532:65532", + "--group-add", + &self.expected.gid, + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--cgroupns", + "private", + "--memory", + "512m", + "--memory-swap", + "512m", + "--cpus", + "1", + "--pids-limit", + "128", + "--tmpfs", + &format!("/tmp:{}", policy::TEMPORARY), + "--log-driver", + "json-file", + "--log-opt", + "max-size=1m", + "--log-opt", + "max-file=1", + "--restart", + "no", + "--network", + &self.expected.network, + "--network-alias", + &self.expected.name, + "--publish", + "127.0.0.1::4566", + "--env-file", + environment, + "--mount", + &format!( + "type=bind,source={},target=/run/rustack/docker.sock,readonly", + self.expected.socket + ), + "--mount", + &format!("type=volume,source={},target=/data", self.expected.volume), + ]); + for (key, value) in self.expected.labels("server") { + args.extend(["--label".into(), format!("{key}={value}")]); + } + args.push(self.expected.source.clone()); + Ok(args) + } + + pub async fn ready(&mut self, cutoff: Instant) -> Result<()> { + let at = cutoff.min(Instant::now() + Duration::from_secs(180)); + loop { + let view = self.inspect_server(at).await?; + ensure!( + view.state.as_ref().and_then(|s| s.running) == Some(true), + "server exited before readiness" + ); + let endpoint = policy::endpoint(&view, &self.expected)?; + self.endpoint = Some(endpoint.clone()); + if let Ok((status, value)) = + super::super::process::get(&endpoint, "/_rustack/capabilities").await + { + if value.get("ready").and_then(serde_json::Value::as_bool) == Some(true) + && self.profile.healthy(&view) + { + ensure!( + status == 200 + && value + .pointer("/workers/docker/ready") + .and_then(serde_json::Value::as_bool) + == Some(true) + && value + .pointer("/workers/docker/sdkGateway") + .and_then(serde_json::Value::as_bool) + == Some(true) + && value + .pointer("/workflows/lambdaRequestResponse") + .and_then(serde_json::Value::as_bool) + == Some(true) + && value + .pointer("/workflows/nestedSync") + .and_then(serde_json::Value::as_bool) + == Some(true) + && value + .pointer("/workflows/stateTypes") + .and_then(serde_json::Value::as_array) + .is_some_and(|s| s.len() == 8) + && value + .pointer("/workers/docker/platform") + .and_then(serde_json::Value::as_str) + == Some(format!("linux/{}", self.inputs.architecture).as_str()), + "false server-container readiness" + ); + ensure!(Instant::now() < at, "late server-container readiness"); + self.capture_identity(at).await?; + return Ok(()); + } + } + ensure!(Instant::now() < at, "server-container readiness deadline"); + sleep(Duration::from_millis(100)).await; + } + } + + pub async fn rejection(&mut self, cutoff: Instant) -> Result<()> { + let at = cutoff.min(Instant::now() + Duration::from_secs(180)); + loop { + let view = self.inspect_server(at).await?; + let state = view.state.as_ref().context("server startup state")?; + if state.running == Some(false) { + let code = state.exit_code.context("server startup exit code")?; + ensure!( + code != 0 && state.oom_killed == Some(false), + "unrelated server startup exit" + ); + let logs = self.logs(at).await?; + let text = std::str::from_utf8(&logs)?; + ensure!( + text.contains("Docker startup endpoint qualification failed"), + "server rejected an unrelated startup prerequisite" + ); + self.capture_identity(at).await?; + self.expected_exit = ExpectedExit::EndpointRejected(code); + return Ok(()); + } + if let Ok(endpoint) = policy::endpoint(&view, &self.expected) { + if let Ok((_, value)) = + super::super::process::get(&endpoint, "/_rustack/capabilities").await + { + ensure!( + value.get("ready").and_then(serde_json::Value::as_bool) != Some(true), + "unreachable server gateway became ready" + ); + } + } + ensure!(Instant::now() < at, "server gateway rejection deadline"); + sleep(Duration::from_millis(100)).await; + } + } + + pub fn endpoint(&self) -> Result<&str> { + self.endpoint + .as_deref() + .context("qualified server endpoint") + } + pub fn identity(&self) -> Result { + self.identity + .context("verified server installation identity") + } + pub fn source(&self, reference: &str) -> Result<&str> { + self.sources + .iter() + .find(|(name, _)| name == reference) + .map(|(_, id)| id.as_str()) + .context("captured source ID") + } + pub async fn publication_receipt(&self, mode: &str) -> Result { + let at = Instant::now() + Duration::from_secs(10); + let mut documents = Vec::new(); + for name in ["server.json", "shutdown.json", "cleanup.json"] { + let value = timeout_at(at, async { + let file = fs::File::open(self.path.join(name)).await?; + let mut bytes = Vec::new(); + file.take(8193).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 8192, "publication receipt bound"); + cli::decode::(&bytes) + }) + .await??; + documents.push(value); + } + let mut documents = documents.into_iter(); + let server = documents.next().context("server receipt")?; + let shutdown = documents.next().context("shutdown receipt")?; + let cleanup = documents.next().context("cleanup receipt")?; + Ok(json!({"schemaVersion":1,"profile":"release","mode":mode, + "architecture":self.inputs.architecture,"server":server,"shutdown":shutdown,"cleanup":cleanup})) + } + pub fn release(&mut self) { + self.root.disable_cleanup(false); + } + + async fn inspect_volume(&self, cutoff: Instant) -> Result { + let [volume]: [Volume; 1] = self + .cli + .json( + &arguments(&["volume", "inspect", &self.expected.volume]), + cutoff, + ) + .await?; + Ok(volume) + } + async fn inspect_server(&self, cutoff: Instant) -> Result { + let Resources::Server { volume, id } = &self.resources else { + anyhow::bail!("server container not captured"); + }; + let view: ContainerInspectResponse = self + .cli + .json( + &arguments(&["container", "inspect", "--format", INSPECT, id]), + cutoff, + ) + .await?; + ensure!( + view.id.as_deref() == Some(id.as_str()), + "server inspected a different container" + ); + policy::server(&view, &self.expected, volume)?; + self.profile + .health(view.config.as_ref().and_then(|c| c.healthcheck.as_ref()))?; + Ok(view) + } + async fn capture_identity(&mut self, cutoff: Instant) -> Result<()> { + self.inspect_server(cutoff).await?; + let archives = match self.transfer_data(DataOperation::Identity, cutoff).await { + Ok(DataOutput::Identity(archives)) => Ok(archives), + Ok(DataOutput::Configuration(_)) => { + Err(anyhow::anyhow!("incorrect data-volume identity operation")) + } + Err(error) => Err(error), + }; + commit_identity( + &mut self.uncertain, + &mut self.identity, + archives, + self.cli.engine, + &self.path, + cutoff, + ) + .await + } + async fn logs(&self, cutoff: Instant) -> Result> { + let Resources::Server { id, .. } = &self.resources else { + anyhow::bail!("server logs without container"); + }; + let output = self.cli.output(&arguments(&["logs", id]), cutoff).await?; + let mut bytes = output.stdout; + bytes.extend_from_slice(&output.stderr); + super::logs::check(&bytes)?; + Ok(bytes) + } + async fn write(&self, name: &str, bytes: &[u8], cutoff: Instant) -> Result<()> { + write_private(&self.path, name, bytes, cutoff).await + } + async fn write_environment(&self, mode: Mode, cutoff: Instant) -> Result<()> { + let relay = self.source(&self.inputs.relay)?; + let configuration = self.profile.configuration(); + let text = format!( + concat!( + "SERVICES=lambda,stepfunctions,s3,dynamodb\n", + "GATEWAY_LISTEN=0.0.0.0:4566\n", + "RUSTACK_CONFIG={configuration}\n", + "DATA_DIR=/data/state\n", + "LAMBDA_EXECUTOR=docker\n", + "LAMBDA_DOCKER_SOCKET=unix:///run/rustack/docker.sock\n", + "LAMBDA_DOCKER_RELAY_IMAGE={relay}\n", + "LAMBDA_DOCKER_NETWORK={network}\n", + "LAMBDA_DOCKER_GATEWAY_ENDPOINT=http://{server}:{port}/\n", + "LAMBDA_DOCKER_CROSS_ARCHITECTURE=false\n", + "LAMBDA_INIT_TIMEOUT_SECS=20\n", + "LOG_LEVEL=error\n", + "LAMBDA_SKIP_SIGNATURE_VALIDATION=false\n", + "STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION=false\n", + "S3_SKIP_SIGNATURE_VALIDATION=false\n", + "DYNAMODB_SKIP_SIGNATURE_VALIDATION=false\n", + "ACCESS_KEY=local-key\n", + "SECRET_KEY=local-secret\n", + "AWS_ACCESS_KEY_ID=local-key\n", + "AWS_SECRET_ACCESS_KEY=local-secret\n", + "AWS_EC2_METADATA_DISABLED=true\n", + "LAMBDA_DOCKER_SDK_ACCESS_KEY_ID=local-key\n", + "LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY=local-secret\n", + ), + configuration = configuration, + relay = relay, + network = self.expected.network, + server = self.expected.name, + port = mode.gateway_port(), + ); + self.write("server.env", text.as_bytes(), cutoff).await + } +} + +async fn write_private(directory: &Path, name: &str, bytes: &[u8], cutoff: Instant) -> Result<()> { + let at = cutoff.min(Instant::now() + Duration::from_secs(10)); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true).mode(0o600); + let mut file = timeout_at(at, options.open(directory.join(name))).await??; + timeout_at(at, file.write_all(bytes)).await??; + timeout_at(at, file.sync_all()).await??; + ensure!(Instant::now() < at, "private server evidence write expired"); + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/container_server/owner/cleanup.rs b/apps/rustack/tests/docker_startup/container_server/owner/cleanup.rs new file mode 100644 index 00000000..03d60df5 --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/owner/cleanup.rs @@ -0,0 +1,335 @@ +//! Shutdown receipt precedes namespace audit and exact stopped-resource removal. + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use serde::Deserialize; +use serde_json::json; +use tokio::time::Instant; + +use super::{ + super::{ + cli::{self, arguments}, + policy, + }, + ExpectedExit, Owner, Resources, SOURCE, Source, +}; + +#[derive(Debug, Deserialize)] +struct ImageRow { + #[serde(rename = "Repository")] + repository: String, +} + +impl Owner { + pub(crate) async fn cleanup(&mut self) -> Result<()> { + let cutoff = Instant::now() + Duration::from_secs(180); + if matches!(self.resources, Resources::Server { .. }) { + self.stop_server(cutoff).await?; + ensure!( + !self.uncertain, + "uncertain server mutation remains retained" + ); + if self.identity.is_none() { + self.capture_identity(cutoff).await?; + } + self.audit_runtime(cutoff).await?; + self.remove_server(cutoff).await?; + } + ensure!( + !self.uncertain, + "uncertain server/volume creation remains retained" + ); + if let Resources::Volume(_) = self.resources { + self.remove_volume(cutoff).await?; + } + self.verify_sources(cutoff).await?; + self.write( + "cleanup.json", + &serde_json::to_vec(&json!({ + "schemaVersion":1,"serverAbsent":true,"volumeAbsent":true, + "uncertain":false,"sourcesPreserved":self.sources, + "engine":self.expected.engine.to_string() + }))?, + cutoff, + ) + .await?; + Ok(()) + } + + async fn stop_server(&mut self, cutoff: Instant) -> Result<()> { + let at = cutoff.min(Instant::now() + Duration::from_secs(70)); + let view = self.inspect_server(at).await?; + let id = view.id.as_deref().context("owned server shutdown ID")?; + let state = view.state.as_ref().context("server shutdown state")?; + if state.running == Some(true) { + let prior = self.uncertain; + self.uncertain = true; + let stopped = self + .cli + .run(&arguments(&["kill", "--signal", "TERM", id]), at) + .await?; + ensure!( + stopped == format!("{id}\n").as_bytes(), + "server signal reply differs" + ); + self.uncertain = prior; + } + let output = self.cli.run(&arguments(&["wait", id]), at).await?; + let text = std::str::from_utf8(&output)? + .strip_suffix('\n') + .context("server wait newline")?; + let code: i64 = text.parse()?; + ensure!( + (0..=255).contains(&code) && code.to_string() == text, + "invalid server exit receipt" + ); + let expected = match self.expected_exit { + ExpectedExit::Success => 0, + ExpectedExit::EndpointRejected(code) => code, + }; + let actual = self.inspect_server(at).await?; + let state = actual.state.as_ref().context("server final state")?; + ensure!( + code == expected + && state.running == Some(false) + && state.exit_code == Some(code) + && state.oom_killed == Some(false) + && state.dead != Some(true), + "unexpected or unjoined server shutdown" + ); + let logs = self.logs(cutoff).await?; + self.write("server.log", &logs, cutoff).await?; + self.write( + "shutdown.json", + &serde_json::to_vec(&json!({"id":id,"exitCode":code, + "engine":self.expected.engine.to_string(),"physicallyStopped":true}))?, + cutoff, + ) + .await?; + Ok(()) + } + + pub(in crate::container_server) async fn runtime_ids( + &self, + cutoff: Instant, + ) -> Result> { + let identity = self.identity()?; + let raw = self + .cli + .run( + &arguments(&[ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + &format!("label=rustack.lambda.ownerId={}", identity.owner), + "--format", + "{{json .ID}}", + ]), + cutoff, + ) + .await?; + let text = std::str::from_utf8(&raw)?; + let mut ids = Vec::new(); + for line in text.lines() { + ensure!(ids.len() < 64, "runtime container inventory count"); + ids.push(cli::container_id(&cli::decode::(line.as_bytes())?)?); + } + Ok(ids) + } + + async fn audit_runtime(&self, cutoff: Instant) -> Result<()> { + ensure!( + self.runtime_ids(cutoff).await?.is_empty(), + "production server left runtime containers" + ); + let identity = self.identity()?; + let volumes = self + .cli + .run( + &arguments(&[ + "volume", + "ls", + "--filter", + &format!("label=rustack.lambda.ownerId={}", identity.owner), + "--format", + "{{json .Name}}", + ]), + cutoff, + ) + .await?; + ensure!( + volumes.iter().all(u8::is_ascii_whitespace), + "production server left runtime volumes" + ); + let raw = self + .cli + .run( + &arguments(&[ + "image", + "ls", + "--all", + "--no-trunc", + "--format", + "{{json .}}", + ]), + cutoff, + ) + .await?; + let text = std::str::from_utf8(&raw)?; + let prefix = format!("rustack-internal/lambda-{}/", identity.owner.simple()); + for (index, line) in text.lines().enumerate() { + ensure!(index < 4096, "complete image inventory count"); + let row: ImageRow = cli::decode(line.as_bytes())?; + ensure!( + row.repository.len() <= 1024 && !row.repository.chars().any(char::is_control), + "image repository inventory bound" + ); + let familiar = canonical(&row.repository); + ensure!( + !familiar.starts_with(&prefix), + "production server left private image aliases" + ); + } + Ok(()) + } + + async fn remove_server(&mut self, cutoff: Instant) -> Result<()> { + let view = self.inspect_server(cutoff).await?; + let state = view.state.as_ref().context("server removal state")?; + ensure!( + state.running == Some(false), + "refuse removal of a running server" + ); + let Resources::Server { volume, id } = &self.resources else { + anyhow::bail!("server removal authority missing"); + }; + let data = volume.clone(); + let id = id.clone(); + self.uncertain = true; + let removed = self + .cli + .run(&arguments(&["container", "rm", &id]), cutoff) + .await?; + ensure!( + removed == format!("{id}\n").as_bytes(), + "server removal reply differs" + ); + let remaining = self + .cli + .run( + &arguments(&[ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + &format!("id={id}"), + "--format", + "{{json .ID}}", + ]), + cutoff, + ) + .await?; + ensure!( + remaining.iter().all(u8::is_ascii_whitespace), + "server container remains after removal" + ); + self.resources = Resources::Volume(data); + self.uncertain = false; + Ok(()) + } + + async fn remove_volume(&mut self, cutoff: Instant) -> Result<()> { + let Resources::Volume(recorded) = &self.resources else { + anyhow::bail!("server volume removal authority missing"); + }; + let volume = self.inspect_volume(cutoff).await?; + policy::volume(&volume, &self.expected)?; + ensure!( + volume.mountpoint == recorded.mountpoint, + "server volume mountpoint changed" + ); + let attachments = self + .cli + .run( + &arguments(&[ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + &format!("volume={}", self.expected.volume), + "--format", + "{{json .ID}}", + ]), + cutoff, + ) + .await?; + ensure!( + attachments.iter().all(u8::is_ascii_whitespace), + "server volume still has an attachment" + ); + self.uncertain = true; + let removed = self + .cli + .run(&arguments(&["volume", "rm", &self.expected.volume]), cutoff) + .await?; + ensure!( + removed == format!("{}\n", self.expected.volume).as_bytes(), + "server volume removal reply differs" + ); + let remaining = self + .cli + .run( + &arguments(&[ + "volume", + "ls", + "--filter", + &format!("label={}={}", policy::NONCE_LABEL, self.expected.nonce), + "--format", + "{{json .Name}}", + ]), + cutoff, + ) + .await?; + ensure!( + remaining.iter().all(u8::is_ascii_whitespace), + "server task volume remains after removal" + ); + self.resources = Resources::Intent; + self.uncertain = false; + Ok(()) + } + + async fn verify_sources(&self, cutoff: Instant) -> Result<()> { + for (reference, expected) in &self.sources { + let source: Source = self + .cli + .json( + &arguments(&["image", "inspect", "--format", SOURCE, reference]), + cutoff, + ) + .await?; + ensure!( + source.id == *expected + && source.os == "linux" + && source.architecture == self.inputs.architecture, + "server fixture source image changed" + ); + } + Ok(()) + } +} + +fn canonical(mut value: &str) -> &str { + for prefix in ["docker.io/", "index.docker.io/", "registry-1.docker.io/"] { + if let Some(tail) = value.strip_prefix(prefix) { + value = tail; + break; + } + } + value.strip_prefix("library/").unwrap_or(value) +} diff --git a/apps/rustack/tests/docker_startup/container_server/owner/configuration.rs b/apps/rustack/tests/docker_startup/container_server/owner/configuration.rs new file mode 100644 index 00000000..bdc1f6ae --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/owner/configuration.rs @@ -0,0 +1,99 @@ +//! Release profile configuration copied into the existing exact data volume. + +use std::{fs::Permissions, os::unix::fs::PermissionsExt, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use tokio::{ + fs, + time::{Instant, timeout_at}, +}; + +use super::{ + super::{cli::arguments, records}, + Owner, Resources, + data::{DataOperation, DataOutput}, +}; + +const CONFIGURATION: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/lambda-fixtures/server-config.yaml" +)); + +impl Owner { + pub(super) async fn install_public_configuration(&mut self, cutoff: Instant) -> Result<()> { + ensure!( + (1..=4096).contains(&CONFIGURATION.len()), + "public configuration byte bound" + ); + self.inspect_server(cutoff).await?; + let Resources::Server { id, .. } = &self.resources else { + anyhow::bail!("public configuration without exact server"); + }; + let id = id.clone(); + self.write("qualification.yaml", CONFIGURATION, cutoff) + .await?; + let path = self.path.join("qualification.yaml"); + let at = cutoff.min(Instant::now() + Duration::from_secs(10)); + timeout_at( + at, + fs::set_permissions(&path, Permissions::from_mode(0o644)), + ) + .await??; + ensure!( + timeout_at(at, fs::metadata(&path)) + .await?? + .permissions() + .mode() + & 0o777 + == 0o644, + "public configuration host mode differs" + ); + let source = path + .to_str() + .context("public configuration path encoding")?; + let DataOutput::Configuration(observed) = self + .transfer_data(DataOperation::Configuration(source), cutoff) + .await? + else { + anyhow::bail!("incorrect data-volume configuration operation"); + }; + self.inspect_server(cutoff).await?; + records::configuration(&observed, CONFIGURATION)?; + self.write( + "configuration.json", + &serde_json::to_vec(&json!({ + "schemaVersion":1,"id":id,"engine":self.expected.engine.to_string(), + "name":"qualification.yaml","bytes":CONFIGURATION.len(),"uid":65532,"gid":65532,"mode":420, + "sha256Base64":STANDARD.encode(Sha256::digest(CONFIGURATION)) + }))?, + cutoff, + ) + .await?; + self.uncertain = false; + Ok(()) + } +} + +pub(super) fn copy_arguments(source: &str, target: &str) -> Vec { + // Moby's explicit CopyUIDGID branch binds extraction to Config.User. + // The default branch can preserve the host archive's UID/GID instead. + arguments(&["cp", "--archive", source, target]) +} + +#[cfg(test)] +mod tests { + use super::copy_arguments; + + #[test] + fn test_should_request_explicit_container_user_ownership_for_public_copy() { + let source = "/private/controlled root/qualification.yaml"; + let target = "controlled-id:/data/qualification.yaml"; + assert_eq!( + copy_arguments(source, target), + ["cp", "--archive", source, target], + ); + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/owner/data.rs b/apps/rustack/tests/docker_startup/container_server/owner/data.rs new file mode 100644 index 00000000..e93c2183 --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/owner/data.rs @@ -0,0 +1,308 @@ +//! Exact unstarted volume helper; the actual server keeps its read-only socket. + +#[path = "data_policy.rs"] +mod policy; + +use std::path::Path; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::models::ContainerInspectResponse; +use serde_json::json; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + super::{ + cli::{self, arguments}, + policy::{Expected, volume}, + records::{self, Identity}, + }, + INSPECT, Owner, configuration, write_private, +}; + +#[derive(Debug)] +pub(super) enum DataOperation<'a> { + Identity, + Configuration(&'a str), +} + +#[derive(Debug)] +pub(super) enum DataOutput { + Identity([Vec; 2]), + Configuration(Vec), +} + +impl Owner { + pub(super) async fn transfer_data( + &mut self, + operation: DataOperation<'_>, + cutoff: Instant, + ) -> Result { + ensure!(!self.uncertain, "uncertain parent before data transfer"); + let data = self.inspect_volume(cutoff).await?; + volume(&data, &self.expected)?; + let transfer = Uuid::new_v4().simple().to_string(); + let name = format!("rustack-transfer-{}-{transfer}", self.expected.nonce); + let writing = matches!(operation, DataOperation::Configuration(_)); + let args = create_arguments(&name, &self.expected); + self.write(&format!("data-{transfer}-intent.json"), &serde_json::to_vec(&json!({ + "schemaVersion":1,"name":name,"engine":self.expected.engine.to_string(), + "parentNonce":self.expected.nonce,"source":self.expected.source,"volume":self.expected.volume, + "configurationWrite":writing,"argv":args + }))?, cutoff).await?; + self.uncertain = true; + let reply = self.cli.run(&args, cutoff).await?; + let text = std::str::from_utf8(&reply)? + .strip_suffix('\n') + .context("data helper create newline")?; + let id = cli::container_id(text)?; + self.write(&format!("data-{transfer}-created.json"), &serde_json::to_vec(&json!({ + "id":id,"name":name,"engine":self.expected.engine.to_string(),"parentNonce":self.expected.nonce + }))?, cutoff).await?; + let result = async { + self.inspect_data(&id, &name, cutoff).await?; + match operation { + DataOperation::Identity => { + let mut archives = [Vec::new(), Vec::new()]; + for (slot, filename) in archives.iter_mut().zip(["owner.v1", "daemon.v1"]) { + let source = format!("{id}:/data/state/.rustack-lambda-docker/{filename}"); + *slot = self + .cli + .run(&arguments(&["cp", &source, "-"]), cutoff) + .await?; + self.write(&format!("data-{transfer}-{filename}.tar"), slot, cutoff) + .await?; + } + Ok(DataOutput::Identity(archives)) + } + DataOperation::Configuration(source) => { + let target = format!("{id}:/data/qualification.yaml"); + drop( + self.cli + .run(&configuration::copy_arguments(source, &target), cutoff) + .await?, + ); + let archive = self + .cli + .run(&arguments(&["cp", &target, "-"]), cutoff) + .await?; + self.write( + &format!("data-{transfer}-configuration.tar"), + &archive, + cutoff, + ) + .await?; + Ok(DataOutput::Configuration(archive)) + } + } + } + .await; + self.inspect_data(&id, &name, cutoff).await?; + let removed = self.cli.run(&arguments(&["rm", &id]), cutoff).await?; + ensure!( + removed == format!("{id}\n").as_bytes(), + "data helper removal reply differs" + ); + let remaining = self + .cli + .run( + &arguments(&[ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + &format!("id={id}"), + "--format", + "{{json .ID}}", + ]), + cutoff, + ) + .await?; + ensure!(remaining.is_empty(), "data helper removal unconfirmed"); + self.write( + &format!("data-{transfer}-removed.json"), + &serde_json::to_vec(&json!({ + "id":id,"engine":self.expected.engine.to_string(),"neverStarted":true,"absent":true + }))?, + cutoff, + ) + .await?; + result + } + + async fn inspect_data(&self, id: &str, name: &str, cutoff: Instant) -> Result<()> { + let view: ContainerInspectResponse = self + .cli + .json( + &arguments(&["container", "inspect", "--format", INSPECT, id]), + cutoff, + ) + .await?; + policy::inspect(&view, &self.expected, id, name) + } +} + +pub(super) async fn commit_identity( + uncertain: &mut bool, + slot: &mut Option, + archives: Result<[Vec; 2]>, + engine: Uuid, + directory: &Path, + cutoff: Instant, +) -> Result<()> { + let [owner, daemon] = archives?; + ensure!(*uncertain, "identity commit without retained transfer"); + let identity = records::identity(&owner, &daemon, engine)?; + write_private( + directory, + "identity.json", + &serde_json::to_vec(&json!({ + "owner":identity.owner.to_string(),"engine":identity.engine.to_string() + }))?, + cutoff, + ) + .await?; + *slot = Some(identity); + *uncertain = false; + Ok(()) +} + +fn create_arguments(name: &str, expected: &Expected) -> Vec { + let mut args = arguments(&[ + "create", + "--pull=never", + "--name", + name, + "--user", + "65532:65532", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--cgroupns", + "private", + "--network", + "none", + "--memory", + "32m", + "--memory-swap", + "32m", + "--cpus", + "0.1", + "--pids-limit", + "16", + "--restart", + "no", + "--no-healthcheck", + "--mount", + &format!( + "type=volume,source={},target=/data,volume-nocopy", + expected.volume + ), + ]); + for (key, value) in expected.labels("data-transfer") { + args.extend(["--label".into(), format!("{key}={value}")]); + } + args.push(expected.source.clone()); + args +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use sha2::{Digest, Sha256}; + use tar::{Builder, Header}; + use tempfile::tempdir; + use tokio::fs; + + use super::*; + + fn archives(engine: Uuid) -> Result<[Vec; 2]> { + let owner = Uuid::from_u128(3); + let mut result = [Vec::new(), Vec::new()]; + for (index, (name, magic)) in [("owner.v1", b"RSDKOWN2"), ("daemon.v1", b"RSDKDMD1")] + .into_iter() + .enumerate() + { + let mut data = magic.to_vec(); + data.extend_from_slice(owner.as_bytes()); + if index == 1 { + data.extend_from_slice(engine.as_bytes()); + } + let checksum = Sha256::digest(&data); + data.extend_from_slice(&checksum); + let mut header = Header::new_ustar(); + header.set_path(name)?; + header.set_size(u64::try_from(data.len())?); + header.set_uid(65532); + header.set_gid(65532); + header.set_mode(0o600); + header.set_cksum(); + let mut builder = Builder::new(Vec::new()); + builder.append(&header, data.as_slice())?; + *result.get_mut(index).context("controlled archive slot")? = builder.into_inner()?; + } + Ok(result) + } + + #[rstest::rstest] + #[case::copy(0)] + #[case::affiliation(1)] + #[case::receipt(2)] + #[tokio::test] + async fn test_should_retain_identity_uncertainty_until_archive_oracle_and_receipt_commit( + #[case] failure: u8, + ) -> Result<()> { + let directory = tempdir()?; + let engine = Uuid::from_u128(2); + let mut uncertain = true; + let mut identity = None; + let input = match failure { + 0 => Err(anyhow::anyhow!("controlled archive copy failure")), + 1 => Ok(archives(Uuid::from_u128(9))?), + _ => { + fs::write(directory.path().join("identity.json"), b"original receipt").await?; + Ok(archives(engine)?) + } + }; + assert!( + commit_identity( + &mut uncertain, + &mut identity, + input, + engine, + directory.path(), + Instant::now() + Duration::from_secs(10) + ) + .await + .is_err() + ); + assert!(uncertain); + assert!(identity.is_none()); + Ok(()) + } + + #[tokio::test] + async fn test_should_clear_identity_uncertainty_only_after_the_durable_receipt() -> Result<()> { + let directory = tempdir()?; + let engine = Uuid::from_u128(2); + let mut uncertain = true; + let mut identity = None; + commit_identity( + &mut uncertain, + &mut identity, + Ok(archives(engine)?), + engine, + directory.path(), + Instant::now() + Duration::from_secs(10), + ) + .await?; + assert!(!uncertain); + assert_eq!(identity.context("committed identity")?.engine, engine); + assert!(directory.path().join("identity.json").is_file()); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/owner/data_policy.rs b/apps/rustack/tests/docker_startup/container_server/owner/data_policy.rs new file mode 100644 index 00000000..5e0146fb --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/owner/data_policy.rs @@ -0,0 +1,192 @@ +//! Independent inspected authority for a helper that must never execute. + +use std::collections::HashMap; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::models::{ + ContainerInspectResponse, ContainerStateStatusEnum, HostConfigCgroupnsModeEnum, MountType, + RestartPolicyNameEnum, +}; + +use super::super::super::policy::Expected; + +pub(super) fn inspect( + view: &ContainerInspectResponse, + expected: &Expected, + id: &str, + name: &str, +) -> Result<()> { + let config = view.config.as_ref().context("data helper configuration")?; + let state = view.state.as_ref().context("data helper state")?; + let labels = config.labels.as_ref().context("data helper labels")?; + ensure!( + view.id.as_deref() == Some(id) + && view.name.as_deref() == Some(format!("/{name}").as_str()) + && view.image.as_deref() == Some(expected.source.as_str()) + && config.image.as_deref() == Some(expected.source.as_str()) + && config.user.as_deref() == Some("65532:65532") + && expected + .labels("data-transfer") + .iter() + .all(|(key, value)| labels.get(key) == Some(value)) + && state.status == Some(ContainerStateStatusEnum::CREATED) + && state.running == Some(false) + && state.pid == Some(0) + && state.dead != Some(true), + "data helper identity or lifecycle differs" + ); + let host = view + .host_config + .as_ref() + .context("data helper host policy")?; + ensure!( + host.readonly_rootfs == Some(true) + && host.privileged == Some(false) + && host.auto_remove == Some(false) + && host.publish_all_ports == Some(false) + && host.memory == Some(33_554_432) + && host.memory_swap == Some(33_554_432) + && host.nano_cpus == Some(100_000_000) + && host.pids_limit == Some(16) + && host.cgroupns_mode == Some(HostConfigCgroupnsModeEnum::PRIVATE) + && host.network_mode.as_deref() == Some("none") + && host.cap_drop.as_deref() == Some(["ALL".to_owned()].as_slice()) + && host.cap_add.as_ref().is_none_or(Vec::is_empty) + && host.security_opt.as_deref() + == Some(["no-new-privileges:true".to_owned()].as_slice()) + && host.group_add.as_ref().is_none_or(Vec::is_empty) + && host.binds.as_ref().is_none_or(Vec::is_empty) + && host.volumes_from.as_ref().is_none_or(Vec::is_empty) + && host.devices.as_ref().is_none_or(Vec::is_empty) + && host.device_requests.as_ref().is_none_or(Vec::is_empty) + && host.port_bindings.as_ref().is_none_or(HashMap::is_empty), + "data helper security policy differs" + ); + let restart = host + .restart_policy + .as_ref() + .context("data helper restart policy")?; + ensure!( + restart.name == Some(RestartPolicyNameEnum::NO) + && restart.maximum_retry_count.unwrap_or_default() == 0, + "data helper restart policy differs" + ); + let health = config + .healthcheck + .as_ref() + .context("data helper health policy")?; + ensure!( + health.test.as_deref() == Some(["NONE".to_owned()].as_slice()), + "data helper healthcheck enabled" + ); + let requested = host + .mounts + .as_deref() + .context("data helper requested mounts")?; + let [mount] = requested else { + anyhow::bail!("extra data helper requested mount"); + }; + ensure!( + mount.typ == Some(MountType::VOLUME) + && mount.source.as_deref() == Some(expected.volume.as_str()) + && mount.target.as_deref() == Some("/data") + && mount.read_only != Some(true) + && mount + .volume_options + .as_ref() + .is_some_and(|options| options.no_copy == Some(true)), + "data helper requested volume differs" + ); + let actual = view + .mounts + .as_deref() + .context("data helper actual mounts")?; + let [mount] = actual else { + anyhow::bail!("extra data helper actual mount"); + }; + ensure!( + mount.typ.as_deref() == Some("volume") + && mount.name.as_deref() == Some(expected.volume.as_str()) + && mount.destination.as_deref() == Some("/data") + && mount.rw == Some(true) + && mount.driver.as_deref() == Some("local"), + "data helper actual volume differs" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + use uuid::Uuid; + + use super::*; + use crate::container_server::cli::decode; + + fn fixture() -> (Expected, Value) { + let expected = Expected { + name: "rustack-server-parent".into(), + volume: "rustack-server-data-parent".into(), + nonce: "1".repeat(32), + engine: Uuid::from_u128(2), + source: format!("sha256:{}", "a".repeat(64)), + network: "f".repeat(64), + socket: "/run/user/501/docker.sock".into(), + gid: "991".into(), + }; + let value = json!({ + "Id":"c".repeat(64),"Name":"/rustack-transfer-test","Image":expected.source, + "State":{"Status":"created","Running":false,"Pid":0,"Dead":false}, + "Config":{"Image":expected.source,"User":"65532:65532","Labels":expected.labels("data-transfer"), + "Healthcheck":{"Test":["NONE"]}}, + "HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"AutoRemove":false,"PublishAllPorts":false, + "Memory":33_554_432,"MemorySwap":33_554_432,"NanoCpus":100_000_000,"PidsLimit":16, + "CgroupnsMode":"private","NetworkMode":"none","CapDrop":["ALL"], + "CapAdd":[],"Binds":[],"PortBindings":{}, + "SecurityOpt":["no-new-privileges:true"],"RestartPolicy":{"Name":"no","MaximumRetryCount":0}, + "Mounts":[{"Type":"volume","Source":expected.volume,"Target":"/data","ReadOnly":false, + "VolumeOptions":{"NoCopy":true}}]}, + "Mounts":[{"Type":"volume","Name":expected.volume,"Destination":"/data","RW":true,"Driver":"local"}] + }); + (expected, value) + } + + #[test] + fn test_should_accept_only_an_owned_never_started_single_volume_helper() -> Result<()> { + let (expected, value) = fixture(); + let view = decode(&serde_json::to_vec(&value)?)?; + inspect(&view, &expected, &"c".repeat(64), "rustack-transfer-test") + } + + #[rstest::rstest] + #[case("/Id", json!("d".repeat(64)))] + #[case("/Image", json!(format!("sha256:{}", "b".repeat(64))))] + #[case("/Name", json!("/foreign"))] + #[case("/Config/User", json!("0:0"))] + #[case("/Config/Labels", json!({}))] + #[case("/State/Status", json!("exited"))] + #[case("/State/Running", json!(true))] + #[case("/State/Pid", json!(2))] + #[case("/HostConfig/NetworkMode", json!("host"))] + #[case("/HostConfig/Privileged", json!(true))] + #[case("/HostConfig/CapAdd", json!(["SYS_ADMIN"]))] + #[case("/HostConfig/Binds", json!(["/run/user/501/docker.sock:/socket"]))] + #[case("/HostConfig/PortBindings", json!({"80/tcp":[{"HostPort":"80"}]}))] + #[case("/HostConfig/Mounts/0/ReadOnly", json!(true))] + #[case("/HostConfig/Mounts/0/VolumeOptions/NoCopy", json!(false))] + #[case("/Mounts/0/Name", json!("foreign-volume"))] + #[case("/Mounts/0/RW", json!(false))] + #[case("/Config/Healthcheck/Test", json!(["CMD","/rustack"]))] + fn test_should_reject_identity_execution_privilege_and_mount_substitution( + #[case] pointer: &str, + #[case] replacement: Value, + ) -> Result<()> { + let (expected, mut value) = fixture(); + *value + .pointer_mut(pointer) + .context("controlled fixture pointer")? = replacement; + let view = decode(&serde_json::to_vec(&value)?)?; + assert!(inspect(&view, &expected, &"c".repeat(64), "rustack-transfer-test").is_err()); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/container_server/policy.rs b/apps/rustack/tests/docker_startup/container_server/policy.rs new file mode 100644 index 00000000..c1187081 --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/policy.rs @@ -0,0 +1,387 @@ +//! Independent inspected server policy and exact mutation authority. + +#[cfg(test)] +#[path = "policy/tests.rs"] +mod tests; + +use std::{collections::BTreeMap, env, path::PathBuf}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::{ + artifact::ImageReference, + docker::{ + DockerSocket, NetworkSelection, + models::{ + ContainerInspectResponse, HostConfig, HostConfigCgroupnsModeEnum, MountType, + NetworkInspect, RestartPolicyNameEnum, Volume, VolumeScopeEnum, + }, + }, +}; +use uuid::Uuid; + +use super::{ImageProfile, cli::container_id}; + +pub(super) const NONCE_LABEL: &str = "rustack.test.server-container.nonce"; +pub(super) const ROLE_LABEL: &str = "rustack.test.server-container.role"; +pub(super) const ENGINE_LABEL: &str = "rustack.test.server-container.engine"; +pub(super) const MEMORY: i64 = 536_870_912; +pub(super) const TEMPORARY: &str = "rw,noexec,nosuid,nodev,size=64m,mode=1777"; + +#[derive(Debug)] +pub(super) struct Inputs { + pub program: PathBuf, + pub client: String, + pub source_socket: String, + pub gid: String, + pub network: String, + pub image: String, + pub sdk_image: String, + pub relay: String, + pub architecture: String, +} +impl Inputs { + pub fn load(profile: ImageProfile) -> Result { + let program = env::var("RUSTACK_DOCKER_BIN")?; + ensure!( + program.len() <= 4096 + && PathBuf::from(&program).is_absolute() + && !program.chars().any(char::is_control), + "absolute Docker CLI required" + ); + let client = super::super::process::socket()?; + let _: DockerSocket = client.parse()?; + let source_socket = env::var("RUSTACK_SERVER_DAEMON_SOCKET")?; + socket_source(&source_socket)?; + let gid = env::var("RUSTACK_SERVER_SOCKET_GID")?; + socket_gid(&gid)?; + let network = env::var("RUSTACK_RUNTIME_NETWORK")?; + let _: NetworkSelection = network.parse()?; + ensure!( + network != "none", + "server container requires an explicit user bridge" + ); + let image = image_variable(profile.variable())?; + let sdk_image = image_variable("RUSTACK_HANDLER_SDK_IMAGE")?; + let relay = image_variable("RUSTACK_RUNTIME_IMAGE")?; + let architecture = super::super::process::architecture()?; + Ok(Self { + program: PathBuf::from(program), + client, + source_socket, + gid, + network, + image, + sdk_image, + relay, + architecture, + }) + } +} + +fn image_variable(key: &str) -> Result { + let value = env::var(key)?; + let _: ImageReference = value.parse()?; + Ok(value) +} + +pub(super) fn socket_source(value: &str) -> Result<()> { + ensure!( + (2..=256).contains(&value.len()) + && value.starts_with('/') + && value + .get(1..) + .context("socket path suffix")? + .split('/') + .all(|part| !part.is_empty() + && part != "." + && part != ".." + && part + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b))), + "invalid explicit daemon-side socket path" + ); + Ok(()) +} + +pub(super) fn socket_gid(value: &str) -> Result { + ensure!( + (1..=10).contains(&value.len()) && value.bytes().all(|b| b.is_ascii_digit()), + "invalid explicit daemon socket GID grammar" + ); + let gid: u32 = value.parse()?; + ensure!( + gid <= 2_147_483_647 && gid.to_string() == value, + "invalid daemon socket GID range" + ); + Ok(gid) +} + +#[derive(Debug)] +pub(super) struct Expected { + pub name: String, + pub volume: String, + pub nonce: String, + pub engine: Uuid, + pub source: String, + pub network: String, + pub socket: String, + pub gid: String, +} +impl Expected { + pub fn labels(&self, role: &str) -> BTreeMap { + BTreeMap::from([ + (NONCE_LABEL.into(), self.nonce.clone()), + (ROLE_LABEL.into(), role.into()), + (ENGINE_LABEL.into(), self.engine.to_string()), + ]) + } +} + +pub(super) fn source_id(value: &str) -> Result { + let digest = value + .strip_prefix("sha256:") + .context("source image ID prefix")?; + container_id(digest)?; + Ok(value.to_owned()) +} + +pub(super) fn network(network: &NetworkInspect, requested: &str) -> Result { + let id = network.id.as_deref().context("network full ID")?; + container_id(id)?; + let name = network.name.as_deref().context("network name")?; + ensure!( + (requested == id || requested == name) + && !matches!(name, "host" | "bridge" | "none" | "default") + && network.driver.as_deref() == Some("bridge") + && network.scope.as_deref() == Some("local") + && network.ingress == Some(false) + && network.config_only == Some(false) + && network + .options + .as_ref() + .and_then(|o| o.get("com.docker.network.bridge.default_bridge")) + .is_none_or(|value| value == "false"), + "server fixture network differs" + ); + Ok(id.to_owned()) +} + +pub(super) fn volume(volume: &Volume, expected: &Expected) -> Result<()> { + ensure!( + volume.name == expected.volume + && volume.driver == "local" + && expected + .labels("server-data") + .iter() + .all(|(key, value)| volume.labels.get(key) == Some(value)) + && volume.options.is_empty() + && volume.scope == Some(VolumeScopeEnum::LOCAL) + && volume.cluster_volume.is_none() + && volume.mountpoint.starts_with('/') + && volume.mountpoint.len() <= 4096, + "server volume ownership differs" + ); + Ok(()) +} + +pub(super) fn server( + view: &ContainerInspectResponse, + expected: &Expected, + data: &Volume, +) -> Result<()> { + volume(data, expected)?; + container_id(view.id.as_deref().context("server container ID")?)?; + let config = view.config.as_ref().context("server configuration")?; + let labels = config.labels.as_ref().context("server labels")?; + ensure!( + view.name.as_deref() == Some(format!("/{}", expected.name).as_str()) + && view.image.as_deref() == Some(expected.source.as_str()) + && config.image.as_deref() == Some(expected.source.as_str()) + && config.user.as_deref() == Some("65532:65532") + && config.working_dir.as_deref() == Some("/data") + && config.entrypoint.as_deref() == Some(["/rustack".to_owned()].as_slice()) + && config.cmd.as_ref().is_none_or(Vec::is_empty) + && expected + .labels("server") + .iter() + .all(|(key, value)| labels.get(key) == Some(value)), + "server immutable identity or execution policy differs" + ); + let host = view.host_config.as_ref().context("server host policy")?; + host_policy(host, expected)?; + mount_policy(view, host, expected, data)?; + Ok(()) +} + +fn host_policy(host: &HostConfig, expected: &Expected) -> Result<()> { + ensure!( + host.readonly_rootfs == Some(true) + && host.privileged == Some(false) + && host.auto_remove == Some(false) + && host.publish_all_ports == Some(false) + && host.memory == Some(MEMORY) + && host.memory_swap == Some(MEMORY) + && host.nano_cpus == Some(1_000_000_000) + && host.pids_limit == Some(128) + && host.oom_kill_disable != Some(true) + && host.cap_drop.as_deref() == Some(["ALL".to_owned()].as_slice()) + && host.cap_add.as_ref().is_none_or(Vec::is_empty) + && host.security_opt.as_deref() + == Some(["no-new-privileges:true".to_owned()].as_slice()) + && host.group_add.as_deref() == Some([expected.gid.clone()].as_slice()) + && host.network_mode.as_deref() == Some(expected.network.as_str()) + && host.pid_mode.as_deref().is_none_or(str::is_empty) + && host.cgroupns_mode == Some(HostConfigCgroupnsModeEnum::PRIVATE) + && host.devices.as_ref().is_none_or(Vec::is_empty) + && host.device_requests.as_ref().is_none_or(Vec::is_empty) + && host.volumes_from.as_ref().is_none_or(Vec::is_empty) + && host.binds.as_ref().is_none_or(Vec::is_empty), + "server kernel/security policy differs" + ); + let restart = host + .restart_policy + .as_ref() + .context("server restart policy")?; + ensure!( + restart.name == Some(RestartPolicyNameEnum::NO) && restart.maximum_retry_count == Some(0), + "server restart policy differs" + ); + let logs = host.log_config.as_ref().context("server log policy")?; + let options = logs.config.as_ref().context("server bounded log options")?; + ensure!( + logs.typ.as_deref() == Some("json-file") + && options.len() == 2 + && options.get("max-size").map(String::as_str) == Some("1m") + && options.get("max-file").map(String::as_str) == Some("1"), + "server log bounds differ" + ); + let temporary = host.tmpfs.as_ref().context("server temporary mount")?; + ensure!( + temporary.len() == 1 && temporary.get("/tmp").map(String::as_str) == Some(TEMPORARY), + "server temporary storage policy differs" + ); + let bindings = host + .port_bindings + .as_ref() + .context("server published port request")?; + ensure!( + bindings.len() == 1 + && bindings + .get("4566/tcp") + .and_then(Option::as_ref) + .is_some_and(|rows| rows.len() == 1 + && rows + .first() + .is_some_and(|row| row.host_ip.as_deref() == Some("127.0.0.1"))), + "server loopback publication differs" + ); + Ok(()) +} + +fn mount_policy( + view: &ContainerInspectResponse, + host: &HostConfig, + expected: &Expected, + data: &Volume, +) -> Result<()> { + let requested = host.mounts.as_ref().context("server requested mounts")?; + ensure!( + requested.len() == 2 + && requested + .iter() + .filter(|m| m.target.as_deref() == Some("/data")) + .count() + == 1 + && requested + .iter() + .filter(|m| m.target.as_deref() == Some("/run/rustack/docker.sock")) + .count() + == 1 + && requested + .iter() + .all(|mount| (mount.typ == Some(MountType::BIND) + && mount.source.as_deref() == Some(expected.socket.as_str()) + && mount.target.as_deref() == Some("/run/rustack/docker.sock") + && mount.read_only == Some(true)) + || (mount.typ == Some(MountType::VOLUME) + && mount.source.as_deref() == Some(expected.volume.as_str()) + && mount.target.as_deref() == Some("/data") + && mount.read_only != Some(true))), + "server mount request differs" + ); + let mounts = view.mounts.as_ref().context("server actual mounts")?; + ensure!( + (2..=3).contains(&mounts.len()) + && mounts + .iter() + .filter(|m| m.destination.as_deref() == Some("/data")) + .count() + == 1 + && mounts + .iter() + .filter(|m| m.destination.as_deref() == Some("/run/rustack/docker.sock")) + .count() + == 1 + && mounts + .iter() + .all(|mount| match mount.destination.as_deref() { + Some("/data") => + mount.typ.as_deref() == Some("volume") + && mount.name.as_deref() == Some(expected.volume.as_str()) + && mount.rw == Some(true) + && mount.driver.as_deref() == Some("local") + && mount.source.as_deref() == Some(data.mountpoint.as_str()), + Some("/run/rustack/docker.sock") => + mount.typ.as_deref() == Some("bind") + && mount.source.as_deref() == Some(expected.socket.as_str()) + && mount.rw == Some(false), + Some("/tmp") => mount.typ.as_deref() == Some("tmpfs"), + _ => false, + }), + "server actual mount affiliation differs" + ); + Ok(()) +} + +pub(super) fn endpoint(view: &ContainerInspectResponse, expected: &Expected) -> Result { + let network = view + .network_settings + .as_ref() + .context("server network settings")?; + let networks = network + .networks + .as_ref() + .context("server attached networks")?; + ensure!( + networks.len() == 1 + && networks.values().all(|entry| entry.network_id.as_deref() + == Some(expected.network.as_str()) + && entry + .aliases + .as_ref() + .is_some_and(|names| names.contains(&expected.name))), + "server DNS/network affiliation differs" + ); + let ports = network + .ports + .as_ref() + .context("server actual publication")?; + ensure!(ports.len() == 1, "extra server published port"); + let rows = ports + .get("4566/tcp") + .and_then(Option::as_ref) + .context("server host port")?; + ensure!(rows.len() == 1, "ambiguous server host publication"); + let row = rows.first().context("empty server port list")?; + ensure!( + row.host_ip.as_deref() == Some("127.0.0.1"), + "server port is not loopback" + ); + let text = row.host_port.as_deref().context("server assigned port")?; + let port: u16 = text.parse()?; + ensure!( + port > 0 && port.to_string() == text, + "server assigned port grammar" + ); + Ok(format!("http://127.0.0.1:{port}")) +} diff --git a/apps/rustack/tests/docker_startup/container_server/policy/tests.rs b/apps/rustack/tests/docker_startup/container_server/policy/tests.rs new file mode 100644 index 00000000..f9b226eb --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/policy/tests.rs @@ -0,0 +1,148 @@ +//! Independent Docker-shaped fixture; mutations must remove cleanup authority. + +use serde_json::{Value, json}; + +use super::{super::cli::decode, *}; + +fn fixture() -> Result<(Expected, Volume, Value)> { + let expected = Expected { + name: "rustack-server-test".into(), + volume: "rustack-server-data-test".into(), + nonce: "1".repeat(32), + engine: Uuid::from_u128(2), + source: format!("sha256:{}", "a".repeat(64)), + network: "f".repeat(64), + socket: "/var/run/docker.sock".into(), + gid: "991".into(), + }; + let data: Volume = decode(&serde_json::to_vec(&json!({ + "Name":"rustack-server-data-test","Driver":"local","Mountpoint":"/volumes/test/_data", + "Scope":"local","Options":null,"Labels":{ + "rustack.test.server-container.nonce":"1".repeat(32), + "rustack.test.server-container.role":"server-data", + "rustack.test.server-container.engine":Uuid::from_u128(2).to_string()}}))?)?; + let view = json!({ + "Id":"c".repeat(64),"Name":"/rustack-server-test","Image":format!("sha256:{}","a".repeat(64)), + "Config":{"Image":format!("sha256:{}","a".repeat(64)),"User":"65532:65532","WorkingDir":"/data", + "Entrypoint":["/rustack"],"Cmd":[],"Labels":{ + "rustack.test.server-container.nonce":"1".repeat(32), + "rustack.test.server-container.role":"server", + "rustack.test.server-container.engine":Uuid::from_u128(2).to_string()}}, + "HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"AutoRemove":false,"PublishAllPorts":false, + "Memory":536_870_912,"MemorySwap":536_870_912,"NanoCpus":1_000_000_000,"PidsLimit":128, + "CapDrop":["ALL"],"SecurityOpt":["no-new-privileges:true"],"GroupAdd":["991"], + "NetworkMode":"f".repeat(64),"PidMode":"","CgroupnsMode":"private", + "RestartPolicy":{"Name":"no","MaximumRetryCount":0}, + "LogConfig":{"Type":"json-file","Config":{"max-size":"1m","max-file":"1"}}, + "Tmpfs":{"/tmp":"rw,noexec,nosuid,nodev,size=64m,mode=1777"}, + "Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/run/rustack/docker.sock","ReadOnly":true}, + {"Type":"volume","Source":"rustack-server-data-test","Target":"/data"}], + "PortBindings":{"4566/tcp":[{"HostIp":"127.0.0.1","HostPort":""}]}}, + "Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Destination":"/run/rustack/docker.sock","RW":false}, + {"Type":"volume","Name":"rustack-server-data-test","Source":"/volumes/test/_data","Destination":"/data","RW":true,"Driver":"local"}], + "NetworkSettings":{"Networks":{"qualified":{"NetworkID":"f".repeat(64),"Aliases":["rustack-server-test"]}}, + "Ports":{"4566/tcp":[{"HostIp":"127.0.0.1","HostPort":"49153"}]}}, + "State":{"Running":true,"OOMKilled":false,"ExitCode":0,"Dead":false} + }); + Ok((expected, data, view)) +} + +#[test] +fn test_should_require_exact_server_identity_security_and_named_mount_affiliation() -> Result<()> { + let (expected, data, view) = fixture()?; + let inspected: ContainerInspectResponse = decode(&serde_json::to_vec(&view)?)?; + server(&inspected, &expected, &data)?; + for (path, replacement) in [ + ("/Name", json!("/foreign")), + ("/Image", json!(format!("sha256:{}", "b".repeat(64)))), + ( + "/Config/Labels/rustack.test.server-container.nonce", + json!("foreign"), + ), + ("/Config/User", json!("0:0")), + ("/HostConfig/ReadonlyRootfs", json!(false)), + ("/HostConfig/Privileged", json!(true)), + ("/HostConfig/NetworkMode", json!("host")), + ("/HostConfig/MemorySwap", json!(-1)), + ("/HostConfig/GroupAdd", json!(["0"])), + ( + "/HostConfig/PortBindings/4566~1tcp/0/HostIp", + json!("0.0.0.0"), + ), + ("/Mounts/0/Source", json!("/foreign/socket")), + ("/Mounts/1/Name", json!("foreign-volume")), + ("/Mounts/1/Source", json!("/volumes/foreign/_data")), + ("/Mounts/1/RW", json!(false)), + ] { + let mut changed = view.clone(); + *changed + .pointer_mut(path) + .context("policy fixture mutation path")? = replacement; + let inspected = decode(&serde_json::to_vec(&changed)?)?; + assert!( + server(&inspected, &expected, &data).is_err(), + "accepted policy mutation {path}" + ); + } + Ok(()) +} + +#[test] +fn test_should_refuse_wrong_actual_dns_network_or_public_host_port_metadata() -> Result<()> { + let (expected, _, view) = fixture()?; + let inspected = decode(&serde_json::to_vec(&view)?)?; + assert_eq!(endpoint(&inspected, &expected)?, "http://127.0.0.1:49153"); + for (path, replacement) in [ + ( + "/NetworkSettings/Networks/qualified/NetworkID", + json!("e".repeat(64)), + ), + ( + "/NetworkSettings/Networks/qualified/Aliases", + json!(["foreign"]), + ), + ( + "/NetworkSettings/Ports/4566~1tcp/0/HostIp", + json!("0.0.0.0"), + ), + ("/NetworkSettings/Ports/4566~1tcp/0/HostPort", json!("0")), + ( + "/NetworkSettings/Ports/4566~1tcp/0/HostPort", + json!("049153"), + ), + ( + "/NetworkSettings/Ports/4566~1tcp/0/HostPort", + json!("65536"), + ), + ] { + let mut changed = view.clone(); + *changed + .pointer_mut(path) + .context("endpoint fixture mutation path")? = replacement; + let inspected = decode(&serde_json::to_vec(&changed)?)?; + assert!( + endpoint(&inspected, &expected).is_err(), + "accepted endpoint mutation {path}" + ); + } + Ok(()) +} + +#[test] +fn test_should_reject_foreign_or_driver_configured_server_volumes() -> Result<()> { + let (expected, data, _) = fixture()?; + volume(&data, &expected)?; + let mut foreign = data.clone(); + foreign.name = "unrelated".into(); + assert!(volume(&foreign, &expected).is_err()); + let mut wrong_driver = data.clone(); + wrong_driver.driver = "nfs".into(); + assert!(volume(&wrong_driver, &expected).is_err()); + let mut configured = data.clone(); + configured.options.insert("device".into(), "/host".into()); + assert!(volume(&configured, &expected).is_err()); + let mut relabeled = data; + relabeled.labels.clear(); + assert!(volume(&relabeled, &expected).is_err()); + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/container_server/records.rs b/apps/rustack/tests/docker_startup/container_server/records.rs new file mode 100644 index 00000000..a1168836 --- /dev/null +++ b/apps/rustack/tests/docker_startup/container_server/records.rs @@ -0,0 +1,377 @@ +//! Bounded raw archives of persistent installation identity; never extraction. + +use std::{ + collections::BTreeSet, + io::{Cursor, Read}, +}; + +use anyhow::{Context, Result, ensure}; +use sha2::{Digest, Sha256}; +use tar::Archive; +use uuid::Uuid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Identity { + pub owner: Uuid, + pub engine: Uuid, +} + +pub(super) fn identity(owner: &[u8], daemon: &[u8], engine: Uuid) -> Result { + let owner = record(owner, "owner.v1", *b"RSDKOWN2", 56)?; + let daemon = record(daemon, "daemon.v1", *b"RSDKDMD1", 72)?; + let id = Uuid::from_slice(owner.get(8..24).context("owner identity bytes")?)?; + let bound_owner = Uuid::from_slice(daemon.get(8..24).context("bound owner bytes")?)?; + let bound_engine = Uuid::from_slice(daemon.get(24..40).context("bound engine bytes")?)?; + ensure!( + !id.is_nil() && id == bound_owner && engine == bound_engine && !engine.is_nil(), + "installation identity affiliation differs" + ); + Ok(Identity { owner: id, engine }) +} + +#[derive(Debug, Clone, Copy)] +struct Member<'a> { + name: &'a str, + length: usize, + uid: u64, + gid: u64, + mode: u32, +} + +fn member(raw: &[u8], expected: Member<'_>) -> Result> { + let Member { + name, + length, + uid, + gid, + mode, + } = expected; + ensure!( + (2048..=32768).contains(&raw.len()) && raw.len().is_multiple_of(512), + "identity archive byte bound" + ); + let mut archive = Archive::new(Cursor::new(raw)); + let mut payload = None; + let mut timing_seen = false; + for (index, entry) in archive.entries()?.raw(true).enumerate() { + ensure!(index < 2, "extra identity archive member"); + let mut entry = entry?; + let path = entry.path_bytes(); + ensure!( + !path.is_empty() && path.len() <= 256, + "identity archive path bound" + ); + if entry.header().entry_type().is_pax_local_extensions() { + ensure!( + index == 0 && !timing_seen && entry.size() <= 1024, + "identity timing metadata bound" + ); + ensure!( + path.is_ascii() + && path.split(|byte| *byte == b'/').all(|part| !part.is_empty() + && part != b"." + && part != b".." + && part + .iter() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(b))), + "identity timing metadata path" + ); + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + timing(&bytes)?; + timing_seen = true; + continue; + } + ensure!( + payload.is_none() + && path.as_ref() == name.as_bytes() + && entry.header().entry_type().is_file() + && entry.size() == u64::try_from(length)? + && entry.header().uid()? == uid + && entry.header().gid()? == gid + && entry.header().mode()? == mode + && entry.link_name_bytes().is_none(), + "identity archive member differs" + ); + let mut bytes = Vec::with_capacity(length); + entry.read_to_end(&mut bytes)?; + ensure!(bytes.len() == length, "truncated installation identity"); + payload = Some(bytes); + } + let position = usize::try_from(archive.into_inner().position())?; + let tail = raw.get(position..).context("identity archive cursor")?; + ensure!( + tail.len() >= 512 && tail.iter().all(|byte| *byte == 0), + "identity archive terminator or trailing content" + ); + let bytes = payload.context("missing installation identity")?; + Ok(bytes) +} + +fn record(raw: &[u8], name: &str, magic: [u8; 8], length: usize) -> Result> { + let bytes = member( + raw, + Member { + name, + length, + uid: 65532, + gid: 65532, + mode: 0o600, + }, + )?; + let end = length.checked_sub(32).context("identity checksum length")?; + let data = bytes.get(..end).context("identity checksum payload")?; + ensure!( + bytes.get(..8) == Some(magic.as_slice()) + && bytes.get(end..) == Some(Sha256::digest(data).as_slice()), + "installation identity checksum or generation differs" + ); + Ok(bytes) +} + +pub(super) fn configuration(raw: &[u8], expected: &[u8]) -> Result<()> { + ensure!( + (1..=4096).contains(&expected.len()), + "public configuration byte bound" + ); + let observed = member( + raw, + Member { + name: "qualification.yaml", + length: expected.len(), + uid: 65532, + gid: 65532, + mode: 0o644, + }, + )?; + ensure!(observed == expected, "copied public configuration differs"); + Ok(()) +} + +fn timing(mut bytes: &[u8]) -> Result<()> { + let mut seen = BTreeSet::new(); + while !bytes.is_empty() { + ensure!(seen.len() < 4, "identity timing field count"); + let space = bytes + .iter() + .position(|byte| *byte == b' ') + .context("PAX timing length")?; + let digits = bytes.get(..space).context("PAX timing length digits")?; + ensure!( + (1..=4).contains(&digits.len()) && digits.iter().all(u8::is_ascii_digit), + "PAX timing length grammar" + ); + let length = std::str::from_utf8(digits)?.parse::()?; + let line = bytes.get(..length).context("PAX timing record bound")?; + ensure!(line.last() == Some(&b'\n'), "PAX timing newline"); + let content = line + .get(space.saturating_add(1)..length.saturating_sub(1)) + .context("PAX timing content")?; + let content = std::str::from_utf8(content)?; + let (key, value) = content.split_once('=').context("PAX timing pair")?; + ensure!( + matches!(key, "mtime" | "atime" | "ctime") + && seen.insert(key) + && (1..=64).contains(&value.len()), + "unexpected identity timing field" + ); + let unsigned = value.strip_prefix('-').unwrap_or(value); + let mut parts = unsigned.split('.'); + let integral = parts.next().context("PAX timing integer")?; + ensure!( + !integral.is_empty() && integral.bytes().all(|byte| byte.is_ascii_digit()), + "PAX timing integer grammar" + ); + if let Some(fraction) = parts.next() { + ensure!( + (1..=9).contains(&fraction.len()) + && fraction.bytes().all(|byte| byte.is_ascii_digit()), + "PAX timing fraction grammar" + ); + } + ensure!(parts.next().is_none(), "PAX timing extra component"); + bytes = bytes.get(length..).context("PAX timing next record")?; + } + ensure!(!seen.is_empty(), "empty identity timing metadata"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use tar::{Builder, EntryType, Header}; + + use super::*; + + #[test] + fn test_should_require_exact_public_config_bytes_container_ownership_and_readable_mode() + -> Result<()> { + let bytes = b"budgets:\n startupSeconds: 180\n"; + for (name, uid, gid, mode, accepted) in [ + ("qualification.yaml", 65532, 65532, 0o644, true), + ("other.yaml", 65532, 65532, 0o644, false), + ("qualification.yaml", 0, 0, 0o644, false), + ("qualification.yaml", 501, 20, 0o644, false), + ("qualification.yaml", 65532, 0, 0o644, false), + ("qualification.yaml", 0, 65532, 0o644, false), + ("qualification.yaml", 65532, 65532, 0o600, false), + ] { + let mut builder = Builder::new(Vec::new()); + let mut header = Header::new_ustar(); + header.set_entry_type(EntryType::Regular); + header.set_uid(uid); + header.set_gid(gid); + header.set_mode(mode); + header.set_size(u64::try_from(bytes.len())?); + header.set_cksum(); + builder.append_data(&mut header, name, bytes.as_slice())?; + let raw = builder.into_inner()?; + assert_eq!(configuration(&raw, bytes).is_ok(), accepted); + assert!(configuration(&raw, b"budgets:\n startupSeconds: 181\n").is_err()); + } + Ok(()) + } + + fn payload(magic: [u8; 8], owner: Uuid, engine: Option) -> Vec { + let mut bytes = magic.to_vec(); + bytes.extend_from_slice(owner.as_bytes()); + if let Some(engine) = engine { + bytes.extend_from_slice(engine.as_bytes()); + } + bytes.extend_from_slice(&Sha256::digest(&bytes)); + bytes + } + fn append( + builder: &mut Builder>, + name: &str, + bytes: &[u8], + kind: EntryType, + uid: u64, + mode: u32, + ) -> Result<()> { + let mut header = Header::new_ustar(); + header.set_entry_type(kind); + header.set_uid(uid); + header.set_gid(65532); + header.set_mode(mode); + header.set_size(u64::try_from(bytes.len())?); + header.set_cksum(); + builder.append_data(&mut header, name, bytes)?; + Ok(()) + } + fn archive(name: &str, bytes: &[u8]) -> Result> { + let mut builder = Builder::new(Vec::new()); + append(&mut builder, name, bytes, EntryType::Regular, 65532, 0o600)?; + Ok(builder.into_inner()?) + } + #[test] + fn test_should_bind_the_actual_persisted_owner_and_engine_records() -> Result<()> { + let owner = Uuid::new_v4(); + let engine = Uuid::new_v4(); + let a = archive("owner.v1", &payload(*b"RSDKOWN2", owner, None))?; + let b = archive("daemon.v1", &payload(*b"RSDKDMD1", owner, Some(engine)))?; + assert_eq!(identity(&a, &b, engine)?, Identity { owner, engine }); + assert!(identity(&a, &b, Uuid::new_v4()).is_err()); + let foreign = archive( + "daemon.v1", + &payload(*b"RSDKDMD1", Uuid::new_v4(), Some(engine)), + )?; + assert!(identity(&a, &foreign, engine).is_err()); + let nil = archive("owner.v1", &payload(*b"RSDKOWN2", Uuid::nil(), None))?; + assert!(identity(&nil, &b, engine).is_err()); + Ok(()) + } + #[test] + fn test_should_reject_record_replacement_paths_types_permissions_and_checksums() -> Result<()> { + let bytes = payload(*b"RSDKOWN2", Uuid::new_v4(), None); + for (name, kind, uid, mode) in [ + ("unrelated", EntryType::Regular, 65532, 0o600), + ("sub/owner.v1", EntryType::Regular, 65532, 0o600), + ("owner.v1", EntryType::Symlink, 65532, 0o600), + ("owner.v1", EntryType::Regular, 0, 0o600), + ("owner.v1", EntryType::Regular, 65532, 0o644), + ] { + let mut builder = Builder::new(Vec::new()); + append(&mut builder, name, &bytes, kind, uid, mode)?; + assert!(record(&builder.into_inner()?, "owner.v1", *b"RSDKOWN2", 56).is_err()); + } + let mut bad = bytes.clone(); + *bad.last_mut().context("test checksum byte")? ^= 1; + assert!(record(&archive("owner.v1", &bad)?, "owner.v1", *b"RSDKOWN2", 56).is_err()); + assert!(record(&archive("owner.v1", &bytes)?, "owner.v1", *b"RSDKOWN3", 56).is_err()); + Ok(()) + } + #[test] + fn test_should_reject_duplicate_extra_truncated_and_concatenated_archives() -> Result<()> { + let bytes = payload(*b"RSDKOWN2", Uuid::new_v4(), None); + let mut builder = Builder::new(Vec::new()); + append( + &mut builder, + "owner.v1", + &bytes, + EntryType::Regular, + 65532, + 0o600, + )?; + append( + &mut builder, + "owner.v1", + &bytes, + EntryType::Regular, + 65532, + 0o600, + )?; + assert!(record(&builder.into_inner()?, "owner.v1", *b"RSDKOWN2", 56).is_err()); + let raw = archive("owner.v1", &bytes)?; + assert!( + record( + raw.get(..1024).context("test short archive")?, + "owner.v1", + *b"RSDKOWN2", + 56 + ) + .is_err() + ); + let mut joined = raw.clone(); + joined.extend_from_slice(&raw); + assert!(record(&joined, "owner.v1", *b"RSDKOWN2", 56).is_err()); + assert!(record(&vec![0; 33280], "owner.v1", *b"RSDKOWN2", 56).is_err()); + Ok(()) + } + #[test] + fn test_should_allow_only_bounded_unique_timing_pax_metadata() -> Result<()> { + timing(b"13 mtime=1.2\n")?; + for bytes in [ + b"12 path=abc\n".as_slice(), + b"13 mtime=1.2\n13 mtime=1.2\n", + b"13 mtime=1e2\n", + b"9999 mtime=1\n", + b"", + b"0 \n", + ] { + assert!(timing(bytes).is_err()); + } + let bytes = payload(*b"RSDKOWN2", Uuid::new_v4(), None); + let mut builder = Builder::new(Vec::new()); + append( + &mut builder, + "PaxHeaders.0/owner.v1", + b"13 mtime=1.2\n", + EntryType::XHeader, + 65532, + 0o600, + )?; + append( + &mut builder, + "owner.v1", + &bytes, + EntryType::Regular, + 65532, + 0o600, + )?; + assert_eq!( + record(&builder.into_inner()?, "owner.v1", *b"RSDKOWN2", 56)?, + bytes + ); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/custom.rs b/apps/rustack/tests/docker_startup/custom.rs new file mode 100644 index 00000000..88a4c653 --- /dev/null +++ b/apps/rustack/tests/docker_startup/custom.rs @@ -0,0 +1,427 @@ +//! Custom-RIC contracts and a separately owned movable operator-side source alias. + +use std::{ + env, + ffi::OsString, + future::pending, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::{ + artifact::ImageReference, + docker::{DockerError, DockerWire}, +}; +use serde::Deserialize; +use serde_json::{Value, json}; +use tokio::{fs, io::AsyncReadExt, runtime::Handle, task::JoinHandle, time::Instant}; +use uuid::Uuid; + +use super::{ + process::{self, Server}, + runner, sdk, +}; + +#[derive(Debug)] +pub(super) struct Alias { + name: String, + ids: [String; 2], + pending: Option, + executable: PathBuf, + record: PathBuf, + command: Option>>, + runtime: Handle, +} +// Observer cancellation leaves the exact process waiter running. This captured- +// runtime reaper joins it; the evidence directory keeps any uncertain tag intent. +impl Drop for Alias { + fn drop(&mut self) { + if let Some(command) = self.command.take() { + self.runtime.spawn(async move { + if command.await.is_err() { + eprintln!( + "custom alias command owner failed; retained intent requires \ + reconciliation" + ); + } + }); + } + } +} +impl Alias { + pub(super) fn name(&self) -> &str { + &self.name + } + pub(super) async fn for_pull(server: &Server, name: String, id: String) -> Result { + ensure!( + name.starts_with("public.ecr.aws:443/lambda/nodejs@sha256:") && name.len() == 104, + "unexpected pull fixture reference" + ); + let executable = + PathBuf::from(env::var_os("RUSTACK_DOCKER_BIN").context("explicit Docker CLI")?); + ensure!(executable.is_absolute(), "Docker CLI must be absolute"); + let owner = Self { + name, + ids: [id.clone(), id.clone()], + pending: Some(id), + executable, + record: server.path()?.join("source-pull-intent.json"), + command: None, + runtime: Handle::current(), + }; + ensure!( + owner.inspect().await?.is_none(), + "pull fixture reference already exists; preserved" + ); + owner.save().await?; + Ok(owner) + } + pub(super) async fn cleanup_after_pull(&mut self, root: &Path) -> Result<()> { + let file = fs::File::open(root.join("pull-complete.json")).await?; + let mut bytes = Vec::new(); + file.take(2049).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 2048, "pull receipt byte cap"); + confirm_pull_receipt(&bytes, &self.name, &self.ids)?; + self.cleanup().await + } + pub(super) async fn new(server: &Server, ids: [String; 2]) -> Result { + let executable = + PathBuf::from(env::var_os("RUSTACK_DOCKER_BIN").context("explicit Docker CLI")?); + ensure!(executable.is_absolute(), "Docker CLI must be absolute"); + let nonce = Uuid::new_v4().simple().to_string(); + let owner = Self { + name: format!("rustack-test-custom-ric-{nonce}:mutable"), + ids, + pending: None, + executable, + record: server.path()?.join(format!("source-alias-{nonce}.json")), + command: None, + runtime: Handle::current(), + }; + ensure!( + owner.inspect().await?.is_none(), + "fixture source alias already exists" + ); + owner.save().await?; + Ok(owner) + } + async fn inspect(&self) -> Result> { + let at = Instant::now() + Duration::from_secs(60); + let mut wire = DockerWire::connect(process::socket()?.parse()?, at).await?; + let reference: ImageReference = self.name.parse()?; + let result = wire.port().inspect_image(&reference, at).await; + let stopped = wire.shutdown().await; + stopped?; + match result { + Ok(image) => Ok(Some(image.id.context("source alias identity")?)), + Err(DockerError::Status(status)) if status.as_u16() == 404 => Ok(None), + Err(error) => Err(error.into()), + } + } + async fn save(&self) -> Result<()> { + let bytes = serde_json::to_vec( + &json!({"alias":self.name,"sourceIds":self.ids,"pendingTarget":self.pending}), + )?; + ensure!(bytes.len() <= 4096, "source intent exceeds limit"); + let pending = self.record.with_extension("pending"); + fs::write(&pending, bytes).await?; + // Keep the preceding complete identity visible if this observer drops + // during writing. Each Alias has its own immutable nonce/path pair. + fs::rename(pending, &self.record).await?; + Ok(()) + } + async fn command(&mut self, arguments: &[&str]) -> Result<()> { + let mut args = vec![OsString::from("--host"), OsString::from(process::socket()?)]; + args.extend(arguments.iter().map(OsString::from)); + ensure!(self.command.is_none(), "alias command already running"); + let child = runner::Runner::spawn(&self.executable, &args)?; + self.command = Some(tokio::spawn(async move { + child.wait(Instant::now() + Duration::from_secs(60)).await + })); + let outcome = self + .command + .as_mut() + .context("retained alias command")? + .await; + self.command.take(); + let report = outcome.context("alias command owner join")??; + ensure!( + !report.forced && report.status.success(), + "owned alias command failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + Ok(()) + } + pub(super) async fn select(&mut self, target: usize) -> Result<()> { + let id = self.ids.get(target).context("source image index")?.clone(); + let current = self.inspect().await?; + if target == 0 { + ensure!(current.is_none(), "alias was concurrently created"); + } else { + ensure!( + current.as_ref() == self.ids.first(), + "alias was concurrently retargeted" + ); + } + self.pending = Some(id.clone()); + self.save().await?; + let name = self.name.clone(); + self.command(&["image", "tag", &id, &name]).await?; + ensure!( + self.inspect().await?.as_ref() == Some(&id), + "tag target did not settle" + ); + self.pending = None; + self.save().await + } + pub(super) async fn cleanup(&mut self) -> Result<()> { + let current = self.inspect().await?; + if let Some(pending) = &self.pending { + ensure!( + current.as_ref() == Some(pending), + "uncertain alias mutation retained for reconciliation" + ); + self.pending = None; + } + if let Some(current) = current { + ensure!( + self.ids.contains(¤t), + "foreign alias target preserved" + ); + let name = self.name.clone(); + self.command(&["image", "rm", "--no-prune", &name]).await?; + } + ensure!( + self.inspect().await?.is_none(), + "fixture alias survived cleanup" + ); + self.save().await + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct PullReceipt { + reference: String, + expected_id: String, + complete: bool, + records: u32, +} + +fn confirm_pull_receipt(bytes: &[u8], name: &str, ids: &[String; 2]) -> Result<()> { + ensure!(bytes.len() <= 2048, "pull receipt byte cap"); + let receipt: PullReceipt = serde_json::from_slice(bytes)?; + ensure!( + receipt.reference == name + && ids.contains(&receipt.expected_id) + && receipt.complete + && (1..=65536).contains(&receipt.records), + "pull completion unproven; source intent retained" + ); + Ok(()) +} + +#[test] +fn test_should_validate_pull_completion_before_granting_cache_cleanup() -> Result<()> { + let name = format!("public.ecr.aws:443/lambda/nodejs@sha256:{}", "a".repeat(64)); + let ids = [ + format!("sha256:{}", "b".repeat(64)), + format!("sha256:{}", "b".repeat(64)), + ]; + let valid = json!({"reference":name,"expectedId":ids.first(),"complete":true,"records":2}); + confirm_pull_receipt(&serde_json::to_vec(&valid)?, &name, &ids)?; + for (field, value) in [ + ("reference", json!("foreign")), + ("expectedId", json!("foreign")), + ("complete", json!(false)), + ("records", json!(0)), + ("records", json!(65537)), + ("records", json!(true)), + ("unknown", json!(1)), + ] { + let mut invalid = valid.clone(); + invalid + .as_object_mut() + .context("receipt object")? + .insert(field.to_owned(), value); + ensure!( + confirm_pull_receipt(&serde_json::to_vec(&invalid)?, &name, &ids).is_err(), + "invalid pull receipt accepted" + ); + } + for raw in [ + b"{".as_slice(), + b"{\"complete\":true,\"complete\":true}", + &[b' '; 2049], + ] { + ensure!( + confirm_pull_receipt(raw, &name, &ids).is_err(), + "unbounded or malformed pull receipt accepted" + ); + } + Ok(()) +} + +async fn python(server: &Server, image: &str, phase: &str, name: &str, tag: &str) -> Result { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let architecture = if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }; + let args = [ + root.join("tests/workflows/custom_ric_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(image), + OsString::from(architecture), + OsString::from(phase), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("explicit collector host")?, + OsString::from(name), + OsString::from(tag), + ]; + let mut child = runner::Runner::spawn(&runner::interpreter()?, &args)?; + let barrier = child.await_pid().await; + let at = if barrier.is_ok() { + Instant::now() + Duration::from_secs(360) + } else { + Instant::now() + }; + let report = child.wait(at).await?; + let pid = barrier?; + ensure!( + report.pid == pid && !report.forced && report.status.success(), + "custom RIC SDK phase failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + let output: Value = serde_json::from_slice(&report.stdout)?; + ensure!( + output["schemaVersion"] == 1, + "missing custom RIC report schema" + ); + Ok(output) +} + +async fn abandon_controls(server: &Server, ids: &[String; 2]) -> Result<()> { + for panic_after_tag in [false, true] { + let mut owner = Alias::new(server, ids.clone()).await?; + owner.select(0).await?; + let (name, record, executable) = ( + owner.name.clone(), + owner.record.clone(), + owner.executable.clone(), + ); + let (entered, observed) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _ = entered.send(()); + assert!( + !panic_after_tag, + "deliberate fixture panic after committed owned tag" + ); + pending::<()>().await; + drop(owner); + }); + observed.await.context("alias abandonment control entry")?; + if !panic_after_tag { + task.abort(); + } + let failure = task + .await + .expect_err("deliberately abandoned alias owner must fail"); + ensure!( + if panic_after_tag { + failure.is_panic() + } else { + failure.is_cancelled() + }, + "wrong fixture owner failure" + ); + let file = fs::File::open(&record).await?; + let mut raw = Vec::new(); + file.take(4097).read_to_end(&mut raw).await?; + ensure!(raw.len() <= 4096, "retained alias record exceeds cap"); + let saved: Value = serde_json::from_slice(&raw)?; + ensure!( + saved["alias"] == name + && saved["sourceIds"] == json!(ids) + && saved["pendingTarget"].is_null(), + "abandoned fixture lost exact ownership evidence" + ); + // Authority transfers only after the old task has joined and its Alias + // dropped; this controlled fully-committed tag has no pending command. + let mut recovered = Alias { + name, + ids: ids.clone(), + pending: None, + executable, + record, + command: None, + runtime: Handle::current(), + }; + ensure!( + recovered.inspect().await?.as_ref() == ids.first(), + "abandoned tag identity differs" + ); + recovered.cleanup().await?; + } + Ok(()) +} + +pub(super) async fn exercise(server: &mut Server) -> Result<()> { + server.ready().await?; + let a = env::var("RUSTACK_CUSTOM_IMAGE_A").context("explicit custom source A")?; + let b = env::var("RUSTACK_CUSTOM_IMAGE_B").context("explicit custom source B")?; + let ids = [server.track_image(&a).await?, server.track_image(&b).await?]; + ensure!( + ids.first() != ids.get(1), + "custom fixture images are identical" + ); + sdk::custom_compute(&server.endpoint, &a, &process::architecture()?).await?; + let mut alias = Alias::new(server, ids.clone()).await?; + let result = async { + let report = python(server, &a, "contracts", "", "").await?; + ensure!( + report["cases"] + == json!([ + "custom-ric-compute-error", + "imageconfig-inherit-clear-replace", + "literal-argv", + "active-cross-invocation-rejection" + ]), + "custom RIC contract inventory differs" + ); + alias.select(0).await?; + let name = format!("MovingImage{}", Uuid::new_v4().simple()); + let report = python(server, &a, "deploy", &name, &alias.name).await?; + ensure!( + report["cases"] == json!(["published-A"]), + "missing publication evidence" + ); + alias.select(1).await?; + let report = python(server, &a, "update", &name, &alias.name).await?; + ensure!( + report["coldRevisionCounters"] == json!([1, 1]), + "missing cold published/latest image evidence" + ); + ensure!( + report["cases"] + == json!([ + "tag-move-retains-A", + "update-selects-B", + "published-alias-isolation", + "published-inflight-A-completes-after-B-update" + ]), + "immutable revision evidence differs" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + let cleanup = alias.cleanup().await; + if cleanup.is_err() && result.is_err() { + eprintln!("custom source alias cleanup remains unconfirmed; primary SDK failure retained"); + } + result?; + cleanup?; + abandon_controls(server, &ids).await +} diff --git a/apps/rustack/tests/docker_startup/faults.rs b/apps/rustack/tests/docker_startup/faults.rs new file mode 100644 index 00000000..77eb07f3 --- /dev/null +++ b/apps/rustack/tests/docker_startup/faults.rs @@ -0,0 +1,164 @@ +//! Actual failing user code, signed caller loss and bounded accepted-work shutdown. + +use std::{env, ffi::OsString, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; +use tokio::{fs, io::AsyncReadExt, time::Instant}; + +use super::{ + process::{self, Profile, Server}, + runner, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Case { + Faults, + Initializing, + Busy, + Saturation, +} +impl Case { + pub(super) fn profile(self) -> Profile { + if self == Self::Faults { + Profile::Connected + } else { + Profile::Controlled + } + } + fn phase(self) -> &'static str { + match self { + Self::Faults => "faults", + Self::Initializing => "initializing", + Self::Busy => "busy", + Self::Saturation => "saturation", + } + } + fn expected(self) -> Value { + match self { + Self::Faults => json!([ + "init-error", + "prepoll-exit", + "bootstrap-timeout", + "handler-exit", + "cgroup-oom", + "handler-timeout" + ]), + Self::Initializing => json!([ + "reserved-zero-no-allocation", + "reserved-range-atomic", + "held-reserved-one", + "initializing-disconnect-removal-before-expiry", + "cold-recovery" + ]), + Self::Busy => json!([ + "reserved-zero-no-allocation", + "reserved-range-atomic", + "held-reserved-one", + "busy-disconnect-removal-before-expiry", + "cold-recovery" + ]), + Self::Saturation => json!([ + "impossible-memory-429", + "event-capacity-128", + "event-129-throttled", + "two-pairs-384MiB", + "held-backlog-shutdown-cutoff" + ]), + } + } +} + +pub(super) async fn exercise(server: &mut Server, case: Case) -> Result<()> { + server.ready().await?; + let source = env::var("RUSTACK_CUSTOM_IMAGE_A").context("explicit fault fixture source")?; + server.track_image(&source).await?; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let architecture = if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }; + let (owner, engine) = server.observation_identity().await?; + let arguments = [ + root.join("tests/workflows/lambda_fault_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(source), + OsString::from(architecture), + OsString::from(case.phase()), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("explicit hold collector host")?, + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit metadata socket")?, + OsString::from(owner), + OsString::from(engine), + ]; + let mut child = runner::Runner::spawn(&runner::interpreter()?, &arguments)?; + let entered = child.await_pid().await; + let gate = if case == Case::Saturation && entered.is_ok() { + child + .await_pid_before(Instant::now() + Duration::from_secs(180)) + .await + .map(Some) + } else { + Ok(None) + }; + let shutdown_started = Instant::now(); + let stopped = if case == Case::Saturation || entered.is_err() || gate.is_err() { + server.stop(case != Case::Saturation).await + } else { + Ok(()) + }; + let shutdown_elapsed = shutdown_started.elapsed(); + let deadline = if entered.is_ok() && gate.is_ok() { + Instant::now() + Duration::from_secs(180) + } else { + Instant::now() + }; + // Every barrier and server-stop outcome is saved before joining the same child. + let report = child.wait(deadline).await?; + let diagnostics = String::from_utf8_lossy(&report.stderr); + let pid = entered.with_context(|| format!("fault SDK entry failed: {diagnostics}"))?; + let ready = gate.with_context(|| format!("fault SDK shutdown gate failed: {diagnostics}"))?; + if let Some(ready) = ready { + ensure!(ready == pid, "shutdown gate PID changed"); + } + stopped?; + if case == Case::Saturation { + ensure!( + shutdown_elapsed >= Duration::from_secs(25), + "server failed before the declared drain cutoff" + ); + let file = fs::File::open(server.path()?.join("server.log")).await?; + let mut bytes = Vec::new(); + file.take(32769).read_to_end(&mut bytes).await?; + ensure!( + bytes.len() <= 32768 + && String::from_utf8_lossy(&bytes) + .contains("Lambda quiesce did not confirm all invocation cleanup"), + "wrong shutdown failure instead of the declared drain cutoff" + ); + } + ensure!( + report.pid == pid && !report.forced && report.status.success(), + "actual fault SDK process failed: {diagnostics}" + ); + let observed: Value = serde_json::from_slice(&report.stdout)?; + ensure!( + observed["schemaVersion"] == 1 && observed["cases"] == case.expected(), + "fault semantic inventory differs" + ); + Ok(()) +} + +pub(super) async fn check_logs(server: &Server) -> Result<()> { + let file = fs::File::open(server.path()?.join("server.log")).await?; + let mut bytes = Vec::new(); + file.take(32769).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 32768, "fault log evidence exceeds cap"); + ensure!( + !String::from_utf8_lossy(&bytes).contains("fixture-init-payload-sentinel") + && !String::from_utf8_lossy(&bytes).contains("runtime-boundary-secret"), + "fixture payload/header leaked into diagnostics" + ); + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/handler_sdk.rs b/apps/rustack/tests/docker_startup/handler_sdk.rs new file mode 100644 index 00000000..29c4397f --- /dev/null +++ b/apps/rustack/tests/docker_startup/handler_sdk.rs @@ -0,0 +1,77 @@ +//! Actual default SDK provider chain on a qualified local gateway. + +use std::{env, ffi::OsString, path::PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; + +use super::{ + image_sdk, + process::{self, Profile, Server}, +}; + +pub(super) async fn exercise(wrong: bool) -> Result<()> { + ensure!( + cfg!(all(feature = "s3", feature = "dynamodb")), + "Docker handler SDK resource requires explicit s3,dynamodb source features" + ); + let profile = if wrong { + Profile::SdkWrongCredentials + } else { + Profile::Sdk + }; + image_sdk::exercise( + profile, + async |server: &mut Server| prepare(server, wrong).await, + |observed| validate(observed, wrong), + ) + .await +} +async fn prepare(server: &mut Server, wrong: bool) -> Result> { + let image = env::var("RUSTACK_HANDLER_SDK_IMAGE").context("explicit handler SDK image")?; + server.track_image(&image).await?; + server.ready().await?; + let (owner, engine) = server.observation_identity().await?; + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/handler_sdk_image.py"); + Ok(vec![ + script.into_os_string(), + OsString::from(&server.endpoint), + OsString::from(image), + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("SDK metadata socket")?, + OsString::from(owner), + OsString::from(engine), + OsString::from(if wrong { "wrong" } else { "positive" }), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("qualified gateway host")?, + ]) +} +fn validate(observed: &Value, wrong: bool) -> Result<()> { + let cases = if wrong { + json!([ + "explicit-invalid-s3-authorization", + "explicit-invalid-dynamodb-authorization", + "no-ambient-fallback" + ]) + } else { + json!([ + "quiet-body-abort", + "local-s3-dynamodb-cold-warm", + "authoritative-SDK-env", + "metadata-secret-omission" + ]) + }; + ensure!( + observed["schemaVersion"] == 1 + && observed["mode"] == if wrong { "wrong" } else { "positive" } + && observed["versions"] == json!({"s3":"3.1105.0","dynamodb":"3.1105.0"}) + && observed["cases"] == cases, + "handler SDK semantic inventory differs" + ); + eprintln!("actual handler SDK versions: {}", observed["versions"]); + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/iac.rs b/apps/rustack/tests/docker_startup/iac.rs new file mode 100644 index 00000000..ac322c6c --- /dev/null +++ b/apps/rustack/tests/docker_startup/iac.rs @@ -0,0 +1,214 @@ +//! Actual Pulumi provider lifecycle and whole-function plugin ownership. + +use std::{collections::HashSet, env, ffi::OsString, path::PathBuf}; + +use anyhow::{Context, Result, anyhow, ensure}; +use serde::Deserialize; +use serde_json::Value; +use uuid::Uuid; + +use super::{ + image_sdk, + process::{self, Profile, Server}, +}; + +const COMMANDS: [&str; 18] = [ + "version", + "login", + "init", + "create", + "refresh-created", + "update", + "refresh-updated", + "reject-publish", + "recover-publish", + "settle-publish", + "reject-logging", + "recover-logging", + "settle-logging", + "reject-encryption", + "recover-encryption", + "settle-encryption", + "destroy", + "remove", +]; +const STAGES: [(&str, u8); 13] = [ + ("created", 4), + ("refreshed-created", 5), + ("updated", 6), + ("refreshed-updated", 7), + ("rejected-publish", 8), + ("recovered-publish", 10), + ("rejected-logging", 11), + ("recovered-logging", 13), + ("rejected-encryption", 14), + ("recovered-encryption", 16), + ("pre-destroy", 16), + ("post-destroy", 17), + ("post-removal", 18), +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Mode { + Lifecycle, + Stop, +} +impl Mode { + fn name(self) -> &'static str { + match self { + Self::Lifecycle => "lifecycle", + Self::Stop => "stop", + } + } +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Versions { + cli: String, + provider: String, + sdk: String, + aws: String, + node: String, + dynamodb: String, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Stage { + stage: String, + ordinal: u8, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Evidence { + schema_version: u8, + mode: String, + versions: Versions, + stages: Vec, + audits: Vec, + commands: Vec, + executions: Vec, + business_requests: Vec, + request_id: String, + physical_stop: bool, + actual_destroy: bool, +} + +fn validate(mode: Mode, observed: &Value) -> Result<()> { + ensure!( + serde_json::to_vec(observed)?.len() <= 16 * 1024, + "IaC report cap" + ); + let value: Evidence = serde_json::from_value(observed.clone()) + .map_err(|_| anyhow!("invalid managed IaC report"))?; + let positive = mode == Mode::Lifecycle; + ensure!( + value.schema_version == 1 + && value.mode == mode.name() + && value.physical_stop != positive + && value.actual_destroy == positive + && value.versions.cli == "3.273.0" + && value.versions.sdk == "3.273.0" + && value.versions.provider == "7.52.1" + && value.versions.aws == "7.52.1" + && value.versions.node == "24.22.1" + && value.versions.dynamodb == "3.1105.0", + "IaC semantic or source versions differ" + ); + let stages = STAGES + .get(..if positive { STAGES.len() } else { 1 }) + .context("literal IaC stage catalog")?; + let commands = COMMANDS + .get(..if positive { COMMANDS.len() } else { 4 }) + .context("literal IaC command catalog")?; + let audits: &[&str] = if positive { + &["pre-destroy", "post-destroy", "post-removal"] + } else { + &[] + }; + ensure!( + value.stages.len() == stages.len() + && value + .stages + .iter() + .map(|row| (row.stage.as_str(), row.ordinal)) + .eq(stages.iter().copied()) + && value + .commands + .iter() + .map(String::as_str) + .eq(commands.iter().copied()) + && value + .audits + .iter() + .map(String::as_str) + .eq(audits.iter().copied()), + "actual IaC lifecycle catalog differs" + ); + ensure!( + value.request_id.len() == 36 + && Uuid::parse_str(&value.request_id) + .is_ok_and(|id| id.get_version_num() == 4 && id.to_string() == value.request_id) + && value.business_requests.len() == if positive { 10 } else { 3 } + && value.business_requests.iter().collect::>().len() + == value.business_requests.len() + && value.business_requests.iter().all(|id| id.len() == 36 + && Uuid::parse_str(id) + .is_ok_and(|uuid| uuid.get_version_num() == 4 && uuid.to_string() == *id)) + && value.executions.len() == if positive { 8 } else { 1 } + && value.executions.iter().collect::>().len() == value.executions.len() + && value + .executions + .iter() + .all(|arn| (1..=256).contains(&arn.len()) + && arn + .starts_with("arn:aws:states:us-east-1:000000000000:execution:IacTarget") + && arn + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_:-".contains(&byte))), + "invalid IaC invocation or independent execution identities" + ); + eprintln!( + "actual managed IaC CLI3.273.0/provider7.52.1: {}", + mode.name() + ); + Ok(()) +} + +pub(super) async fn exercise(mode: Mode) -> Result<()> { + image_sdk::exercise( + Profile::Iac, + async |server: &mut Server| { + let tool = env::var("RUSTACK_PULUMI_IMAGE").context("explicit Pulumi image")?; + let business = env::var("RUSTACK_NODE_IMAGE").context("explicit business image")?; + server.track_image(&tool).await?; + server.track_image(&business).await?; + let capabilities = server.ready().await?; + ensure!( + capabilities["workflows"]["lambdaRequestResponse"] == true + && capabilities["workflows"]["limits"]["dispatches"] == 2, + "IaC capability/profile mismatch" + ); + let (owner, engine) = server.observation_identity().await?; + Ok(vec![ + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/pulumi_image_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(tool), + OsString::from(business), + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("IaC metadata socket")?, + OsString::from(owner), + OsString::from(engine), + OsString::from(mode.name()), + server.path()?.as_os_str().to_owned(), + ]) + }, + |observed| validate(mode, observed), + ) + .await +} diff --git a/apps/rustack/tests/docker_startup/image_sdk.rs b/apps/rustack/tests/docker_startup/image_sdk.rs new file mode 100644 index 00000000..4ebe50c6 --- /dev/null +++ b/apps/rustack/tests/docker_startup/image_sdk.rs @@ -0,0 +1,101 @@ +//! Shared ordinary image SDK child and server ownership through every exit. + +use std::{ffi::OsString, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use tokio::time::{Instant, timeout_at}; + +use super::{ + faults::check_logs, + process::{Profile, Server}, + runner::{Report, Runner, interpreter}, +}; + +pub(super) async fn exercise(profile: Profile, prepare: F, validate: V) -> Result<()> +where + F: for<'a> AsyncFnOnce(&'a mut Server) -> Result>, + V: FnOnce(&Value) -> Result<()>, +{ + exercise_with_audit(profile, prepare, validate, async |_, _| Ok(())).await +} + +pub(super) async fn exercise_with_audit( + profile: Profile, + prepare: F, + validate: V, + after_stop: A, +) -> Result<()> +where + F: for<'a> AsyncFnOnce(&'a mut Server) -> Result>, + V: FnOnce(&Value) -> Result<()>, + A: for<'a> AsyncFnOnce(&'a Server, &'a Value) -> Result<()>, +{ + let mut server = Server::spawn(profile).await?; + server + .root + .as_mut() + .context("image SDK evidence root")? + .disable_cleanup(true); + let cutoff = + Instant::now() + Duration::from_secs(if profile == Profile::Iac { 1020 } else { 480 }); + let prepared = timeout_at(cutoff, prepare(&mut server)) + .await + .context("ordinary image SDK preparation expired") + .and_then(|result| result); + let result = async { + let arguments = prepared?; + let child = Runner::spawn(&interpreter()?, &arguments)?; + let (entered, report) = join_with_pid_before(child, cutoff).await?; + ensure!( + report.pid == entered? && !report.forced && report.status.success(), + "ordinary image SDK failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + let observed: Value = serde_json::from_slice(&report.stdout)?; + validate(&observed)?; + Ok::(observed) + } + .await; + let stopped = server.stop(true).await; + let audited = server.audit().await; + let logs = check_logs(&server).await; + let saved = match &result { + Ok(observed) if stopped.is_ok() && audited.is_ok() && logs.is_ok() => { + after_stop(&server, observed).await + } + _ => Ok(()), + }; + if result.is_err() || stopped.is_err() || audited.is_err() || logs.is_err() || saved.is_err() { + server.retain_on_error().await; + } + result?; + stopped?; + audited?; + logs?; + saved?; + server + .root + .as_mut() + .context("image SDK root after cleanup")? + .disable_cleanup(false); + Ok(()) +} + +/// Join this same child after the original-cutoff PID barrier, including failure. +pub(super) async fn join_with_pid_before( + mut child: Runner, + cutoff: Instant, +) -> Result<(Result, Report)> { + let entered = child + .await_pid_before(cutoff.min(Instant::now() + Duration::from_secs(5))) + .await; + let report = child + .wait(if entered.is_ok() { + cutoff + } else { + Instant::now() + }) + .await?; + Ok((entered, report)) +} diff --git a/apps/rustack/tests/docker_startup/io_faults.rs b/apps/rustack/tests/docker_startup/io_faults.rs new file mode 100644 index 00000000..3593a23f --- /dev/null +++ b/apps/rustack/tests/docker_startup/io_faults.rs @@ -0,0 +1,348 @@ +//! A joined real-daemon proxy remains live through ordinary server cleanup. + +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; +use tempfile::Builder; +use tokio::{fs, time::Instant}; + +use super::{ + custom::Alias, + faults::check_logs, + process::{Profile, Server, architecture}, + runner::{Runner, interpreter}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Case { + Create, + Delete, + Loss, + Saturation, + CreateTimeout, + DeleteTimeout, + Pull, + Negative, + RuntimeBounds, + CompositePressure, + GlobalLimit, + DefaultLimit, + MemoryLimit, + EnvironmentLimit, + ZipCreate, + ZipDelete, +} +impl Case { + fn zip(self) -> bool { + matches!(self, Self::ZipCreate | Self::ZipDelete) + } + fn sources(self) -> Result<[String; 2]> { + let [first, second] = if self.zip() { + ["RUSTACK_PROVIDED_ZIP_A", "RUSTACK_PROVIDED_ZIP_B"] + } else { + ["RUSTACK_CUSTOM_IMAGE_A", "RUSTACK_CUSTOM_IMAGE_B"] + }; + Ok([ + env::var(first).context("explicit IO source A")?, + env::var(second).context("explicit IO source B")?, + ]) + } + fn phase(self) -> &'static str { + match self { + Self::ZipCreate => "zip_create", + Self::ZipDelete => "zip_delete", + Self::Create => "create", + Self::Delete => "delete", + Self::Loss => "loss", + Self::Saturation => "saturation", + Self::CreateTimeout => "create_timeout", + Self::DeleteTimeout => "delete_timeout", + Self::Pull => "pull", + Self::Negative => "negative", + Self::RuntimeBounds => "runtime_bounds", + Self::CompositePressure => "composite_pressure", + Self::GlobalLimit => "global_limit", + Self::DefaultLimit => "default_limit", + Self::MemoryLimit => "memory_limit", + Self::EnvironmentLimit => "environment_limit", + } + } + fn expected(self) -> Value { + match self { + Self::ZipCreate | Self::ZipDelete | Self::Create | Self::Delete => json!([ + format!("{}-held-real-io", self.phase()), + "retained-reserved-slot", + "caller-disconnected", + "post-release-physical-retirement", + "cold-computed-recovery" + ]), + Self::Loss => json!([ + "image-management-ack-lost", + "failed-update-preserves-active-A", + "published-A-preserved", + "recovered-B-computation", + "missing-source-never-active", + "full-loss-revokes-readiness" + ]), + Self::Saturation => json!([ + "two-held-pending-deployments", + "third-rejected-before-insertion", + "deleted-preparation-retains-capacity", + "deleted-job-never-resurrects", + "capacity-recovery-computation" + ]), + Self::CreateTimeout | Self::DeleteTimeout => json!([ + format!("{}-held-after-effect", self.phase()), + "live-caller-production-io-expiration", + "physical-retirement-before-release", + "typed-primary-error", + "cold-computed-recovery" + ]), + Self::Pull => json!([ + "actual-ifMissing-pull", + "truncated-progress-rejected", + "failed-update-preserves-active-A", + "backend-pull-positively-drained", + "recovered-B-computation" + ]), + Self::CompositePressure => json!([ + "large-nested-map-owned-child", + "oversized-runtime-frame-under-load", + "live-ready-and-stop", + "physical-pair-retirement", + "typed-payload-quota", + "cold-slot-recovery" + ]), + Self::RuntimeBounds => json!([ + "runtime-http-405-431-413-413-400", + "same-attempt-computed-result", + "warm-counter-two", + "retired-healthy" + ]), + Self::GlobalLimit | Self::DefaultLimit => json!([ + format!("{}-positive-capacity-proof", self.phase()), + "held-delete-retains-charge", + "physical-retirement-before-recovery", + "computed-recovery" + ]), + Self::MemoryLimit | Self::EnvironmentLimit => json!([ + format!("{}-independent-ceiling", self.phase()), + "admitted-B-waits-without-allocation", + "retained-until-old-pair-retired", + "B-cold-computed-recovery" + ]), + Self::Negative => json!([ + "real-wrong-os-rejected", + "real-wrong-architecture-rejected", + "protocol-free-bootstrap-timeout", + "cold-computed-recovery" + ]), + } + } +} + +pub(super) async fn exercise(case: Case) -> Result<()> { + let mut root = Builder::new().prefix("rustack-io-").tempdir_in("/tmp")?; + root.disable_cleanup(true); + eprintln!("management IO evidence: {}", root.path().display()); + let repository = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let sources = case.sources()?; + let architecture = if architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }; + let arguments = [ + repository + .join("tests/workflows/lambda_io_fault_sdk.py") + .into_os_string(), + root.path().as_os_str().to_owned(), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit actual daemon")?, + OsString::from(&sources[0]), + OsString::from(&sources[1]), + OsString::from(architecture), + OsString::from(case.phase()), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("explicit init hold host")?, + ]; + let mut child = Runner::spawn(&interpreter()?, &arguments)?; + let mut server = None; + let mut pull = None; + let semantic = handoff( + &mut server, + &mut pull, + &mut child, + root.path(), + &sources, + case, + ) + .await; + // Keep the proxy transparent and alive through server shutdown on every result. + let restored = fs::write(root.path().join("restore"), b"restore").await; + let stopped = match &mut server { + Some(ordinary) => { + if case == Case::Loss { + ordinary.stop_after_transport_loss().await + } else { + ordinary.stop(true).await + } + } + None => Ok(()), + }; + let cache_cleanup = match &mut pull { + Some(owner) => owner.cleanup_after_pull(root.path()).await, + None => Ok(()), + }; + let (audited, logs) = match &server { + Some(ordinary) => (ordinary.audit().await, check_logs(ordinary).await), + None => (Ok(()), Ok(())), + }; + let stop = fs::write(root.path().join("stop"), b"stop").await; + let joined = child.wait(Instant::now() + Duration::from_secs(15)).await; + let result = (|| { + let report = joined?; + let diagnostic = String::from_utf8_lossy(&report.stderr); + semantic.with_context(|| format!("IO SDK handoff failed: {diagnostic}"))?; + restored?; + stop?; + ensure!( + !report.forced && report.status.success(), + "IO SDK failed: {diagnostic}" + ); + stopped?; + cache_cleanup?; + audited?; + logs?; + let observed: Value = serde_json::from_slice(&report.stdout)?; + ensure!( + observed["schemaVersion"] == 1 && observed["cases"] == case.expected(), + "IO semantic inventory differs" + ); + Ok::<(), anyhow::Error>(()) + })(); + if result.is_err() { + if let Some(ordinary) = &mut server { + ordinary.retain_on_error().await; + } + } else { + root.disable_cleanup(false); + if let Some(ordinary) = &mut server { + ordinary + .root + .as_mut() + .context("ordinary evidence guard")? + .disable_cleanup(false); + } + } + result +} + +async fn handoff( + server: &mut Option, + pull: &mut Option, + child: &mut Runner, + root: &Path, + sources: &[String; 2], + case: Case, +) -> Result<()> { + let pid = child.await_pid().await?; + ensure!( + child.await_pid().await? == pid, + "proxy readiness PID changed" + ); + let profile = match case { + Case::ZipCreate | Case::ZipDelete => Profile::ZipControlled, + Case::CreateTimeout | Case::DeleteTimeout => Profile::Deadline, + Case::Pull => Profile::Pull, + Case::Negative => Profile::Negative, + Case::MemoryLimit => Profile::MemoryPressure, + Case::DefaultLimit | Case::EnvironmentLimit => Profile::EnvironmentPressure, + _ => Profile::Controlled, + }; + let mut ordinary = Server::spawn_with_socket(profile, Some(&root.join("engine.sock"))).await?; + ordinary + .root + .as_mut() + .context("ordinary evidence guard")? + .disable_cleanup(true); + *server = Some(ordinary); + let ordinary = server.as_mut().context("ordinary server owner")?; + if case.zip() { + ordinary + .track_image(&env::var("RUSTACK_ZIP_BASE_IMAGE")?) + .await?; + } else { + for source in sources { + ordinary.track_image(source).await?; + } + } + let mut fixtures = Vec::new(); + if case == Case::Negative { + let selected = architecture()?; + let opposite = if selected == "arm64" { + "amd64" + } else { + "arm64" + }; + let mut identities = Vec::new(); + for (name, os, arch) in [ + ("invalid-os", "windows", "amd64"), + ("invalid-architecture", "linux", opposite), + ("no-runtime", "linux", selected.as_str()), + ] { + let reference = format!("rustack-{name}:{selected}"); + let id = ordinary.track_platform_image(&reference, os, arch).await?; + ensure!( + !identities.contains(&id), + "negative fixtures share content identity" + ); + identities.push(id.clone()); + let proof = json!({"reference":reference,"id":id,"os":os,"architecture":arch}); + fs::write( + root.join(format!("{name}-platform.json")), + serde_json::to_vec(&proof)?, + ) + .await?; + fixtures.push(reference); + } + } + if case == Case::Pull { + let source = + env::var("RUSTACK_PULL_IMAGE").context("explicit immutable public pull fixture")?; + let suffix = source + .strip_prefix("public.ecr.aws/lambda/nodejs@sha256:") + .context("public Node digest fixture required")?; + ensure!( + suffix.len() == 64 + && suffix + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()), + "canonical source digest required" + ); + let id = ordinary.track_image(&source).await?; + let alternate = format!("public.ecr.aws:443/lambda/nodejs@sha256:{suffix}"); + *pull = Some(Alias::for_pull(ordinary, alternate.clone(), id.clone()).await?); + fixtures.extend([source, alternate, id]); + } + ordinary.ready().await?; + let (owner, engine) = ordinary.observation_identity().await?; + let bytes = serde_json::to_vec( + &json!({"endpoint":ordinary.endpoint,"owner":owner,"engine":engine,"fixtures":fixtures}), + )?; + ensure!(bytes.len() <= 2048, "proxy handoff byte cap"); + fs::write(root.join("config.pending"), bytes).await?; + fs::rename(root.join("config.pending"), root.join("config.json")).await?; + ensure!( + child + .await_pid_before(Instant::now() + Duration::from_secs(180)) + .await? + == pid, + "SDK completion PID changed" + ); + Ok::<(), anyhow::Error>(()) +} diff --git a/apps/rustack/tests/docker_startup/node.rs b/apps/rustack/tests/docker_startup/node.rs new file mode 100644 index 00000000..3867b6a0 --- /dev/null +++ b/apps/rustack/tests/docker_startup/node.rs @@ -0,0 +1,95 @@ +//! Real official Node runtime consumption with retained server and SDK owners. + +use std::{env, ffi::OsString, path::PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; + +use super::{ + image_sdk, + process::{self, Profile, Server}, + sdk, +}; + +pub(super) async fn exercise() -> Result<()> { + image_sdk::exercise(Profile::Connected, prepare, validate).await +} + +async fn prepare(server: &mut Server) -> Result> { + let image = env::var("RUSTACK_NODE_IMAGE").context("explicit official Node fixture")?; + server.track_image(&image).await?; + server.ready().await?; + sdk::node_compute(&server.endpoint, &image, &process::architecture()?).await?; + let (owner, engine) = server.observation_identity().await?; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + Ok(vec![ + root.join("tests/workflows/node_image_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + OsString::from(image), + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Node metadata socket")?, + OsString::from(owner), + OsString::from(engine), + ]) +} + +fn validate(observed: &Value) -> Result<()> { + ensure!( + observed["schemaVersion"] == 1 + && observed["cases"] + == json!([ + "cold-warm-context-filesystem-logs", + "published-alias-command-isolation", + "FunctionError-replacement-warm", + "ASL-node-computation" + ]) + && observed["nodeVersion"] + .as_str() + .is_some_and(|version| version.starts_with("24.")), + "Node semantic inventory or runtime differs" + ); + eprintln!("official Node runtime version: {}", observed["nodeVersion"]); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{ffi::OsString, path::Path}; + + use anyhow::{Result, ensure}; + use nix::{errno::Errno, sys::signal::kill, unistd::Pid}; + use tokio::time::{Duration, Instant}; + + use super::super::{image_sdk::join_with_pid_before, runner::Runner}; + + #[tokio::test] + async fn test_should_join_late_node_sdk_child_before_reporting_expired_handshake() -> Result<()> + { + let cutoff = Instant::now() - Duration::from_millis(1); + let mut child = Runner::spawn( + Path::new("/bin/sh"), + &[ + OsString::from("-c"), + OsString::from("echo $$; exec /bin/sleep 60"), + ], + )?; + let held = child.await_pid().await; + let (entered, report) = join_with_pid_before(child, cutoff).await?; + let held_pid = held?; + ensure!( + entered.is_err() && report.pid == held_pid && report.forced && !report.status.success(), + "late SDK accepted" + ); + let pid = Pid::from_raw(i32::try_from(report.pid)?); + ensure!( + kill(pid, None) == Err(Errno::ESRCH), + "late SDK child remains after joined wait" + ); + Ok(()) + } +} diff --git a/apps/rustack/tests/docker_startup/performance.rs b/apps/rustack/tests/docker_startup/performance.rs new file mode 100644 index 00000000..22a23613 --- /dev/null +++ b/apps/rustack/tests/docker_startup/performance.rs @@ -0,0 +1,145 @@ +//! Real signed image measurements, published only after physical server cleanup. +#![warn( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::{env, ffi::OsString, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use tokio::{ + fs, + io::AsyncReadExt, + time::{Instant, timeout}, +}; + +use super::{ + image_sdk, + process::{Profile, Server, architecture}, + runner::{Runner, interpreter}, +}; + +pub(super) async fn exercise(phase: &'static str) -> Result<()> { + image_sdk::exercise_with_audit( + Profile::Performance, + async move |server| prepare(server, phase).await, + move |observed| validate(observed, phase), + async move |server, observed| audit(server, observed, phase).await, + ) + .await +} + +async fn prepare(server: &mut Server, phase: &str) -> Result> { + server.ready().await?; + let baseline = read_log(server.path()?.join("server.log")).await?; + ensure!( + baseline.len() <= 32768 && baseline.ends_with(b"\n"), + "initialization baseline bound or framing" + ); + timeout( + Duration::from_secs(5), + fs::write(server.path()?.join("initialization-baseline.log"), baseline), + ) + .await + .context("initialization baseline write expired")??; + let source = env::var("RUSTACK_CUSTOM_IMAGE_A")?; + server.track_image(&source).await?; + let (owner, engine) = server.observation_identity().await?; + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/image_performance_sdk.py"); + Ok(vec![ + script.into_os_string(), + OsString::from(&server.endpoint), + OsString::from(source), + OsString::from(if architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + OsString::from(phase), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit metadata socket")?, + OsString::from(owner), + OsString::from(engine), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("explicit collector gateway")?, + OsString::from(server.process_id()?.to_string()), + OsString::from("v1"), + env::var_os("RUSTACK_WORKFLOWS_SDK_SITE_PACKAGES").unwrap_or_else(|| OsString::from("-")), + ]) +} + +fn validate(observed: &Value, phase: &str) -> Result<()> { + ensure!( + observed.get("schemaVersion") == Some(&Value::from(1)) + && observed.get("phase").and_then(Value::as_str) == Some(phase) + && observed.get("emptyAfterWorkload") == Some(&Value::Bool(true)) + && observed + .get("referenceQualification") + .and_then(Value::as_str) + == Some("pending"), + "image performance report identity or cleanup differs" + ); + Ok(()) +} + +async fn audit(server: &Server, observed: &Value, phase: &str) -> Result<()> { + let log = read_log(server.path()?.join("server.log")).await?; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/image_performance_report.py"); + let directory = server.path()?; + let input = directory.join("image-performance.json"); + let mut observed = observed.clone(); + observed + .as_object_mut() + .context("measurement report object")? + .insert( + "debugAssertions".to_owned(), + Value::Bool(cfg!(debug_assertions)), + ); + timeout( + Duration::from_secs(5), + fs::write(&input, serde_json::to_vec(&observed)?), + ) + .await + .context("measurement evidence write expired")??; + let child = Runner::spawn( + &interpreter()?, + &[ + root.into_os_string(), + input.into_os_string(), + directory.join("server.log").into_os_string(), + OsString::from(phase), + directory + .join("initialization-baseline.log") + .into_os_string(), + ], + )?; + let (entered, report) = + image_sdk::join_with_pid_before(child, Instant::now() + Duration::from_secs(10)).await?; + ensure!( + report.pid == entered? && !report.forced && report.status.success(), + "independent image measurement validation failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + let qualified: Value = serde_json::from_slice(&report.stdout)?; + ensure!(!log.is_empty(), "missing actual initialization log"); + println!( + "WORKFLOWS_IMAGE_PERFORMANCE {}", + serde_json::to_string(&qualified)? + ); + Ok(()) +} + +async fn read_log(path: PathBuf) -> Result> { + timeout(Duration::from_secs(5), async { + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(32769).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 32768, "initialization log evidence bound"); + Ok(bytes) + }) + .await + .context("initialization log read expired")? +} diff --git a/apps/rustack/tests/docker_startup/process.rs b/apps/rustack/tests/docker_startup/process.rs new file mode 100644 index 00000000..665b7f3a --- /dev/null +++ b/apps/rustack/tests/docker_startup/process.rs @@ -0,0 +1,742 @@ +//! Exact owned executable, data directory and post-exit daemon audit. + +use std::{ + env, + fs::Permissions, + io::SeekFrom, + os::unix::{fs::PermissionsExt, process::ExitStatusExt}, + path::{Path, PathBuf}, + process::{ExitStatus, Stdio}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use nix::{ + sys::signal::{Signal, kill}, + unistd::Pid, +}; +use rustack_lambda_core::{ + artifact::ImageReference, + docker::{DirectoryLease, DockerWire, reconcile_namespace}, +}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tempfile::{Builder as TempDirBuilder, TempDir}; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + process::{Child, Command}, + time::{Instant, sleep, timeout}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Profile { + Disconnected, + Connected, + Performance, + Sdk, + SdkWrongCredentials, + Callback, + Iac, + Controlled, + Deadline, + Pull, + Negative, + MemoryPressure, + EnvironmentPressure, + MissingRelay, + Legacy, + WrongEndpoint, + Archive, + Zip, + ZipControlled, + ZipArchive, + ZipArchiveNoSource, +} +impl Profile { + fn configure_diagnostics(self, command: &mut Command) { + if self == Self::Performance { + command + .env( + "LOG_LEVEL", + "error,rustack_lambda_core::docker::initialization=info", + ) + .env("NO_COLOR", "1") + .env("LAMBDA_IDLE_TIMEOUT_SECS", "2"); + } else { + command.env("LOG_LEVEL", "error"); + } + } + fn sdk(self) -> bool { + matches!( + self, + Self::Sdk | Self::SdkWrongCredentials | Self::Callback | Self::Iac + ) + } + fn services(self) -> &'static str { + if self.sdk() { + "lambda,stepfunctions,s3,dynamodb" + } else { + "lambda,stepfunctions" + } + } + pub fn rejects(self) -> bool { + matches!( + self, + Self::MissingRelay | Self::Legacy | Self::WrongEndpoint + ) + } + pub(super) fn matches_failure(self, text: &str) -> bool { + let expected = match self { + Self::MissingRelay => { + "Docker startup relay preparation failed: Docker response status 404 Not Found" + } + Self::Legacy => "Docker ownership predates journal provenance", + Self::WrongEndpoint => "Docker startup endpoint qualification failed", + Self::Archive => "Restored image storage proof mismatch", + Self::ZipArchive => "Invalid or oversized Docker ZIP archive candidate", + Self::ZipArchiveNoSource => "snapshot CBOR scalar limit exceeded", + _ => return false, + }; + text.contains(expected) + } + fn init_seconds(self) -> &'static str { + if matches!( + self, + Self::Sdk + | Self::Performance + | Self::SdkWrongCredentials + | Self::Callback + | Self::Iac + | Self::ZipControlled + | Self::Controlled + | Self::Deadline + | Self::Pull + | Self::MemoryPressure + | Self::EnvironmentPressure + ) { + "20" + } else { + "5" + } + } + fn zip(self) -> bool { + matches!(self, Self::Zip | Self::ZipControlled | Self::ZipArchive) + } + fn network(self) -> bool { + matches!( + self, + Self::Sdk + | Self::Performance + | Self::SdkWrongCredentials + | Self::Callback + | Self::Iac + | Self::Zip + | Self::ZipControlled + | Self::ZipArchive + | Self::Connected + | Self::Controlled + | Self::Deadline + | Self::Pull + | Self::Negative + | Self::MemoryPressure + | Self::EnvironmentPressure + | Self::WrongEndpoint + ) + } +} +#[derive(Debug)] +pub(super) struct Server { + child: Child, + pub root: Option, + pub endpoint: String, + profile: Profile, + sources: Vec<(ImageReference, String)>, + socket_override: Option, + log_start: u64, +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +fn private_root() -> Result { + TempDirBuilder::new() + .permissions(Permissions::from_mode(0o700)) + .tempdir() + .context("create private Docker fixture evidence") +} +pub(super) fn image() -> Result { + Ok(env::var("RUSTACK_RUNTIME_IMAGE")?) +} +pub(super) fn architecture() -> Result { + let value = env::var("RUSTACK_RUNTIME_ARCH")?; + ensure!( + matches!(value.as_str(), "arm64" | "amd64"), + "explicit runtime architecture" + ); + Ok(value) +} +pub(super) fn handler() -> Result { + match env::var("RUSTACK_RUNTIME_HANDLER_IMAGE") { + Ok(value) => Ok(value), + Err(env::VarError::NotPresent) => Ok(format!("rustack-p0-handler:{}", architecture()?)), + Err(error) => Err(error.into()), + } +} +pub(super) fn socket() -> Result { + Ok(format!( + "unix://{}", + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")? + )) +} +impl Server { + pub(super) fn process_id(&self) -> Result { + self.child.id().context("owned server PID") + } + pub async fn spawn(profile: Profile) -> Result { + Self::spawn_with_socket(profile, None).await + } + pub async fn spawn_with_socket( + profile: Profile, + socket_override: Option<&Path>, + ) -> Result { + let root = private_root()?; + let reserved = TcpListener::bind("0.0.0.0:0").await?; + let port = reserved.local_addr()?.port(); + drop(reserved); + let config = root.path().join("config.yaml"); + let mut text = "budgets:\n startupSeconds: 180\n shutdownSeconds: 60\n \ + lambdaDockerIoSeconds: 60\n lambdaDockerStartSeconds: 60\n \ + lambdaDockerCleanupSeconds: 60\n lambdaDockerEnvironments: 2\n \ + lambdaDockerImagePins: 16\n lambdaDockerDeploymentJobs: 2\n" + .to_owned(); + if profile == Profile::Deadline { + text = text.replace("lambdaDockerIoSeconds: 60", "lambdaDockerIoSeconds: 5"); + } + if matches!( + profile, + Profile::ZipControlled + | Profile::Controlled + | Profile::Deadline + | Profile::Pull + | Profile::Negative + | Profile::EnvironmentPressure + ) { + text.push_str(" lambdaDockerMemoryBytes: 402653184\n"); + } + if profile == Profile::MemoryPressure { + text.push_str(" lambdaDockerMemoryBytes: 268435456\n"); + } + if matches!( + profile, + Profile::EnvironmentPressure + | Profile::Sdk + | Profile::SdkWrongCredentials + | Profile::Performance + ) { + text = text.replace("lambdaDockerEnvironments: 2", "lambdaDockerEnvironments: 1"); + } + if profile == Profile::Callback { + text.push_str( + " lambdaDockerMemoryBytes: 805306368\n workflowDispatches: 1\n \ + workflowCallbackTokens: 2\n workflowCallbackTombstones: 64\n \ + workflowCallbackCommands: 8\n workflowCallbackWorkers: 1\n", + ); + } + if profile == Profile::Iac { + text.push_str(" lambdaDockerMemoryBytes: 1610612736\n workflowDispatches: 2\n"); + } + fs::write(&config, text).await?; + if profile == Profile::Pull { + let auth = root.path().join("registry.yaml"); + fs::write(&auth, b"{}\n").await?; + fs::set_permissions(&auth, Permissions::from_mode(0o600)).await?; + } + if profile == Profile::Legacy { + let lease = DirectoryLease::acquire(root.path().join("data"), deadline()).await?; + drop(lease); + let file = root.path().join("data/.rustack-lambda-docker/owner.v1"); + let mut bytes = fs::read(&file).await?; + bytes + .get_mut(..8) + .context("owner magic")? + .copy_from_slice(b"RSDKOWN1"); + let digest = Sha256::digest(bytes.get(..24).context("owner payload")?); + bytes + .get_mut(24..) + .context("owner checksum")? + .copy_from_slice(&digest); + fs::write(&file, &bytes).await?; + fs::write(root.path().join("legacy.expected"), bytes).await?; + } + let sources = fixture_sources().await?; + fs::copy(env!("CARGO_BIN_EXE_rustack"), root.path().join("rustack")).await?; + let child = launch(root.path(), profile, port, socket_override).await?; + Ok(Self { + child, + root: Some(root), + endpoint: format!("http://127.0.0.1:{port}"), + profile, + sources, + socket_override: socket_override.map(Path::to_path_buf), + log_start: 0, + }) + } + pub async fn track_image(&mut self, source: &str) -> Result { + let reference: ImageReference = source.parse()?; + let mut wire = DockerWire::connect(socket()?.parse()?, deadline()).await?; + let result = wire.port().inspect_image(&reference, deadline()).await; + let stopped = wire.shutdown().await; + stopped?; + let id = result?.id.context("fixture source identity")?; + ensure!( + id.len() == 71 + && id.starts_with("sha256:") + && id + .get(7..) + .is_some_and(|value| value.bytes().all(|byte| byte.is_ascii_hexdigit())), + "invalid source content identity" + ); + self.sources.push((reference, id.clone())); + Ok(id) + } + pub async fn track_platform_image( + &mut self, + source: &str, + os: &str, + architecture: &str, + ) -> Result { + let id = self.track_image(source).await?; + let reference: ImageReference = id.parse()?; + let mut wire = DockerWire::connect(socket()?.parse()?, deadline()).await?; + let observed = wire.port().inspect_image(&reference, deadline()).await; + let stopped = wire.shutdown().await; + stopped?; + let observed = observed?; + ensure!( + observed.id.as_ref() == Some(&id) + && observed.os.as_deref() == Some(os) + && observed.architecture.as_deref() == Some(architecture) + && observed.size.is_some_and(|size| size > 0), + "negative fixture platform/identity precondition differs" + ); + Ok(id) + } + pub async fn observation_identity(&self) -> Result<(String, String)> { + let base = self.path()?.join("data/.rustack-lambda-docker"); + let magic = if self.profile.zip() { + b"RSDKOWN3" + } else { + b"RSDKOWN2" + }; + let owner = observation_record(base.join("owner.v1"), 56, magic).await?; + let id = uuid::Uuid::from_slice(owner.get(8..24).context("owner UUID")?)?; + ensure!(!id.is_nil(), "nil owner identity"); + let daemon = observation_record(base.join("daemon.v1"), 72, b"RSDKDMD1").await?; + ensure!( + daemon.get(8..24) == Some(id.as_bytes().as_slice()), + "daemon record owner differs" + ); + let engine = uuid::Uuid::from_slice(daemon.get(24..40).context("daemon UUID")?)?; + ensure!(!engine.is_nil(), "nil daemon identity"); + Ok((id.to_string(), engine.to_string())) + } + pub fn path(&self) -> Result<&Path> { + Ok(self.root.as_ref().context("owned evidence root")?.path()) + } + pub async fn ready(&mut self) -> Result { + let at = Instant::now() + Duration::from_secs(180); + loop { + ensure!( + self.child.try_wait()?.is_none(), + "ordinary binary exited before readiness" + ); + if let Ok((status, value)) = get(&self.endpoint, "/_rustack/capabilities").await { + if value["ready"] == true { + ensure!( + status == 200 && value["workers"]["docker"]["ready"] == true, + "false backend readiness" + ); + ensure!( + value["workers"]["docker"]["platform"] + == format!("linux/{}", architecture()?), + "wrong qualified platform" + ); + ensure!( + value["workers"]["docker"]["sdkGateway"] == self.profile.network(), + "wrong endpoint capability" + ); + ensure!( + value["workers"]["docker"]["zip"] == self.profile.zip(), + "false ZIP capability" + ); + return Ok(value); + } + } + ensure!(Instant::now() < at, "ordinary startup observer expired"); + sleep(Duration::from_millis(25)).await; + } + } + pub async fn failure(&mut self) -> Result<()> { + self.failure_with(None).await + } + pub async fn failure_with(&mut self, expected: Option<&str>) -> Result<()> { + let at = Instant::now() + Duration::from_secs(180); + loop { + if let Some(status) = self.child.try_wait()? { + ensure!(!status.success(), "negative startup exited successfully"); + let mut file = fs::File::open(self.path()?.join("server.log")).await?; + file.seek(SeekFrom::Start(self.log_start)).await?; + let mut bytes = Vec::new(); + file.take(32769).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 32768, "negative startup log cap"); + let text = std::str::from_utf8(&bytes)?; + ensure!( + expected.map_or_else( + || self.profile.matches_failure(text), + |expected| text.contains(expected) + ), + "startup failed at an unrelated prerequisite: {text}" + ); + return Ok(()); + } + if let Ok((_, value)) = get(&self.endpoint, "/_rustack/capabilities").await { + ensure!( + value["ready"] != true, + "negative startup opened ordinary ingress" + ); + } + ensure!(Instant::now() < at, "negative startup observer expired"); + sleep(Duration::from_millis(25)).await; + } + } + pub async fn stop(&mut self, success: bool) -> Result<()> { + let status = self.stop_status().await?; + ensure!( + status.success() == success, + "unexpected ordinary server exit status" + ); + Ok(()) + } + pub async fn reap(&mut self) -> Result<()> { + self.stop_status().await?; + Ok(()) + } + pub async fn stop_after_transport_loss(&mut self) -> Result<()> { + let status = self.stop_status().await?; + if !status.success() { + let file = fs::File::open(self.path()?.join("server.log")).await?; + let mut raw = Vec::new(); + file.take(32769).read_to_end(&mut raw).await?; + ensure!(raw.len() <= 32768, "transport-loss diagnostic cap"); + let text = String::from_utf8_lossy(&raw); + ensure!( + text.contains("Error: Docker not available") + || text.contains("Lambda quiesce did not confirm all invocation cleanup"), + "unrelated shutdown error after transport loss" + ); + } + Ok(()) + } + async fn stop_status(&mut self) -> Result { + if self.child.try_wait()?.is_none() { + let id = self.child.id().context("owned server PID")?; + kill(Pid::from_raw(id.try_into()?), Signal::SIGTERM)?; + } + let result = timeout(Duration::from_secs(70), self.child.wait()).await; + let status = if let Ok(result) = result { + result? + } else { + self.child.start_kill()?; + timeout(Duration::from_secs(5), self.child.wait()).await??; + anyhow::bail!("forced server reap; graceful cleanup unproven"); + }; + Ok(status) + } + pub async fn crash(&mut self) -> Result<()> { + ensure!( + self.child.try_wait()?.is_none(), + "crash target already exited" + ); + self.child.start_kill()?; + let status = timeout(Duration::from_secs(10), self.child.wait()).await??; + ensure!( + status.signal() == Some(Signal::SIGKILL as i32), + "crash did not join SIGKILL" + ); + Ok(()) + } + pub async fn restart(&mut self) -> Result<()> { + let port = self + .endpoint + .rsplit(':') + .next() + .context("owned endpoint port")? + .parse()?; + self.log_start = fs::metadata(self.path()?.join("server.log")).await?.len(); + self.child = launch( + self.path()?, + self.profile, + port, + self.socket_override.as_deref(), + ) + .await?; + Ok(()) + } + pub async fn audit(&self) -> Result<()> { + ensure!(self.child.id().is_none(), "audit before child reap"); + let path = self.path()?; + let lease = if self.profile.zip() { + DirectoryLease::acquire_code(path.join("data"), deadline()).await? + } else { + DirectoryLease::acquire(path.join("data"), deadline()).await? + }; + let mut wire = DockerWire::connect(socket()?.parse()?, deadline()).await?; + let result = async { + if self.profile == Profile::Legacy { + ensure!( + fs::read(path.join("data/.rustack-lambda-docker/owner.v1")).await? + == fs::read(path.join("legacy.expected")).await?, + "legacy identity was rewritten" + ); + ensure!( + !fs::try_exists(path.join("data/.rustack-lambda-docker/daemon.v1")).await?, + "legacy startup reached daemon binding" + ); + } else { + ensure!( + fs::try_exists(path.join("data/.rustack-lambda-docker/daemon.v1")).await?, + "fixture never reached production daemon binding" + ); + let port = wire.port().bind_directory(&lease, deadline()).await?; + let recovery = reconcile_namespace(&lease, &port, deadline()).await?; + ensure!( + recovery.containers.settled == 0 + && recovery.containers.remaining == 0 + && recovery.pins.settled == 0 + && recovery.pins.remaining == 0 + && recovery.code.settled == 0 + && recovery.code.remaining == 0, + "ordinary shutdown required post-exit recovery" + ); + } + for (reference, id) in &self.sources { + ensure!( + wire.port() + .inspect_image(reference, deadline()) + .await? + .id + .as_ref() + == Some(id), + "operator source changed" + ); + } + Ok::<(), anyhow::Error>(()) + } + .await; + let stopped = wire.shutdown().await; + result?; + stopped?; + Ok(()) + } + pub async fn retain_on_error(&mut self) { + if let Ok(path) = self.path() { + if let Ok(file) = fs::File::open(path.join("server.log")).await { + let mut bytes = Vec::new(); + let _ = file.take(32768).read_to_end(&mut bytes).await; + eprintln!( + "ordinary server diagnostics: {}", + String::from_utf8_lossy(&bytes) + ); + } + } + if let Some(root) = self.root.take() { + eprintln!("retained startup evidence: {}", root.keep().display()); + } + } +} +async fn fixture_sources() -> Result> { + let mut wire = DockerWire::connect(socket()?.parse()?, deadline()).await?; + let result = async { + let mut sources = Vec::new(); + for reference in [image()?, handler()?] { + let reference: ImageReference = reference.parse()?; + let id = wire + .port() + .inspect_image(&reference, deadline()) + .await? + .id + .context("source image identity")?; + sources.push((reference, id)); + } + Ok::<_, anyhow::Error>(sources) + } + .await; + let stopped = wire.shutdown().await; + stopped?; + result +} +async fn observation_record(path: PathBuf, length: usize, magic: &[u8; 8]) -> Result> { + ensure!((32..=72).contains(&length), "invalid fixture record cap"); + let file = fs::File::open(path).await?; + let mut raw = Vec::new(); + file.take(u64::try_from(length)? + 1) + .read_to_end(&mut raw) + .await?; + ensure!( + raw.len() == length && raw.get(..8) == Some(magic.as_slice()), + "unexpected runtime identity record" + ); + let payload_end = length.checked_sub(32).context("identity checksum offset")?; + let payload = raw.get(..payload_end).context("identity payload")?; + ensure!( + raw.get(payload_end..) == Some(Sha256::digest(payload).as_slice()), + "identity record checksum" + ); + Ok(raw) +} +fn configure_sdk(command: &mut Command, profile: Profile) { + if profile.sdk() { + let wrong = profile == Profile::SdkWrongCredentials; + command + .env("S3_SKIP_SIGNATURE_VALIDATION", "false") + .env("DYNAMODB_SKIP_SIGNATURE_VALIDATION", "false") + .env("LAMBDA_DOCKER_SDK_ACCESS_KEY_ID", "local-key") + .env( + "LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY", + if wrong { + "sdk-wrong-secret" + } else { + "local-secret" + }, + ) + .env("AWS_ACCESS_KEY_ID", "local-key") + .env("AWS_SECRET_ACCESS_KEY", "local-secret"); + } +} + +async fn launch( + root: &Path, + profile: Profile, + port: u16, + socket_override: Option<&Path>, +) -> Result { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(root.join("server.log")) + .await? + .into_std() + .await; + let mut command = Command::new(root.join("rustack")); + command + .env_clear() + .env("SERVICES", profile.services()) + .env("GATEWAY_LISTEN", format!("0.0.0.0:{port}")) + .env("RUSTACK_CONFIG", root.join("config.yaml")) + .env("DATA_DIR", root.join("data")) + .env("LAMBDA_EXECUTOR", "docker") + .env( + "LAMBDA_DOCKER_SOCKET", + match socket_override { + Some(path) => format!("unix://{}", path.display()), + None => socket()?, + }, + ) + .env( + "LAMBDA_DOCKER_RELAY_IMAGE", + if profile == Profile::MissingRelay { + format!("missing-{}:never", uuid::Uuid::new_v4().simple()) + } else { + image()? + }, + ) + .env( + "LAMBDA_DOCKER_CROSS_ARCHITECTURE", + if architecture()? == "amd64" { + "true" + } else { + "false" + }, + ) + .env("LAMBDA_INIT_TIMEOUT_SECS", profile.init_seconds()) + .env("LAMBDA_SKIP_SIGNATURE_VALIDATION", "false") + .env("STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION", "false") + .env("ACCESS_KEY", "local-key") + .env("SECRET_KEY", "local-secret") + .stdin(Stdio::null()) + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)) + .kill_on_drop(true); + configure_sdk(&mut command, profile); + profile.configure_diagnostics(&mut command); + if profile.zip() { + command.env( + "LAMBDA_DOCKER_ZIP_IMAGE", + env::var("RUSTACK_ZIP_BASE_IMAGE")?, + ); + } + if profile == Profile::Pull { + command + .env("LAMBDA_DOCKER_PULL_POLICY", "ifMissing") + .env("LAMBDA_DOCKER_REGISTRIES", "public.ecr.aws:443") + .env( + "LAMBDA_DOCKER_REGISTRY_AUTH_FILE", + fs::canonicalize(root.join("registry.yaml")).await?, + ); + } + if profile.network() { + command.env( + "LAMBDA_DOCKER_NETWORK", + env::var("RUSTACK_RUNTIME_NETWORK")?, + ); + let host = env::var("RUSTACK_RUNTIME_GATEWAY_HOST")?.parse::()?; + let target = if profile == Profile::WrongEndpoint { + let closed = TcpListener::bind("0.0.0.0:0").await?; + closed.local_addr()?.port() + } else { + port + }; + command.env( + "LAMBDA_DOCKER_GATEWAY_ENDPOINT", + format!("http://{}", std::net::SocketAddr::new(host, target)), + ); + } else { + command.env("LAMBDA_DOCKER_NETWORK", "none"); + } + if matches!( + profile, + Profile::Archive | Profile::ZipArchive | Profile::ZipArchiveNoSource | Profile::Callback + ) { + command + .arg("--snapshot") + .arg("images") + .env("RUSTACK_SNAPSHOT_DIR", root.join("snapshots")); + } + Ok(command.spawn()?) +} +pub(super) async fn get(endpoint: &str, path: &str) -> Result<(u16, Value)> { + timeout(Duration::from_secs(2), async { + let mut stream = + TcpStream::connect(endpoint.strip_prefix("http://").context("local endpoint")?).await?; + stream + .write_all( + format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .as_bytes(), + ) + .await?; + let mut bytes = Vec::new(); + stream.take(65537).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 65536, "diagnostic response cap"); + let text = std::str::from_utf8(&bytes)?; + let (head, body) = text.split_once("\r\n\r\n").context("HTTP framing")?; + let status = head + .split_whitespace() + .nth(1) + .context("HTTP status")? + .parse()?; + Ok((status, serde_json::from_str(body)?)) + }) + .await + .context("diagnostic observer expired")? +} diff --git a/apps/rustack/tests/docker_startup/provided.rs b/apps/rustack/tests/docker_startup/provided.rs new file mode 100644 index 00000000..84029638 --- /dev/null +++ b/apps/rustack/tests/docker_startup/provided.rs @@ -0,0 +1,142 @@ +//! Ordinary signed provided runtime operations; original ZIP files are shared with Native. + +use std::{env, ffi::OsString, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::storage::compute_sha256; +use serde_json::{Value, json}; +use tokio::{ + fs, + io::AsyncReadExt, + time::{Instant, timeout}, +}; + +use super::{ + faults::check_logs, + process::{self, Profile, Server}, + runner::{Runner, interpreter}, +}; + +pub(super) async fn exercise(phase: &str) -> Result<()> { + let mut server = Server::spawn(if matches!(phase, "initializing" | "busy") { + Profile::ZipControlled + } else { + Profile::Zip + }) + .await?; + let result = timeout(Duration::from_secs(360), run(&mut server, phase)) + .await + .context("ordinary provided runtime observer expired") + .and_then(|result| result); + let stopped = server.stop(true).await; + let audited = server.audit().await; + let logs = check_logs(&server).await; + if result.is_err() || stopped.is_err() || audited.is_err() || logs.is_err() { + server.retain_on_error().await; + } + result?; + stopped?; + audited?; + logs +} + +async fn run(server: &mut Server, phase: &str) -> Result<()> { + server.ready().await?; + server + .track_image(&env::var("RUSTACK_ZIP_BASE_IMAGE")?) + .await?; + let (owner, engine) = server.observation_identity().await?; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let arguments = [ + root.join("tests/workflows/provided_zip_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + env::var_os("RUSTACK_PROVIDED_ZIP_A").context("original provided ZIP A")?, + env::var_os("RUSTACK_PROVIDED_ZIP_B").context("original provided ZIP B")?, + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + OsString::from(phase), + env::var_os("RUSTACK_RUNTIME_GATEWAY_HOST").context("explicit hold collector host")?, + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit actual metadata socket")?, + OsString::from(owner), + OsString::from(engine), + ]; + let mut child = Runner::spawn(&interpreter()?, &arguments)?; + let entered = child.await_pid().await; + let report = child + .wait(if entered.is_ok() { + Instant::now() + Duration::from_secs(300) + } else { + Instant::now() + }) + .await?; + let pid = entered?; + ensure!( + report.pid == pid && !report.forced && report.status.success(), + "ordinary provided SDK failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + let observed: Value = serde_json::from_slice(&report.stdout)?; + let expected = match phase { + "deployments" => json!([ + "cold-warm-readonly", + "transactional-rejection", + "immutable-A-B-alias", + "ASL-Lambda-computation" + ]), + "faults" => json!([ + "init-error", + "handler-exit", + "handler-timeout", + "FunctionError-recovery" + ]), + "initializing" | "busy" => json!([ + format!("{phase}-causal-disconnect"), + "physical-retirement", + "cold-recovery" + ]), + _ => anyhow::bail!("unknown provided resource case"), + }; + ensure!( + observed["schemaVersion"] == 1 && observed["cases"] == expected, + "ZIP semantic inventory differs" + ); + verify_digests(&observed).await +} + +pub(super) async fn verify_digests(observed: &Value) -> Result<()> { + let digests = observed["zipSha256"] + .as_array() + .context("ZIP identity report")?; + ensure!( + digests.len() == 2 && digests.first() != digests.get(1), + "ZIP revision identity differs" + ); + for (index, key) in ["RUSTACK_PROVIDED_ZIP_A", "RUSTACK_PROVIDED_ZIP_B"] + .into_iter() + .enumerate() + { + let path = env::var(key)?; + let metadata = fs::symlink_metadata(&path).await?; + ensure!( + metadata.is_file() && metadata.len() <= 50 * 1024 * 1024, + "bounded original ZIP file" + ); + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(50 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= 50 * 1024 * 1024, "original ZIP byte cap"); + let hash = compute_sha256(&bytes); + ensure!( + digests.get(index).and_then(Value::as_str) == Some(hash.as_str()), + "SDK original ZIP digest changed" + ); + eprintln!("provided original ZIP {index} digest: {hash}"); + } + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/provided_archive.rs b/apps/rustack/tests/docker_startup/provided_archive.rs new file mode 100644 index 00000000..46712b7f --- /dev/null +++ b/apps/rustack/tests/docker_startup/provided_archive.rs @@ -0,0 +1,211 @@ +//! Ordinary mixed archives; both server handles outlive every observer outcome. + +use std::{env, ffi::OsString, path::PathBuf, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use tokio::{ + fs, + time::{Instant, timeout}, +}; + +use super::{ + archive, copy_tree, fingerprint, + process::{self, Profile, Server}, + provided::verify_digests, + runner::{Runner, interpreter}, +}; + +fn guard(server: &mut Server, retained: bool) -> Result<()> { + server + .root + .as_mut() + .context("provided archive evidence guard")? + .disable_cleanup(retained); + Ok(()) +} +pub(super) async fn exercise(no_source: bool) -> Result<()> { + let mut source = Server::spawn(Profile::ZipArchive).await?; + guard(&mut source, true)?; + eprintln!("provided archive evidence: {}", source.path()?.display()); + let mut consumer = None; + let observed = timeout( + Duration::from_mins(12), + run(&mut source, &mut consumer, no_source), + ) + .await + .context("provided archive observer expired") + .and_then(|result| result); + let source_stop = source.stop(no_source).await; + let source_audit = source.audit().await; + let consumer_cleanup = if let Some(server) = &mut consumer { + let stopped = server.stop(false).await; + let audit = server.audit().await; + stopped.and(audit) + } else { + Ok(()) + }; + if observed.is_err() + || source_stop.is_err() + || source_audit.is_err() + || consumer_cleanup.is_err() + { + source.retain_on_error().await; + if let Some(server) = &mut consumer { + server.retain_on_error().await; + } + } + observed?; + source_stop?; + source_audit?; + consumer_cleanup?; + guard(&mut source, false)?; + if let Some(server) = &mut consumer { + guard(server, false)?; + } + Ok(()) +} +async fn run(source: &mut Server, consumer: &mut Option, no_source: bool) -> Result<()> { + source.ready().await?; + source + .track_image(&env::var("RUSTACK_ZIP_BASE_IMAGE")?) + .await?; + source.track_image(&process::handler()?).await?; + let saved = sdk(source, "prepare").await?; + let checkpoint = source.path()?.join("zip-checkpoint.json"); + let bytes = serde_json::to_vec(&saved)?; + ensure!(bytes.len() <= 65536, "bounded archive checkpoint"); + fs::write(&checkpoint, bytes).await?; + source.stop(true).await?; + source.audit().await?; + let snapshots = source.path()?.join("snapshots"); + let current = snapshots.join("images"); + if no_source { + *consumer = Some(Server::spawn(Profile::ZipArchiveNoSource).await?); + let target = consumer.as_mut().context("image-only archive consumer")?; + guard(target, true)?; + eprintln!( + "provided archive image-only evidence: {}", + target.path()?.display() + ); + target.ready().await?; + ensure!( + process::get(&target.endpoint, "/_rustack/capabilities") + .await? + .1["workers"]["docker"]["zip"] + == false, + "negative archive source was installed" + ); + target.stop(true).await?; + target.audit().await?; + let target_snapshots = target.path()?.join("snapshots"); + let target_current = target_snapshots.join("images"); + fs::rename(&target_current, target_snapshots.join(".baseline-empty")).await?; + copy_tree(¤t, &target_current).await?; + copy_tree(¤t, &target_snapshots.join(".images.previous")).await?; + let before = fingerprint(&target_snapshots).await?; + target.restart().await?; + target.failure().await?; + ensure!( + fingerprint(&target_snapshots).await? == before, + "unconfigured ZIP import changed complete archive generations" + ); + } else { + let original = serde_json::to_value(archive::read(source.path()?).await?)?; + let previous = snapshots.join(".images.previous"); + copy_tree(¤t, &previous).await?; + source.restart().await?; + source.ready().await?; + ensure!( + !fs::try_exists(&previous).await?, + "successful ZIP import kept previous generation" + ); + let checked = sdk(source, "verify").await?; + ensure!( + checked["checkpoint"] == saved["checkpoint"], + "SDK restored checkpoint changed" + ); + source.stop(true).await?; + source.audit().await?; + ensure!( + serde_json::to_value(archive::read(source.path()?).await?)? == original, + "restored ZIP/Image metadata/resource/revision identity changed" + ); + copy_tree(¤t, &previous).await?; + archive::corrupt_zip(source.path()?).await?; + let before = fingerprint(&snapshots).await?; + source.restart().await?; + source.failure().await?; + ensure!( + fingerprint(&snapshots).await? == before, + "invalid ZIP import changed complete archive generations" + ); + } + Ok(()) +} +async fn sdk(server: &Server, phase: &str) -> Result { + let (owner, engine) = server.observation_identity().await?; + let repository = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let arguments = [ + repository + .join("tests/workflows/provided_zip_archive_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + env::var_os("RUSTACK_PROVIDED_ZIP_A").context("original archive ZIP A")?, + env::var_os("RUSTACK_PROVIDED_ZIP_B").context("original archive ZIP B")?, + OsString::from(if process::architecture()? == "arm64" { + "arm64" + } else { + "x86_64" + }), + OsString::from(process::handler()?), + OsString::from(phase), + server.path()?.join("zip-checkpoint.json").into_os_string(), + env::var_os("RUSTACK_RUNTIME_DOCKER_SOCKET").context("actual archive daemon socket")?, + OsString::from(owner), + OsString::from(engine), + ]; + let mut child = Runner::spawn(&interpreter()?, &arguments)?; + let entered = child.await_pid().await; + let report = child + .wait(if entered.is_ok() { + Instant::now() + Duration::from_secs(300) + } else { + Instant::now() + }) + .await?; + let pid = entered?; + ensure!( + report.pid == pid && !report.forced && report.status.success(), + "provided archive SDK failed: {}", + String::from_utf8_lossy(&report.stderr) + ); + ensure!( + report.stdout.len() <= 65536, + "archive checkpoint output limit" + ); + let observed: Value = serde_json::from_slice(&report.stdout)?; + let expected = if phase == "prepare" { + serde_json::json!([ + "original-ZIP-revisions", + "mixed-Image", + "terminal-history-checkpoint" + ]) + } else { + ensure!(phase == "verify", "unknown archive SDK phase"); + serde_json::json!([ + "original-ZIP-revisions", + "mixed-Image", + "terminal-history-no-replay", + "cold-warm-after-restart" + ]) + }; + ensure!( + observed["schemaVersion"] == 1 + && observed["phase"] == phase + && observed["cases"] == expected, + "provided archive semantic inventory differs" + ); + verify_digests(&observed).await?; + Ok(observed) +} diff --git a/apps/rustack/tests/docker_startup/recovery_gate.rs b/apps/rustack/tests/docker_startup/recovery_gate.rs new file mode 100644 index 00000000..41a154ed --- /dev/null +++ b/apps/rustack/tests/docker_startup/recovery_gate.rs @@ -0,0 +1,157 @@ +//! Retained socket proxy with a causal startup barrier for one crashed owner. + +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use tempfile::TempDir; +use tokio::{ + io::copy_bidirectional, + net::{UnixListener, UnixStream}, + sync::watch, + task::{JoinHandle, JoinSet}, + time::timeout, +}; + +#[derive(Debug)] +pub(super) struct Gate { + directory: Option, + pub socket: PathBuf, + target: PathBuf, + held: watch::Sender, + entered: watch::Receiver, + stop: watch::Sender, + root: Option>>, +} +impl Gate { + pub fn start(target: &Path) -> Result { + let mut directory = tempfile::tempdir()?; + directory.disable_cleanup(true); + let socket = directory.path().join("recovery.sock"); + let listener = UnixListener::bind(&socket)?; + let (held, barrier) = watch::channel(false); + let (observed, entered) = watch::channel(false); + let (stop, shutdown) = watch::channel(false); + let target = target.to_owned(); + let root = tokio::spawn(serve(listener, target.clone(), barrier, observed, shutdown)); + Ok(Self { + directory: Some(directory), + socket, + target, + held, + entered, + stop, + root: Some(root), + }) + } + pub fn arm(&mut self) { + self.held.send_replace(true); + } + pub async fn blocked(&mut self) -> Result<()> { + timeout(Duration::from_secs(20), async { + while !*self.entered.borrow() { + self.entered.changed().await?; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("recovery did not reach held socket")??; + ensure!(*self.held.borrow(), "recovery barrier released early"); + Ok(()) + } + pub fn release(&self) { + self.held.send_replace(false); + } + pub async fn disconnect(&mut self) -> Result<()> { + self.release(); + self.stop.send_replace(true); + if let Some(root) = &mut self.root { + timeout(Duration::from_secs(10), root) + .await + .context("proxy root retained")???; + self.root = None; + } + Ok(()) + } + pub async fn resume(&mut self) -> Result<()> { + ensure!(self.root.is_none(), "proxy resumed before join"); + tokio::fs::remove_file(&self.socket).await?; + let listener = UnixListener::bind(&self.socket)?; + let (held, barrier) = watch::channel(false); + let (observed, entered) = watch::channel(false); + let (stop, shutdown) = watch::channel(false); + self.held = held; + self.entered = entered; + self.stop = stop; + self.root = Some(tokio::spawn(serve( + listener, + self.target.clone(), + barrier, + observed, + shutdown, + ))); + Ok(()) + } + pub async fn close(&mut self) -> Result<()> { + self.disconnect().await?; + if let Some(mut directory) = self.directory.take() { + directory.disable_cleanup(false); + } + Ok(()) + } +} +impl Drop for Gate { + fn drop(&mut self) { + self.held.send_replace(false); + self.stop.send_replace(true); + // The retained actor is intentionally detached on observer loss; its + // shutdown channel joins every connection. Evidence stays retained. + if let Some(root) = self.root.take() { + drop(root); + } + } +} +async fn serve( + listener: UnixListener, + target: PathBuf, + mut held: watch::Receiver, + entered: watch::Sender, + mut stop: watch::Receiver, +) -> Result<()> { + let mut children = JoinSet::new(); + loop { + tokio::select! { + biased; + _ = stop.changed() => break, + Some(result) = children.join_next() => { result.context("proxy connection panicked")?; }, + accepted = listener.accept(), if children.len() < 16 => { + let (mut inbound, _) = accepted?; + if *held.borrow() { + entered.send_replace(true); + loop { + if !*held.borrow() {break;} + tokio::select! { _ = stop.changed() => break, _ = held.changed() => {} } + } + if *stop.borrow() {break;} + } + let target = target.clone(); + let mut connection_stop = stop.clone(); + children.spawn(async move { + tokio::select! { + _ = connection_stop.changed() => {}, + _ = timeout(Duration::from_secs(120), async { + let mut outbound = UnixStream::connect(target).await?; + copy_bidirectional(&mut inbound, &mut outbound).await + }) => {}, + } + }); + } + } + } + while let Some(result) = children.join_next().await { + result.context("proxy connection panicked")?; + } + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/sdk.rs b/apps/rustack/tests/docker_startup/sdk.rs new file mode 100644 index 00000000..dc033675 --- /dev/null +++ b/apps/rustack/tests/docker_startup/sdk.rs @@ -0,0 +1,350 @@ +//! Signed ordinary SDK calls through the actual production startup consumer. + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use aws_credential_types::Credentials; +use aws_sdk_lambda::{ + Client, + config::{BehaviorVersion, Region, retry::RetryConfig, timeout::TimeoutConfig}, + primitives::Blob, + types::{Architecture, FunctionCode, ImageConfig, PackageType, Runtime, State}, +}; +use aws_sdk_sfn::{Client as States, error::ProvideErrorMetadata}; +use serde_json::{Value, json}; +use tokio::time::{Instant, sleep}; +use uuid::Uuid; + +fn timeouts() -> TimeoutConfig { + TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(2)) + .read_timeout(Duration::from_secs(60)) + .operation_timeout(Duration::from_secs(60)) + .build() +} +fn credentials() -> Credentials { + Credentials::new("local-key", "local-secret", None, None, "docker-fixture") +} +fn lambda(endpoint: &str) -> Client { + Client::from_conf( + aws_sdk_lambda::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(endpoint) + .credentials_provider(credentials()) + .retry_config(RetryConfig::disabled()) + .timeout_config(timeouts()) + .build(), + ) +} +async fn active(client: &Client, name: &str, expected: &str) -> Result<()> { + let at = Instant::now() + Duration::from_secs(120); + loop { + let config = client + .get_function_configuration() + .function_name(name) + .send() + .await?; + ensure!(Instant::now() < at, "deployment observer expired"); + let state = config.state().map(State::as_str); + if state == Some(expected) { + return Ok(()); + } + ensure!( + state != Some("Failed"), + "deployment did not reach expected state" + ); + sleep(Duration::from_millis(25)).await; + } +} +async fn invoke(client: &Client, name: &str, event: Value) -> Result<(Value, Option)> { + let result = client + .invoke() + .function_name(name) + .payload(Blob::new(serde_json::to_vec(&event)?)) + .send() + .await?; + ensure!(result.status_code() == 200, "Invoke status"); + let value = serde_json::from_slice(result.payload().context("Invoke payload")?.as_ref())?; + Ok((value, result.function_error().map(str::to_owned))) +} +pub(super) async fn exercise( + endpoint: &str, + image: &str, + architecture: &str, + workflow: bool, + preserve: bool, +) -> Result<()> { + let client = lambda(endpoint); + let name = format!("DockerStartup{}", Uuid::new_v4().simple()); + let selected = if architecture == "arm64" { + Architecture::Arm64 + } else { + Architecture::X8664 + }; + let created = client + .create_function() + .function_name(&name) + .role("arn:aws:iam::000000000000:role/Docker") + .package_type(PackageType::Image) + .architectures(selected.clone()) + .memory_size(128) + .timeout(30) + .code(FunctionCode::builder().image_uri(image).build()) + .send() + .await?; + let arn = created.function_arn().context("Image ARN")?.to_owned(); + active(&client, &name, "Active").await?; + for (value, counter) in [(35, 1), (0, 2)] { + let (output, error) = invoke(&client, &name, json!({"value":value})).await?; + ensure!( + error.is_none() + && output["computed"] == value + 7 + && output["counter"] == counter + && output["uid"] == 1000, + "real computed/warm/user evidence differs" + ); + ensure!( + output["requestId"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "missing RIC request identity" + ); + } + let version = client.publish_version().function_name(&name).send().await?; + let version = version.version().context("published version")?; + client + .create_alias() + .function_name(&name) + .name("live") + .function_version(version) + .send() + .await?; + let alias = client + .invoke() + .function_name(&name) + .qualifier("live") + .payload(Blob::new(br#"{"value":1}"#.as_slice())) + .send() + .await?; + ensure!( + alias.executed_version() == Some(version) && alias.function_error().is_none(), + "published alias did not execute captured image" + ); + let (_, failure) = invoke(&client, &name, json!({"fail":true,"value":1})).await?; + ensure!( + failure.as_deref() == Some("Unhandled"), + "FunctionError was not distinct from HTTP success" + ); + let (wrong_name, stalled) = failure_cases(&client, image, selected, &name).await?; + if workflow { + workflow_image(endpoint, &arn).await?; + } + if !preserve { + client + .delete_function() + .function_name(&stalled) + .send() + .await?; + client + .delete_function() + .function_name(&wrong_name) + .send() + .await?; + client.delete_function().function_name(&name).send().await?; + } + Ok(()) +} +async fn failure_cases( + client: &Client, + image: &str, + selected: Architecture, + name: &str, +) -> Result<(String, String)> { + let zip = client + .create_function() + .function_name(format!("{name}Zip")) + .role("arn:aws:iam::000000000000:role/Docker") + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .code( + FunctionCode::builder() + .zip_file(Blob::new(b"unextracted".as_slice())) + .build(), + ) + .send() + .await + .expect_err("Docker ZIP must reject before extraction"); + ensure!( + zip.as_service_error().and_then(|error| error.code()) + == Some("InvalidParameterValueException"), + "wrong Docker ZIP error" + ); + let wrong_name = format!("{name}OtherArch"); + client + .create_function() + .function_name(&wrong_name) + .role("arn:aws:iam::000000000000:role/Docker") + .package_type(PackageType::Image) + .architectures(if selected == Architecture::Arm64 { + Architecture::X8664 + } else { + Architecture::Arm64 + }) + .code(FunctionCode::builder().image_uri(image).build()) + .send() + .await?; + active(client, &wrong_name, "Failed").await?; + let stalled = format!("{name}NoRic"); + client + .create_function() + .function_name(&stalled) + .role("arn:aws:iam::000000000000:role/Docker") + .package_type(PackageType::Image) + .architectures(selected) + .code(FunctionCode::builder().image_uri(image).build()) + .image_config( + ImageConfig::builder() + .entry_point("/var/lang/bin/python3") + .entry_point("-c") + .entry_point("import time;time.sleep(60)") + .set_command(Some(Vec::new())) + .build(), + ) + .send() + .await?; + active(client, &stalled, "Active").await?; + let started = Instant::now(); + let result = client + .invoke() + .function_name(&stalled) + .payload(Blob::new(b"{}".as_slice())) + .send() + .await; + let error = result.expect_err("missing RIC must fail invocation"); + ensure!( + started.elapsed() < Duration::from_secs(45) + && error + .as_service_error() + .and_then(|error| error.message()) + .is_some_and(|message| message.contains("bootstrap initialization deadline")), + "independent bootstrap timeout was ignored or misclassified" + ); + Ok((wrong_name, stalled)) +} +async fn workflow_image(endpoint: &str, function: &str) -> Result<()> { + let client = States::from_conf( + aws_sdk_sfn::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(endpoint) + .credentials_provider(credentials()) + .retry_config(RetryConfig::disabled()) + .timeout_config(timeouts()) + .build(), + ); + let machine=client.create_state_machine().name(format!("DockerWorkflow{}",Uuid::new_v4().simple())).role_arn("arn:aws:iam::000000000000:role/Docker") + .definition(json!({"StartAt":"Compute","States":{"Compute":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":function,"Payload.$":"$"},"OutputPath":"$.Payload","End":true}}}).to_string()).send().await?; + let machine = machine.state_machine_arn().to_owned(); + let execution = client + .start_execution() + .state_machine_arn(&machine) + .input(r#"{"value":35}"#) + .send() + .await?; + let execution = execution.execution_arn(); + let at = Instant::now() + Duration::from_secs(120); + loop { + let result = client + .describe_execution() + .execution_arn(execution) + .send() + .await?; + if result.status().as_str() != "RUNNING" { + ensure!( + result.status().as_str() == "SUCCEEDED", + "image workflow failed: {:?}", + result.error() + ); + let output: Value = serde_json::from_str(result.output().context("workflow output")?)?; + ensure!( + output["computed"] == 42, + "ASL did not compute through actual image" + ); + break; + } + ensure!(Instant::now() < at, "image workflow observer expired"); + sleep(Duration::from_millis(25)).await; + } + client + .delete_state_machine() + .state_machine_arn(machine) + .send() + .await?; + Ok(()) +} + +#[derive(Debug, Clone, Copy)] +enum ComputeFixture { + CustomRic, + OfficialNode, +} + +pub(super) async fn custom_compute(endpoint: &str, image: &str, architecture: &str) -> Result<()> { + compute_image(endpoint, image, architecture, ComputeFixture::CustomRic).await +} + +pub(super) async fn node_compute(endpoint: &str, image: &str, architecture: &str) -> Result<()> { + compute_image(endpoint, image, architecture, ComputeFixture::OfficialNode).await +} + +async fn compute_image( + endpoint: &str, + image: &str, + architecture: &str, + fixture: ComputeFixture, +) -> Result<()> { + let client = lambda(endpoint); + let name = format!("RustImage{}", Uuid::new_v4().simple()); + let (memory, value) = match fixture { + ComputeFixture::CustomRic => (128, 35), + ComputeFixture::OfficialNode => (256, 21), + }; + client + .create_function() + .function_name(&name) + .role("arn:aws:iam::000000000000:role/Docker") + .package_type(PackageType::Image) + .architectures(if architecture == "arm64" { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .memory_size(memory) + .timeout(30) + .code(FunctionCode::builder().image_uri(image).build()) + .send() + .await?; + active(&client, &name, "Active").await?; + let (output, error) = invoke(&client, &name, json!({"value":value})).await?; + let identity_matches = match fixture { + ComputeFixture::CustomRic => output["imageRevision"] == "A", + ComputeFixture::OfficialNode => { + output["runtime"] == "node24" && output["handler"] == "handler" + } + }; + ensure!( + error.is_none() && output["computed"] == 42 && output["counter"] == 1 && identity_matches, + "Rust SDK image did not compute with the expected runtime" + ); + client.delete_function().function_name(&name).send().await?; + Ok(()) +} + +#[path = "sdk/archive.rs"] +mod archive; +pub(super) use archive::{checkpoint as archive_checkpoint, verify as verify_archive}; + +#[path = "sdk/recovery.rs"] +mod recovery; +pub(super) use recovery::{Witness, containers as owned_containers}; diff --git a/apps/rustack/tests/docker_startup/sdk/archive.rs b/apps/rustack/tests/docker_startup/sdk/archive.rs new file mode 100644 index 00000000..00bf28a2 --- /dev/null +++ b/apps/rustack/tests/docker_startup/sdk/archive.rs @@ -0,0 +1,186 @@ +//! Actual restored definitions, immutable aliases, cold computation and old history. + +use super::*; + +#[derive(Debug)] +pub(crate) struct ArchiveCheckpoint { + function: String, + hash: String, + revision: String, + old_request: String, + machine: String, + execution: String, + history: String, + output: String, +} +fn states(endpoint: &str) -> States { + States::from_conf( + aws_sdk_sfn::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(endpoint) + .credentials_provider(credentials()) + .retry_config(RetryConfig::disabled()) + .timeout_config(timeouts()) + .build(), + ) +} +async fn terminal(client: &States, execution: &str) -> Result { + let at = Instant::now() + Duration::from_secs(120); + loop { + let result = client + .describe_execution() + .execution_arn(execution) + .send() + .await?; + if result.status().as_str() != "RUNNING" { + ensure!( + result.status().as_str() == "SUCCEEDED", + "saved image workflow failed" + ); + let output = result.output().context("image workflow output")?.to_owned(); + let body: Value = serde_json::from_str(&output)?; + ensure!( + body["computed"] == 49, + "restored image task did not compute" + ); + return Ok(output); + } + ensure!( + Instant::now() < at, + "image archive workflow observer expired" + ); + sleep(Duration::from_millis(25)).await; + } +} +async fn history(client: &States, execution: &str) -> Result { + let result = client + .get_execution_history() + .execution_arn(execution) + .send() + .await?; + ensure!( + (5..=32).contains(&result.events().len()) && result.next_token().is_none(), + "image archive history bounds" + ); + let value = format!("{:?}", result.events()); + ensure!(value.len() <= 65_536, "image archive history byte cap"); + Ok(value) +} +pub(crate) async fn checkpoint(endpoint: &str) -> Result { + let client = lambda(endpoint); + let listed = client.list_functions().send().await?; + ensure!( + listed.functions().len() <= 8 && listed.next_marker().is_none(), + "archive fixture function bounds" + ); + let function = listed + .functions() + .iter() + .filter_map(|function| function.function_name()) + .find(|name| { + name.starts_with("DockerStartup") + && !name.ends_with("OtherArch") + && !name.ends_with("NoRic") + }) + .context("actual primary saved image")? + .to_owned(); + let config = client + .get_function_configuration() + .function_name(&function) + .send() + .await?; + let hash = config.code_sha256().context("saved image hash")?.to_owned(); + let revision = config.revision_id().context("saved revision")?.to_owned(); + let arn = config.function_arn().context("saved function ARN")?; + let workflows = states(endpoint); + let machine = workflows.create_state_machine().name(format!("ArchiveImage{}", Uuid::new_v4().simple())) + .role_arn("arn:aws:iam::000000000000:role/Docker") + .definition(json!({"StartAt":"Compute","States":{"Compute":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":format!("{arn}:live"),"Payload":{"value":42}},"OutputPath":"$.Payload","End":true}}}).to_string()) + .send().await?.state_machine_arn().to_owned(); + let execution = workflows + .start_execution() + .state_machine_arn(&machine) + .send() + .await? + .execution_arn() + .to_owned(); + let output = terminal(&workflows, &execution).await?; + let history = history(&workflows, &execution).await?; + let (actual, error) = invoke(&client, &function, json!({"value":3})).await?; + ensure!( + error.is_none() && actual["computed"] == 10, + "pre-save computation" + ); + let old_request = actual["requestId"] + .as_str() + .context("pre-save request")? + .to_owned(); + Ok(ArchiveCheckpoint { + function, + hash, + revision, + old_request, + machine, + execution, + history, + output, + }) +} +pub(crate) async fn verify(endpoint: &str, saved: &ArchiveCheckpoint) -> Result<()> { + let client = lambda(endpoint); + let config = client + .get_function_configuration() + .function_name(&saved.function) + .send() + .await?; + ensure!( + config.code_sha256() == Some(saved.hash.as_str()) + && config.revision_id() == Some(saved.revision.as_str()), + "restored image identity/configuration changed" + ); + let workflows = states(endpoint); + ensure!( + history(&workflows, &saved.execution).await? == saved.history, + "restored history changed or replayed" + ); + ensure!( + terminal(&workflows, &saved.execution).await? == saved.output, + "saved terminal output changed" + ); + let (latest, error) = invoke(&client, &saved.function, json!({"value":100})).await?; + ensure!( + error.is_none() + && latest["computed"] == 107 + && latest["counter"] == 1 + && latest["requestId"].as_str() != Some(saved.old_request.as_str()), + "restored latest did not execute cold without replay" + ); + let alias = client + .invoke() + .function_name(&saved.function) + .qualifier("live") + .payload(Blob::new(br#"{"value":50}"#.as_slice())) + .send() + .await?; + let value: Value = + serde_json::from_slice(alias.payload().context("restored alias payload")?.as_ref())?; + ensure!( + alias.function_error().is_none() + && alias.executed_version() == Some("1") + && value["computed"] == 57 + && value["counter"] == 1, + "restored immutable alias failed cold computation" + ); + let next = workflows + .start_execution() + .state_machine_arn(&saved.machine) + .send() + .await?; + terminal(&workflows, next.execution_arn()).await?; + ensure!( + history(&workflows, &saved.execution).await? == saved.history, + "new work rewrote restored history" + ); + Ok(()) +} diff --git a/apps/rustack/tests/docker_startup/sdk/recovery.rs b/apps/rustack/tests/docker_startup/sdk/recovery.rs new file mode 100644 index 00000000..2aa5a2ec --- /dev/null +++ b/apps/rustack/tests/docker_startup/sdk/recovery.rs @@ -0,0 +1,101 @@ +//! Actual warm service witness across another installation's crash recovery. + +use std::collections::BTreeSet; + +use rustack_lambda_core::docker::{DockerWire, OwnerFilter}; + +use super::*; +use crate::process::{self, Server}; + +#[derive(Debug)] +pub(crate) struct Witness { + client: Client, + name: String, + counter: u64, +} +impl Witness { + pub(crate) async fn start(server: &Server) -> Result { + Self::start_image(server, &process::handler()?).await + } + pub(crate) async fn start_image(server: &Server, image: &str) -> Result { + let client = lambda(&server.endpoint); + let name = format!("Survivor{}", Uuid::new_v4().simple()); + client + .create_function() + .function_name(&name) + .role("arn:aws:iam::000000000000:role/Docker") + .package_type(PackageType::Image) + .architectures(if process::architecture()? == "arm64" { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .memory_size(128) + .timeout(30) + .code(FunctionCode::builder().image_uri(image).build()) + .send() + .await?; + active(&client, &name, "Active").await?; + let mut witness = Self { + client, + name, + counter: 0, + }; + witness.advance().await?; + Ok(witness) + } + pub(crate) async fn missing(&self) -> Result<()> { + let error = self + .client + .get_function_configuration() + .function_name(&self.name) + .send() + .await + .err() + .context("unsaved function survived failed shutdown")?; + ensure!( + error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("ResourceNotFoundException"), + "wrong rejected unsaved function error" + ); + Ok(()) + } + pub(crate) async fn cold_revision(&mut self, expected: &str) -> Result<()> { + let (value, error) = invoke(&self.client, &self.name, json!({"value":35})).await?; + ensure!( + error.is_none() + && value["computed"] == 42 + && value["counter"] == 1 + && value["imageRevision"] == expected, + "restore substituted the changed tag" + ); + self.counter = 1; + Ok(()) + } + pub(crate) async fn advance(&mut self) -> Result<()> { + let (value, error) = invoke(&self.client, &self.name, json!({"value":35})).await?; + self.counter = self.counter.checked_add(1).context("witness counter")?; + ensure!( + error.is_none() && value["computed"] == 42 && value["counter"] == self.counter, + "surviving installation lost its warm environment" + ); + Ok(()) + } +} +pub(crate) async fn containers(server: &Server) -> Result> { + let (owner, _) = server.observation_identity().await?; + let deadline = Instant::now() + Duration::from_secs(10); + let mut wire = DockerWire::connect(process::socket()?.parse()?, deadline).await?; + let result = wire + .port() + .list_containers(OwnerFilter::new(owner.parse()?, None, None)?, deadline) + .await; + let stopped = wire.shutdown().await; + stopped?; + result? + .into_iter() + .map(|row| row.id.context("observed full container ID")) + .collect() +} diff --git a/apps/rustack/tests/localstack_compat.rs b/apps/rustack/tests/localstack_compat.rs new file mode 100644 index 00000000..a2502063 --- /dev/null +++ b/apps/rustack/tests/localstack_compat.rs @@ -0,0 +1,471 @@ +//! Original external pytest cases against a directly owned actual Rustack process. +#![cfg(all(unix, feature = "lambda", feature = "stepfunctions"))] +#![forbid(unsafe_code)] + +#[path = "workflows_sdk/runner.rs"] +mod runner; + +use std::{ + env, + ffi::OsString, + fmt::Write as FmtWrite, + fs::Permissions, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{ExitStatus, Stdio}, + time::Duration, +}; + +use anyhow::{Context, Result, bail, ensure}; +use nix::{ + sys::signal::{Signal, kill}, + unistd::Pid, +}; +use runner::{Runner, interpreter}; +use rustack_lambda_core::docker::{DirectoryLease, DockerWire, reconcile_namespace}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tokio::{ + fs, + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + process::{Child, Command}, + task::JoinHandle, + time::{Instant, sleep, timeout, timeout_at}, +}; +use uuid::Uuid; + +#[derive(Debug)] +struct Server { + child: Child, + output: Vec>>>, + endpoint: String, + root: PathBuf, + docker: bool, +} + +impl Server { + async fn spawn(root: &Path, docker: bool) -> Result { + let reservation = TcpListener::bind("127.0.0.1:0").await?; + let address = reservation.local_addr()?; + drop(reservation); + let config = root.join("config.yaml"); + let access = format!("compat-{}", Uuid::new_v4().simple()); + let secret = Uuid::new_v4().simple().to_string(); + let auth = root.join("auth.json"); + save( + &auth, + &serde_json::to_vec(&json!({"accessKey":access, "secretKey":secret}))?, + ) + .await?; + save( + &config, + b"budgets:\n startupSeconds: 180\n shutdownSeconds: 60\n workflowDrainSeconds: 0\n \ + workflowTransitions: 100000\n workflowHistoryBytes: 33554432\n \ + workflowTotalHistoryBytes: 268435456\n lambdaDockerIoSeconds: 60\n \ + lambdaDockerStartSeconds: 60\n lambdaDockerCleanupSeconds: 60\n \ + lambdaDockerEnvironments: 2\n", + ) + .await?; + let mut command = Command::new(env!("CARGO_BIN_EXE_rustack")); + command + .env_clear() + .env("GATEWAY_LISTEN", address.to_string()) + .env( + "SERVICES", + if docker { + "stepfunctions,lambda" + } else { + "stepfunctions" + }, + ) + .env("RUSTACK_CONFIG", &config) + .env("DATA_DIR", root.join("data")) + .env( + "LAMBDA_EXECUTOR", + if docker { "docker" } else { "disabled" }, + ) + .env("STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION", "false") + .env("LAMBDA_SKIP_SIGNATURE_VALIDATION", "false") + .env("ACCESS_KEY", access) + .env("SECRET_KEY", secret) + .env("LOG_LEVEL", "error") + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + if docker { + command + .env( + "LAMBDA_DOCKER_SOCKET", + format!("unix://{}", env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")?), + ) + .env( + "LAMBDA_DOCKER_RELAY_IMAGE", + env::var("RUSTACK_RUNTIME_IMAGE")?, + ) + .env("LAMBDA_DOCKER_NETWORK", "none") + .env("LAMBDA_DOCKER_CROSS_ARCHITECTURE", "false") + .env("LAMBDA_INIT_TIMEOUT_SECS", "20"); + } + let child = command + .spawn() + .context("start actual compatibility server")?; + let mut owned = Self { + child, + output: Vec::new(), + endpoint: format!("http://{address}"), + root: root.to_owned(), + docker, + }; + if let Some(stream) = owned.child.stdout.take() { + owned.output.push(tokio::spawn(capture(stream))); + } + if let Some(stream) = owned.child.stderr.take() { + owned.output.push(tokio::spawn(capture(stream))); + } + Ok(owned) + } + + async fn ready(&mut self) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(180); + while Instant::now() < deadline { + ensure!( + self.child.try_wait()?.is_none(), + "compatibility server exited during startup" + ); + if health(&self.endpoint).await.is_ok() { + return Ok(()); + } + sleep(Duration::from_millis(50)).await; + } + bail!("compatibility readiness deadline") + } + + async fn stop(&mut self) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(90); + let status = self.retire(deadline).await; + let mut logs = Vec::new(); + let mut errors = Vec::new(); + for handle in self.output.drain(..) { + match join_capture(handle, deadline).await { + Ok(bytes) => logs.extend(bytes), + Err(error) => errors.push(error.to_string()), + } + } + let saved = save(&self.root.join("server.log"), &logs).await; + let server_joined = self.child.id().is_none(); + let audit = if self.docker && server_joined { + self.audit().await + } else { + Ok(()) + }; + let original_bytes = timeout( + Duration::from_secs(5), + fs::read(self.root.join("server-intent.json")), + ) + .await + .context("server intent read timeout")??; + let original: Value = serde_json::from_slice(&original_bytes)?; + let expected = original + .get("sha256") + .and_then(Value::as_str) + .context("server intent fingerprint")?; + let same_binary = binary_hash().await? == expected; + let cleanup = json!({"serverJoined":server_joined, "captureTasksJoined":true, + "ownedResourcesAbsent":server_joined && audit.is_ok(), "serverBinaryUnchanged":same_binary, + "dockerAuditPerformed":self.docker && server_joined, + "exitSuccess":status.as_ref().is_ok_and(ExitStatus::success), + "serverIntentSha256":hexadecimal(&Sha256::digest(&original_bytes))?}); + save( + &self.root.join("server-cleanup.json"), + &serde_json::to_vec(&cleanup)?, + ) + .await?; + let status = status?; + audit?; + saved?; + ensure!( + same_binary, + "compatibility server binary changed during execution" + ); + ensure!( + errors.is_empty(), + "compatibility capture failure: {errors:?}" + ); + ensure!(status.success(), "compatibility server exit: {status}"); + Ok(()) + } + + async fn retire(&mut self, deadline: Instant) -> Result { + if let Some(status) = self.child.try_wait()? { + return Ok(status); + } + let pid = i32::try_from(self.child.id().context("owned compatibility server PID")?)?; + let signalled = kill(Pid::from_raw(pid), Signal::SIGTERM); + let status = if let Ok(status) = + timeout_at(deadline - Duration::from_secs(10), self.child.wait()).await + { + status? + } else { + let killed = self.child.start_kill(); + let status = timeout_at(deadline, self.child.wait()) + .await + .context("forced server reap")??; + killed?; + status + }; + signalled.context("stop exact compatibility server")?; + Ok(status) + } + + async fn audit(&self) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(60); + let lease = DirectoryLease::acquire(self.root.join("data"), deadline).await?; + let socket = format!("unix://{}", env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")?); + let mut wire = DockerWire::connect(socket.parse()?, deadline).await?; + let checked = async { + let port = wire.port().bind_directory(&lease, deadline).await?; + let report = reconcile_namespace(&lease, &port, deadline).await?; + ensure!( + report.containers.settled == 0 + && report.containers.remaining == 0 + && report.pins.settled == 0 + && report.pins.remaining == 0 + && report.code.settled == 0 + && report.code.remaining == 0, + "compatibility shutdown left owned Docker resources" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + let stopped = wire.shutdown().await; + checked?; + stopped?; + Ok(()) + } +} + +async fn capture(source: R) -> Result> { + let mut bytes = Vec::new(); + source.take(1_048_577).read_to_end(&mut bytes).await?; + ensure!( + bytes.len() <= 1_048_576, + "compatibility server diagnostic bound" + ); + Ok(bytes) +} + +async fn join_capture( + mut handle: JoinHandle>>, + deadline: Instant, +) -> Result> { + if let Ok(result) = timeout_at(deadline, &mut handle).await { + result.context("compatibility capture task panicked")? + } else { + handle.abort(); + let joined = handle.await; + if let Err(error) = joined { + ensure!( + error.is_cancelled(), + "capture task panicked during retirement: {error}" + ); + } + bail!("compatibility capture deadline; task aborted and joined") + } +} + +async fn save(path: &Path, bytes: &[u8]) -> Result<()> { + timeout(Duration::from_secs(5), async { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .await?; + file.write_all(bytes).await?; + file.sync_all().await?; + Ok::<(), anyhow::Error>(()) + }) + .await + .context("compatibility evidence IO timeout")? +} + +async fn binary_hash() -> Result { + timeout(Duration::from_secs(10), async { + let mut source = fs::File::open(env!("CARGO_BIN_EXE_rustack")).await?; + let mut hash = Sha256::new(); + let mut bytes = vec![0_u8; 65536].into_boxed_slice(); + let mut total = 0_u64; + loop { + let count = source.read(&mut bytes).await?; + if count == 0 { + break; + } + total = total.saturating_add(u64::try_from(count)?); + ensure!(total <= 268_435_456, "compatibility executable size bound"); + hash.update(bytes.get(..count).context("executable read slice")?); + } + hexadecimal(&hash.finalize()) + }) + .await + .context("compatibility executable fingerprint timeout")? +} + +fn hexadecimal(bytes: &[u8]) -> Result { + let mut text = String::with_capacity(bytes.len().saturating_mul(2)); + for byte in bytes { + write!(&mut text, "{byte:02x}")?; + } + Ok(text) +} + +async fn health(endpoint: &str) -> Result<()> { + timeout(Duration::from_secs(2), async { + let address = endpoint + .strip_prefix("http://") + .context("compatibility endpoint")?; + let mut stream = TcpStream::connect(address).await?; + stream + .write_all( + b"GET /_localstack/health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ) + .await?; + let mut bytes = Vec::new(); + stream.take(16385).read_to_end(&mut bytes).await?; + ensure!( + bytes.len() <= 16384 && bytes.starts_with(b"HTTP/1.1 200"), + "compatibility readiness response" + ); + Ok::<(), anyhow::Error>(()) + }) + .await + .context("compatibility readiness timeout")? +} + +async fn python(arguments: &[OsString]) -> Result<()> { + let mut child = Runner::spawn(&interpreter()?, arguments)?; + let entered = child.await_pid().await; + let report = child.wait(Instant::now() + Duration::from_mins(30)).await?; + ensure!( + entered? == report.pid, + "compatibility child identity changed" + ); + if !report.status.success() { + eprintln!( + "compatibility child: {} {}", + String::from_utf8_lossy(&report.stdout), + String::from_utf8_lossy(&report.stderr) + ); + } + ensure!( + !report.forced && report.status.success(), + "external compatibility failed; retained evidence" + ); + Ok(()) +} + +async fn exercise(docker: bool) -> Result<()> { + let root = PathBuf::from( + env::var_os("RUSTACK_LOCALSTACK_EVIDENCE") + .context("explicit compatibility evidence directory")?, + ); + ensure!( + root.is_absolute(), + "compatibility evidence must be absolute" + ); + let parent = root.parent().context("compatibility evidence parent")?; + let canonical = timeout(Duration::from_secs(5), fs::canonicalize(parent)) + .await + .context("evidence parent IO timeout")??; + ensure!( + Some(canonical.as_path()) == root.parent(), + "compatibility evidence parent must be canonical" + ); + timeout(Duration::from_secs(5), fs::create_dir(&root)) + .await + .context("create evidence timeout")??; + timeout( + Duration::from_secs(5), + fs::set_permissions(&root, Permissions::from_mode(0o700)), + ) + .await + .context("private evidence timeout")??; + save( + &root.join("server-intent.json"), + &serde_json::to_vec(&json!({ + "path":env!("CARGO_BIN_EXE_rustack"), "sha256":binary_hash().await?, "docker":docker}))?, + ) + .await?; + let script = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/workflows/localstack_compat.py"); + let mode = if docker { "docker" } else { "pure" }; + let images = if docker { + env::var_os("RUSTACK_LOCALSTACK_IMAGES") + .context("explicit immutable handler image receipt")? + } else { + OsString::from("-") + }; + let socket = if docker { + OsString::from(format!( + "unix://{}", + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")? + )) + } else { + OsString::from("-") + }; + python(&[ + script.clone().into_os_string(), + "preflight".into(), + root.clone().into_os_string(), + mode.into(), + images.clone(), + socket, + ]) + .await?; + let mut server = Server::spawn(&root, docker).await?; + let result = async { + server.ready().await?; + python(&[ + script.into_os_string(), + "run".into(), + root.into_os_string(), + mode.into(), + OsString::from(&server.endpoint), + images, + ]) + .await + } + .await; + let cleanup = server.stop().await; + if let Err(error) = &cleanup { + eprintln!("compatibility cleanup failed: {error:#}"); + } + result?; + cleanup?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires pinned external Python environment; make test-workflows-localstack-pure"] +async fn test_should_execute_original_localstack_pure_cases() -> Result<()> { + exercise(false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires explicit Docker and upstream handler images; make \ + test-workflows-localstack-docker"] +async fn test_should_execute_original_localstack_image_cases() -> Result<()> { + exercise(true).await +} + +#[tokio::test] +async fn test_should_join_capture_when_a_writer_outlives_server_retirement() -> Result<()> { + let (reader, writer) = tokio::io::duplex(16); + let task = tokio::spawn(capture(reader)); + let result = join_capture(task, Instant::now() + Duration::from_millis(10)).await; + ensure!(result.is_err(), "held writer escaped capture deadline"); + drop(writer); + Ok(()) +} diff --git a/apps/rustack/tests/workflows_action.rs b/apps/rustack/tests/workflows_action.rs new file mode 100644 index 00000000..0c2b1329 --- /dev/null +++ b/apps/rustack/tests/workflows_action.rs @@ -0,0 +1,240 @@ +//! Actual composite-Action YAML contract and bounded control metadata export. +#![cfg(unix)] +#![forbid(unsafe_code)] +#![warn( + clippy::pedantic, + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::{ + collections::BTreeMap, + ffi::OsString, + fs::Permissions, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, + time::{Duration, Instant, timeout}, +}; +use yaml_rust2::{Yaml, YamlLoader}; + +#[path = "workflows_sdk/runner.rs"] +mod runner; +use runner::{Runner, interpreter}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ActionContract { + schema_version: u8, + defaults: BTreeMap, + environment: BTreeMap, + run: String, +} + +fn field<'a>(value: &'a Yaml, name: &str) -> Result<&'a Yaml> { + value + .as_hash() + .context("Action mapping required")? + .get(&Yaml::String(name.to_owned())) + .context("Action field missing") +} + +fn string(value: &Yaml, maximum: usize) -> Result { + let value = value.as_str().context("Action string required")?; + ensure!(value.len() <= maximum, "Action string budget"); + Ok(value.to_owned()) +} + +fn contract(source: &str) -> Result { + ensure!(source.len() <= 131_072, "Action source byte budget"); + let documents = YamlLoader::load_from_str(source).context("Action YAML")?; + let [root]: [Yaml; 1] = documents + .try_into() + .map_err(|_| anyhow::anyhow!("one Action document required"))?; + let inputs = field(&root, "inputs")? + .as_hash() + .context("Action inputs mapping")?; + ensure!(inputs.len() <= 32, "Action input count budget"); + let mut defaults = BTreeMap::new(); + for (name, definition) in inputs { + defaults.insert( + string(name, 64)?, + string(field(definition, "default")?, 512)?, + ); + } + let runs = field(&root, "runs")?; + ensure!( + string(field(runs, "using")?, 32)? == "composite", + "composite Action required" + ); + let steps = field(runs, "steps")?.as_vec().context("Action steps")?; + ensure!(steps.len() == 1, "one retained Action entrypoint required"); + let step = steps.first().context("Action step")?; + ensure!( + string(field(step, "id")?, 64)? == "start", + "public output step changed" + ); + ensure!( + string(field(step, "shell")?, 32)? == "bash", + "Action shell changed" + ); + let run = string(field(step, "run")?, 16_384)?; + ensure!( + run.trim() == "exec python3 -B \"$RUSTACK_ACTION_ROOT/tools/github-action/runtime.py\"" + && !run.contains("${{"), + "Action source must not interpolate context" + ); + let env = field(step, "env")? + .as_hash() + .context("Action environment")?; + ensure!(env.len() <= 128, "Action environment budget"); + let mut environment = BTreeMap::new(); + for (name, value) in env { + environment.insert(string(name, 96)?, string(value, 512)?); + } + ensure!( + environment.get("RUSTACK_ACTION_ROOT").map(String::as_str) + == Some("${{ github.action_path }}"), + "trusted Action root binding missing" + ); + ensure!( + environment.len() == defaults.len() + 1, + "Action input/environment drift" + ); + for key in defaults.keys() { + let name = format!("RUSTACK_ACTION_{}", key.to_uppercase().replace('-', "_")); + let expression = format!("${{{{ inputs.{key} }}}}"); + ensure!( + environment.get(&name) == Some(&expression), + "Action input env binding changed" + ); + } + let outputs = field(&root, "outputs")?; + for (name, value) in [ + ("endpoint", "${{ steps.start.outputs.endpoint }}"), + ("container-id", "${{ steps.start.outputs.container_id }}"), + ] { + ensure!( + string(field(field(outputs, name)?, "value")?, 256)? == value, + "Action output contract changed" + ); + } + Ok(ActionContract { + schema_version: 1, + defaults, + environment, + run, + }) +} + +async fn read_contract() -> Result { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let source = timeout(Duration::from_secs(5), async { + let file = fs::File::open(root.join("action.yml")).await?; + ensure!( + file.metadata().await?.len() <= 131_072, + "Action source byte budget" + ); + let mut bytes = Vec::new(); + file.take(131_073).read_to_end(&mut bytes).await?; + ensure!(bytes.len() <= 131_072, "Action source grew beyond budget"); + Ok::<_, anyhow::Error>(String::from_utf8(bytes)?) + }) + .await + .context("Action source IO deadline")??; + contract(&source) +} + +#[tokio::test] +async fn test_should_validate_actual_action_entrypoint_contract() -> Result<()> { + let value = read_contract().await?; + ensure!( + value.defaults.contains_key("stepfunctions-skip-auth"), + "Step Functions Action setting missing" + ); + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ControlReport { + schema_version: u8, + cases: u32, + successful: bool, +} + +async fn private_write(path: &Path, body: &[u8]) -> Result<()> { + ensure!(body.len() <= 131_072, "Action control artifact byte budget"); + timeout(Duration::from_secs(5), async { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true).mode(0o600); + let mut file = options.open(path).await?; + file.write_all(body).await?; + file.sync_all().await?; + Ok::<_, anyhow::Error>(()) + }) + .await + .context("Action private artifact deadline")??; + Ok(()) +} + +#[tokio::test] +#[ignore = "requires provisioned pinned Python; run make test-workflows-action-fixture"] +async fn test_should_execute_actual_action_process_controls() -> Result<()> { + let mut root: TempDir = tempfile::tempdir()?; + root.disable_cleanup(true); + timeout( + Duration::from_secs(5), + fs::set_permissions(root.path(), Permissions::from_mode(0o700)), + ) + .await + .context("Action control root permissions deadline")??; + ensure!( + fs::metadata(root.path()).await?.permissions().mode() & 0o777 == 0o700, + "Action controls require a private directory" + ); + let value = read_contract().await?; + let metadata = root.path().join("action.json"); + private_write(&metadata, &serde_json::to_vec(&value)?).await?; + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/workflows/test_github_action.py"); + let arguments: Vec = vec![script.into_os_string(), metadata.into_os_string()]; + let cutoff = Instant::now() + Duration::from_secs(180); + let mut child = Runner::spawn(&interpreter()?, &arguments)?; + let entered = child.await_pid().await; + let report = child.wait(cutoff).await?; + private_write(&root.path().join("stdout"), &report.stdout).await?; + private_write(&root.path().join("stderr"), &report.stderr).await?; + ensure!( + entered.is_ok_and(|pid| pid == report.pid) && !report.forced && report.status.success(), + "Action controls failed; private evidence {} (stdout SHA256-base64 {}, stderr \ + SHA256-base64 {})", + root.path().display(), + STANDARD.encode(Sha256::digest(&report.stdout)), + STANDARD.encode(Sha256::digest(&report.stderr)) + ); + let observed: ControlReport = + serde_json::from_slice(&report.stdout).context("Action control result schema")?; + ensure!( + observed.schema_version == 1 && observed.successful && (24..=64).contains(&observed.cases), + "Action control catalog incomplete" + ); + println!( + "Actual composite Action controls: {} cases; independent Rust PID/capture/timeout owner \ + joined", + observed.cases + ); + root.disable_cleanup(false); + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk.rs b/apps/rustack/tests/workflows_sdk.rs new file mode 100644 index 00000000..4a100793 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk.rs @@ -0,0 +1,227 @@ +//! Explicit ordinary SDK qualification of the actual binary and Native runtime. +#![cfg(all(feature = "stepfunctions", feature = "lambda", unix))] +#![forbid(unsafe_code)] + +#[path = "workflows_sdk/documented.rs"] +mod documented; +#[path = "workflows_sdk/performance.rs"] +mod performance; +#[path = "workflows_sdk/persistence.rs"] +mod persistence; +#[path = "workflows_sdk/process.rs"] +mod process; +#[path = "workflows_sdk/runner.rs"] +mod runner; +#[path = "workflows_sdk/rust.rs"] +mod rust; + +use std::{ffi::OsString, path::Path, time::Duration}; + +use anyhow::{Context, Error as AnyError, Result, ensure}; +use process::{Server, assert_absent}; +use runner::{Runner, interpreter}; +use tokio::{ + fs, + time::{Instant, timeout}, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-stepfunctions-sdk"] +async fn test_should_accept_ordinary_rust_sdk_through_actual_runtime() -> Result<()> { + let mut server = Server::start().await?; + let result = match timeout(Duration::from_secs(120), rust::exercise(&server)).await { + Ok(result) => result, + Err(error) => Err(error.into()), + }; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + if let Err(error) = &result { + eprintln!("Rust SDK exercise failed: {error:#}"); + } + let observed = cleanup?; + result?; + ensure!(observed > 0, "no actual bootstrap process was observed"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires uv and actual host bootstrap; run make test-stepfunctions-sdk"] +async fn test_should_accept_ordinary_boto3_through_actual_runtime() -> Result<()> { + let python = interpreter()?; + let mut server = Server::start().await?; + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let result = async { + let arguments = [ + root.join("tests/workflows/stepfunctions_sdk.py") + .into_os_string(), + OsString::from(&server.endpoint), + server.bootstrap()?.as_os_str().to_owned(), + server.markers.as_os_str().to_owned(), + ]; + let report = Runner::spawn(&python, &arguments)? + .wait(Instant::now() + Duration::from_secs(120)) + .await?; + if !report.status.success() { + eprintln!("boto3 stderr: {}", String::from_utf8_lossy(&report.stderr)); + } + ensure!( + !report.forced && report.status.success(), + "boto3 qualification failed or exceeded deadline" + ); + let result: serde_json::Value = serde_json::from_slice(&report.stdout)?; + ensure!( + result["catalogCases"] == 17 + && result["m1Actions"] == 15 + && result["nestedCases"] == 6 + && result["nestedLifecycle"] == 2 + && result["structuredCases"] == 6 + && result["structuredLifecycle"] == 2 + && result["actualNative"] == true, + "missing SDK acceptance evidence" + ); + Ok(()) + } + .await; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + let observed = cleanup?; + result?; + ensure!(observed > 0, "no actual bootstrap process was observed"); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires pinned Python; run make test-stepfunctions-sdk"] +async fn test_should_reap_actual_python_after_observed_execution_timeout() -> Result<()> { + for valid in [true, false] { + let program = if valid { + "import os,time; print(os.getpid(), flush=True); time.sleep(60)" + } else { + "import time; print('invalid-pid', flush=True); time.sleep(60)" + }; + let arguments = [OsString::from("-c"), OsString::from(program)]; + let mut runner = Runner::spawn(&interpreter()?, &arguments)?; + let barrier = runner.await_pid().await; + let report = runner + .wait(Instant::now() + Duration::from_millis(25)) + .await?; + ensure!( + report.forced && !report.status.success(), + "did not stop the actual Python process" + ); + assert_absent(report.pid).await?; + if valid { + ensure!(barrier? == report.pid, "wrong consumed PID"); + } else { + ensure!(barrier.is_err(), "malformed PID barrier accepted"); + } + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-stepfunctions-sdk"] +async fn test_should_run_actual_nested_native_with_one_dispatch_slot() -> Result<()> { + let mut server = Server::start_with_dispatch(1).await?; + let result = match timeout(Duration::from_secs(60), rust::nested_only(&server)).await { + Ok(result) => result, + Err(error) => Err(error.into()), + }; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + let observed = cleanup?; + result?; + ensure!(observed > 0, "no actual nested Native handler was observed"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual handler SDK fixtures; run make test-stepfunctions-sdk"] +async fn test_should_run_signed_rust_and_boto3_handler_callbacks_with_one_dispatch_slot() +-> Result<()> { + let mut server = Server::start_with_dispatch(1).await?; + let result = match timeout(Duration::from_secs(180), rust::callbacks_only(&server)).await { + Ok(result) => result, + Err(error) => Err(error.into()), + }; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + if let Err(error) = &result { + eprintln!("handler SDK callback qualification failed: {error:#}"); + } + let observed = cleanup?; + result?; + ensure!(observed > 0, "no actual handler SDK process observed"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-stepfunctions-sdk"] +async fn test_should_join_actual_server_and_native_children_after_fixture_gate_release_failure() +-> Result<()> { + let mut server = Server::start().await?; + let operation = async { + timeout(Duration::from_secs(30), rust::native_probe_only(&server)).await??; + let private = server + .markers + .parent() + .context("owned fixture root")? + .to_owned(); + let mut native = Vec::new(); + let mut directory = fs::read_dir(&server.markers).await?; + while let Some(entry) = directory.next_entry().await? { + ensure!( + native.len() < 64 && entry.metadata().await?.len() <= 11, + "invalid PID evidence" + ); + native.push(fs::read_to_string(entry.path()).await?.parse::()?); + } + fs::rename( + private.join("callback-gates"), + private.join("missing-callback-gates"), + ) + .await?; + Ok::<_, AnyError>((private, native)) + } + .await; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + assert_absent(server.pid).await?; + let (private, native) = operation?; + for pid in &native { + assert_absent(*pid).await?; + } + ensure!( + cleanup.is_err() + && server + .shutdown(Instant::now() + Duration::from_secs(1)) + .await + .is_err(), + "fixture gate failure was not retained" + ); + ensure!( + fs::try_exists(&private).await?, + "failed qualification evidence was discarded" + ); + fs::remove_dir_all(&private).await?; + ensure!(!native.is_empty(), "no actual Native child observed"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "actual owned pure-service smoke; run make smoke-workflows"] +async fn test_should_execute_documented_pure_workflow_in_owned_strict_service() -> Result<()> { + let mut server = Server::start_pure().await?; + let result = timeout(Duration::from_secs(60), documented::exercise(&server)).await; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + ensure!(cleanup? == 0, "pure smoke created Native Lambda processes"); + result??; + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk/documented.rs b/apps/rustack/tests/workflows_sdk/documented.rs new file mode 100644 index 00000000..1044f51d --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/documented.rs @@ -0,0 +1,171 @@ +//! Source-controlled operator example through an owned strict-auth pure service. +#![warn( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use anyhow::{Context, Result, ensure}; +use aws_sdk_sfn::{ + Client, + types::{ExecutionStatus, HistoryEvent, StateEnteredEventDetails, StateExitedEventDetails}, +}; +use serde_json::{Value, json}; + +use super::{ + process::{SECRET, Server}, + rust::{history, sequential_predecessor, sfn, terminal}, +}; + +const DEFINITION: &str = include_str!("../../../../examples/workflows/pure.asl.json"); + +async fn run(client: &Client, machine: &str, positive: bool) -> Result<()> { + let input = if positive { + r#"{ "n": 6 }"# + } else { + r#"{"n":0}"# + }; + let started = client + .start_execution() + .state_machine_arn(machine) + .name(if positive { "Positive" } else { "Rejected" }) + .input(input) + .send() + .await?; + let result = terminal(client, started.execution_arn()).await?; + ensure!( + result.input() == Some(input), + "documented workflow lost raw input" + ); + if positive { + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "positive example failed" + ); + ensure!( + serde_json::from_str::(result.output().context("missing output")?)? + == json!({"doubled":12}), + "wrong documented result" + ); + } else { + ensure!( + result.status() == &ExecutionStatus::Failed + && result.error() == Some("Example.NotPositive"), + "Fail branch did not reject nonpositive input" + ); + } + let events = history(client, started.execution_arn(), false, true).await?; + for (index, event) in events.iter().enumerate() { + let previous = i64::try_from(index)?; + ensure!( + event.id() == previous.checked_add(1).context("event ID overflow")? + && event.previous_event_id() == sequential_predecessor(index)?, + "invalid example causal history" + ); + } + let kinds: Vec<&str> = events.iter().map(|event| event.r#type().as_str()).collect(); + let expected_kinds: &[&str] = if positive { + &[ + "ExecutionStarted", + "PassStateEntered", + "PassStateExited", + "ChoiceStateEntered", + "ChoiceStateExited", + "WaitStateEntered", + "WaitStateExited", + "SucceedStateEntered", + "SucceedStateExited", + "ExecutionSucceeded", + ] + } else { + &[ + "ExecutionStarted", + "PassStateEntered", + "PassStateExited", + "ChoiceStateEntered", + "ChoiceStateExited", + "FailStateEntered", + "ExecutionFailed", + ] + }; + ensure!(kinds == expected_kinds, "wrong documented event sequence"); + let entered: Vec<&str> = events + .iter() + .filter_map(HistoryEvent::state_entered_event_details) + .map(StateEnteredEventDetails::name) + .collect(); + let expected: &[&str] = if positive { + &["Compute", "Positive", "Pause", "Done"] + } else { + &["Compute", "Positive", "Reject"] + }; + ensure!(entered == expected, "wrong example branch/state order"); + let exited: Vec<&str> = events + .iter() + .filter_map(HistoryEvent::state_exited_event_details) + .map(StateExitedEventDetails::name) + .collect(); + let expected_exits: &[&str] = if positive { + &["Compute", "Positive", "Pause", "Done"] + } else { + &["Compute", "Positive"] + }; + ensure!(exited == expected_exits, "wrong example state exit names"); + Ok(()) +} + +pub(super) async fn exercise(server: &Server) -> Result<()> { + let capabilities = server.capabilities().await?; + for key in [ + "lambdaRequestResponse", + "callbacks", + "dockerQualified", + "snapshotResume", + ] { + ensure!( + capabilities.pointer(&format!("/workflows/{key}")) == Some(&Value::Bool(false)), + "pure service advertised unavailable integration" + ); + } + ensure!( + capabilities.pointer("/workflows/snapshot") == Some(&json!("resources-and-history")) + && capabilities.pointer("/workflows/paginationEpoch") + == Some(&json!("fresh-on-restart")), + "wrong pure snapshot capability" + ); + ensure!( + server.bootstrap().is_err(), + "pure fixture exposed a Native bootstrap" + ); + let client = sfn(server, SECRET); + let validation = client + .validate_state_machine_definition() + .definition(DEFINITION) + .send() + .await?; + ensure!( + validation.result().as_str() == "OK", + "documented raw definition did not validate" + ); + let created = client + .create_state_machine() + .name("Documented") + .role_arn("arn:aws:iam::000000000000:role/Workflows") + .definition(DEFINITION) + .send() + .await?; + let machine = created.state_machine_arn(); + let captured = client + .describe_state_machine() + .state_machine_arn(machine) + .send() + .await?; + ensure!( + captured.definition() == DEFINITION, + "documented raw source was normalized or replaced" + ); + run(&client, machine, true).await?; + run(&client, machine, false).await?; + server.readiness().await +} diff --git a/apps/rustack/tests/workflows_sdk/performance.rs b/apps/rustack/tests/workflows_sdk/performance.rs new file mode 100644 index 00000000..cb5b3c9e --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/performance.rs @@ -0,0 +1,245 @@ +//! Actual signed SDK workloads; measurements publish only after process cleanup. +#![warn( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::{env::consts, future::Future, thread, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use tokio::time::{Instant, timeout}; + +use super::{process::Server, rust::sfn}; + +#[path = "performance/latency.rs"] +mod latency; +#[path = "performance/load.rs"] +mod load; +#[cfg(test)] +#[path = "performance/tests.rs"] +mod tests; + +const SAMPLES: usize = 100; +const OPERATION_BUDGET: Duration = Duration::from_secs(2); + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Samples { + count: usize, + observed_micros: Vec, + p50_micros: u64, + p95_micros: u64, + max_micros: u64, +} +impl Samples { + fn new(observed_micros: Vec) -> Result { + ensure!(observed_micros.len() == SAMPLES, "wrong measurement count"); + let mut ranked = observed_micros.clone(); + ranked.sort_unstable(); + Ok(Self { + count: observed_micros.len(), + observed_micros, + p50_micros: *ranked.get(49).context("missing median rank")?, + p95_micros: *ranked.get(94).context("missing p95 rank")?, + max_micros: *ranked.last().context("missing maximum")?, + }) + } + fn check_p95(&self, budget: u64) -> Result<()> { + ensure!(self.p95_micros <= budget, "workflow p95 budget exceeded"); + Ok(()) + } +} + +#[derive(Debug, Serialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +enum Measurement { + PureLatency { + state_count: usize, + input_bytes: usize, + executions: usize, + budget_micros: u64, + latency: Samples, + }, + LoopResponsiveness { + open_execution_slots: usize, + completed_cycles: usize, + input_bytes: usize, + history_event_high_water: i64, + exhaustion: load::ExhaustionProbes, + readiness: Samples, + stop: Samples, + }, +} +impl Measurement { + fn check(&self) -> Result<()> { + match self { + Self::PureLatency { + latency, + budget_micros, + .. + } => latency.check_p95(*budget_micros), + Self::LoopResponsiveness { + readiness, + stop, + exhaustion, + .. + } => { + exhaustion.check()?; + ensure!( + readiness.max_micros < 2_000_000, + "readiness budget exceeded" + ); + ensure!(stop.max_micros < 2_000_000, "Stop budget exceeded"); + Ok(()) + } + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Host { + os: &'static str, + architecture: &'static str, + available_logical_cpus: usize, + debug_assertions: bool, + reference_qualification: &'static str, +} +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Limits { + open_executions: usize, + transitions: u32, + history_events: usize, + history_bytes: usize, + total_history_bytes: usize, +} +impl Limits { + async fn read(server: &Server) -> Result { + let capabilities = server.capabilities().await?; + let limits: Self = serde_json::from_value( + capabilities + .pointer("/workflows/limits") + .context("missing workflow limits")? + .clone(), + )?; + ensure!( + limits.open_executions == 1 + && limits.transitions == 1_000_000 + && limits.history_events == 25_000 + && limits.history_bytes == 33_554_432 + && limits.total_history_bytes == 134_217_728, + "actual performance limits differ from contract" + ); + Ok(limits) + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Report { + schema_version: u8, + host: Host, + limits: Limits, + native_processes_observed: usize, + server_joined: bool, + measurement: Measurement, +} +impl Report { + fn emit( + measurement: Measurement, + limits: Limits, + native_processes_observed: usize, + ) -> Result<()> { + ensure!( + native_processes_observed == 0, + "pure workload invoked Lambda" + ); + let report = Self { + schema_version: 1, + host: Host { + os: consts::OS, + architecture: consts::ARCH, + available_logical_cpus: thread::available_parallelism()?.get(), + debug_assertions: cfg!(debug_assertions), + reference_qualification: "pending", + }, + limits, + native_processes_observed, + server_joined: true, + measurement, + }; + let encoded = serde_json::to_string(&report)?; + ensure!(encoded.len() <= 65_536, "oversized measurement report"); + println!("WORKFLOWS_PERFORMANCE {encoded}"); + report.measurement.check() + } +} + +fn micros(duration: Duration) -> Result { + let rounded = duration + .as_nanos() + .checked_add(999) + .context("duration overflow")? + / 1000; + Ok(u64::try_from(rounded)?) +} +async fn bounded>( + future: impl Future>, +) -> Result { + let began = Instant::now(); + let result = timeout(OPERATION_BUDGET, future) + .await + .context("workload operation expired")?; + ensure!(began.elapsed() < OPERATION_BUDGET, "late workload result"); + result.map_err(Into::into) +} +fn input() -> String { + format!(r#"{{"payload":"{}"}}"#, "x".repeat(1010)) +} + +async fn run(latency_workload: bool) -> Result<()> { + let mut server = Server::start_performance().await?; + let client = sfn(&server, super::process::SECRET); + let began = Instant::now(); + let result = timeout(Duration::from_secs(120), async { + let limits = Limits::read(&server).await?; + let measurement = if latency_workload { + latency::measure(&client).await? + } else { + load::measure(&client, &server).await? + }; + Ok::<_, anyhow::Error>((limits, measurement)) + }) + .await + .context("complete workload expired"); + let result = if began.elapsed() >= Duration::from_secs(120) { + Err(anyhow::anyhow!("late complete workload result")) + } else { + result + }; + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(45)) + .await; + let observed = cleanup?; + let (limits, measurement) = result??; + Report::emit(measurement, limits, observed) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "real measured SDK workload; run make test-workflows-performance"] +async fn test_should_measure_one_hundred_fifty_state_signed_sdk_executions() -> Result<()> { + run(true).await +} +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "real loop/Stop workload; run make test-workflows-performance"] +async fn test_should_keep_health_and_stop_responsive_and_reuse_one_execution_slot() -> Result<()> { + run(false).await +} diff --git a/apps/rustack/tests/workflows_sdk/performance/latency.rs b/apps/rustack/tests/workflows_sdk/performance/latency.rs new file mode 100644 index 00000000..576bf0ae --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/performance/latency.rs @@ -0,0 +1,153 @@ +//! Full pure workflow latency, with exact public output/history validation. + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use aws_sdk_sfn::{ + Client, + operation::describe_execution::DescribeExecutionOutput, + types::{ + ExecutionStatus, HistoryEvent, HistoryEventType, StateEnteredEventDetails, + StateExitedEventDetails, + }, +}; +use serde_json::{Map, Value, json}; +use tokio::time::{Instant, sleep}; + +use super::{Measurement, SAMPLES, Samples, bounded, input, micros}; +use crate::rust::create; + +fn definition() -> Value { + let mut states = Map::new(); + for index in 0..50 { + let name = format!("S{index:02}"); + let state = if index == 49 { + json!({"Type":"Succeed"}) + } else if index % 2 == 0 { + json!({"Type":"Pass","Next":format!("S{:02}",index+1)}) + } else { + json!({"Type":"Choice","Choices":[{"Variable":"$.payload","IsString":true,"Next":format!("S{:02}",index+1)}],"Default":"S49"}) + }; + states.insert(name, state); + } + json!({"StartAt":"S00","States":states}) +} +pub(super) async fn terminal(client: &Client, arn: &str) -> Result { + bounded(async { + loop { + let result = client + .describe_execution() + .execution_arn(arn) + .send() + .await?; + if result.status() != &ExecutionStatus::Running { + return Ok::<_, anyhow::Error>(result); + } + sleep(Duration::from_millis(1)).await; + } + }) + .await +} +async fn check_history(client: &Client, arn: &str) -> Result<()> { + let history = bounded( + client + .get_execution_history() + .execution_arn(arn) + .max_results(1000) + .include_execution_data(false) + .send(), + ) + .await?; + ensure!( + history.next_token().is_none(), + "unexpected history pagination" + ); + validate_history(history.events()) +} +pub(super) fn validate_history(events: &[HistoryEvent]) -> Result<()> { + ensure!(events.len() == 102, "wrong pure history event count"); + for (index, event) in events.iter().enumerate() { + let number = i64::try_from(index)?; + let previous = if index == 1 { 0 } else { number }; + ensure!( + event.id() == number.checked_add(1).context("event ID overflow")? + && event.previous_event_id() == previous, + "invalid sequential causal history" + ); + if index == 0 { + ensure!( + event.r#type() == &HistoryEventType::ExecutionStarted, + "missing first start" + ); + } else if index == 101 { + ensure!( + event.r#type() == &HistoryEventType::ExecutionSucceeded, + "missing final success" + ); + } else { + let offset = index.checked_sub(1).context("invalid state event index")?; + let state = offset / 2; + let entered = offset % 2 == 0; + let expected = match (state, entered) { + (49, true) => HistoryEventType::SucceedStateEntered, + (49, false) => HistoryEventType::SucceedStateExited, + (state, true) if state % 2 == 0 => HistoryEventType::PassStateEntered, + (state, false) if state % 2 == 0 => HistoryEventType::PassStateExited, + (_, true) => HistoryEventType::ChoiceStateEntered, + (_, false) => HistoryEventType::ChoiceStateExited, + }; + let name = if entered { + event + .state_entered_event_details() + .map(StateEnteredEventDetails::name) + } else { + event + .state_exited_event_details() + .map(StateExitedEventDetails::name) + }; + ensure!( + event.r#type() == &expected && name == Some(format!("S{state:02}").as_str()), + "wrong state event kind/name" + ); + } + } + Ok(()) +} + +pub(super) async fn measure(client: &Client) -> Result { + let input = input(); + ensure!(input.len() == 1024, "incorrect workload input size"); + let machine = bounded(create(client, "Latency", &definition())).await?; + let mut samples = Vec::with_capacity(SAMPLES); + for index in 0..SAMPLES { + let began = Instant::now(); + let arn = bounded(Box::pin(async { + let started = client + .start_execution() + .state_machine_arn(&machine) + .name(format!("Latency{index}")) + .input(&input) + .send() + .await?; + let arn = started.execution_arn().to_owned(); + let result = terminal(client, &arn).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded + && result.input() == Some(input.as_str()) + && result.output() == Some(input.as_str()), + "wrong pure workflow result" + ); + Ok::<_, anyhow::Error>(arn) + })) + .await?; + samples.push(micros(began.elapsed())?); + check_history(client, &arn).await?; + } + Ok(Measurement::PureLatency { + state_count: 50, + input_bytes: input.len(), + executions: SAMPLES, + budget_micros: 100_000, + latency: Samples::new(samples)?, + }) +} diff --git a/apps/rustack/tests/workflows_sdk/performance/load.rs b/apps/rustack/tests/workflows_sdk/performance/load.rs new file mode 100644 index 00000000..b94a3fb4 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/performance/load.rs @@ -0,0 +1,209 @@ +//! Real running loops, bounded history exhaustion and one-slot Stop cycles. + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use aws_sdk_sfn::{ + Client, + types::{ExecutionStatus, HistoryEventType}, +}; +use serde::Serialize; +use serde_json::json; +use tokio::time::{Instant, sleep, timeout}; + +use super::{Measurement, SAMPLES, Samples, bounded, input, latency, micros}; +use crate::{process::Server, rust::create}; + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ExhaustionProbes { + probe_count: u32, + live_probe_count: u32, + max_readiness_micros: u64, +} +impl ExhaustionProbes { + pub(super) fn record( + &mut self, + before: &ExecutionStatus, + after: &ExecutionStatus, + elapsed: u64, + ) -> Result<()> { + self.probe_count = self + .probe_count + .checked_add(1) + .context("probe count overflow")?; + ensure!(self.probe_count <= 10_000, "unbounded exhaustion probes"); + if before == &ExecutionStatus::Running && after == &ExecutionStatus::Running { + self.live_probe_count = self + .live_probe_count + .checked_add(1) + .context("live probe count overflow")?; + } + self.max_readiness_micros = self.max_readiness_micros.max(elapsed); + Ok(()) + } + pub(super) fn check(&self) -> Result<()> { + ensure!( + self.live_probe_count > 0, + "no readiness probe overlapped live exhaustion work" + ); + ensure!( + self.max_readiness_micros < 2_000_000, + "exhaustion readiness budget exceeded" + ); + Ok(()) + } +} + +async fn start(client: &Client, machine: &str, name: String, input: &str) -> Result { + let result = bounded( + client + .start_execution() + .state_machine_arn(machine) + .name(name) + .input(input) + .send(), + ) + .await?; + Ok(result.execution_arn().to_owned()) +} +async fn live_history(client: &Client, arn: &str) -> Result<()> { + bounded(async { + loop { + let result = client + .get_execution_history() + .execution_arn(arn) + .max_results(5) + .include_execution_data(false) + .send() + .await?; + if result.events().len() >= 5 { + let state = client + .describe_execution() + .execution_arn(arn) + .send() + .await?; + ensure!( + state.status() == &ExecutionStatus::Running, + "loop ended before Stop" + ); + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(1)).await; + } + }) + .await +} +async fn exhaust( + client: &Client, + server: &Server, + machine: &str, + input: &str, +) -> Result<(i64, ExhaustionProbes)> { + let arn = start(client, machine, "Exhaust".to_owned(), input).await?; + let mut probes = ExhaustionProbes::default(); + let began = Instant::now(); + let result = timeout(Duration::from_secs(20), async { + loop { + let before = bounded(client.describe_execution().execution_arn(&arn).send()).await?; + if before.status() != &ExecutionStatus::Running { + return Ok::<_, anyhow::Error>(before); + } + let began = Instant::now(); + server.readiness().await?; + let elapsed = micros(began.elapsed())?; + let state = bounded(client.describe_execution().execution_arn(&arn).send()).await?; + probes.record(before.status(), state.status(), elapsed)?; + if state.status() != &ExecutionStatus::Running { + return Ok::<_, anyhow::Error>(state); + } + sleep(Duration::from_millis(1)).await; + } + }) + .await + .context("loop did not reach its configured history limit")??; + ensure!( + began.elapsed() < Duration::from_secs(20), + "late exhaustion result" + ); + ensure!( + result.status() == &ExecutionStatus::Failed + && result.error() == Some("States.HistoryLimitExceeded"), + "loop did not fail at history bound" + ); + let history = bounded( + client + .get_execution_history() + .execution_arn(&arn) + .reverse_order(true) + .include_execution_data(false) + .max_results(1) + .send(), + ) + .await?; + let event = history + .events() + .first() + .context("missing reserved terminal event")?; + ensure!( + event.r#type() == &HistoryEventType::ExecutionFailed && (5..=25_000).contains(&event.id()), + "invalid bounded terminal history" + ); + ensure!( + event + .execution_failed_event_details() + .and_then(|details| details.error()) + == Some("States.HistoryLimitExceeded"), + "terminal history disagrees with outcome" + ); + probes.check()?; + Ok((event.id(), probes)) +} +pub(super) async fn measure(client: &Client, server: &Server) -> Result { + let definition = json!({"StartAt":"Choose","States":{ + "Choose":{"Type":"Choice","Choices":[{"Variable":"$.halt","IsPresent":true,"Next":"Done"}],"Default":"Loop"}, + "Loop":{"Type":"Pass","Next":"Choose"}, + "Done":{"Type":"Succeed"} + }}); + let machine = bounded(create(client, "Loop", &definition)).await?; + let input = input(); + let (history_event_high_water, exhaustion) = exhaust(client, server, &machine, &input).await?; + let mut readiness = Vec::with_capacity(SAMPLES); + let mut stop = Vec::with_capacity(SAMPLES); + for index in 0..SAMPLES { + let arn = start(client, &machine, format!("Stop{index}"), &input).await?; + live_history(client, &arn).await?; + let began = Instant::now(); + server.readiness().await?; + readiness.push(micros(began.elapsed())?); + let began = Instant::now(); + let state = bounded(Box::pin(async { + client + .stop_execution() + .execution_arn(&arn) + .error("Performance.Stop") + .cause("fixed synthetic workload") + .send() + .await?; + latency::terminal(client, &arn).await + })) + .await?; + stop.push(micros(began.elapsed())?); + ensure!( + state.status() == &ExecutionStatus::Aborted + && state.error() == Some("Performance.Stop") + && state.input() == Some(input.as_str()), + "Stop did not abort the live loop" + ); + } + server.readiness().await?; + Ok(Measurement::LoopResponsiveness { + open_execution_slots: 1, + completed_cycles: SAMPLES, + input_bytes: input.len(), + history_event_high_water, + exhaustion, + readiness: Samples::new(readiness)?, + stop: Samples::new(stop)?, + }) +} diff --git a/apps/rustack/tests/workflows_sdk/performance/tests.rs b/apps/rustack/tests/workflows_sdk/performance/tests.rs new file mode 100644 index 00000000..88750eba --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/performance/tests.rs @@ -0,0 +1,180 @@ +//! Independent rank and microsecond-boundary controls for measurement reports. + +use std::time::Duration; + +use anyhow::{Context, Result}; +use aws_sdk_sfn::{ + primitives::DateTime, + types::{ + ExecutionStatus, HistoryEvent, HistoryEventType, + HistoryEventType::{ + ChoiceStateEntered, ChoiceStateExited, PassStateEntered, PassStateExited, + SucceedStateEntered, SucceedStateExited, + }, + StateEnteredEventDetails, StateExitedEventDetails, + builders::HistoryEventBuilder, + }, +}; + +use super::{SAMPLES, Samples, latency::validate_history, load::ExhaustionProbes, micros}; + +#[test] +fn test_should_use_exact_nearest_ranks_and_preserve_sample_order() -> Result<()> { + let original: Vec = (1..=100).rev().collect(); + let samples = Samples::new(original.clone())?; + assert_eq!(samples.observed_micros, original); + assert_eq!( + (samples.p50_micros, samples.p95_micros, samples.max_micros), + (50, 95, 100) + ); + assert!(Samples::new(vec![1; 99]).is_err()); + assert!(Samples::new(vec![1; 101]).is_err()); + Ok(()) +} +#[test] +fn test_should_round_up_and_enforce_exact_p95_budget() -> Result<()> { + assert_eq!(micros(Duration::from_nanos(1))?, 1); + assert_eq!(micros(Duration::from_nanos(999))?, 1); + assert_eq!(micros(Duration::from_nanos(1000))?, 1); + assert_eq!(micros(Duration::from_nanos(1001))?, 2); + assert!(micros(Duration::MAX).is_err()); + let mut values = vec![100_000; SAMPLES]; + Samples::new(values.clone())?.check_p95(100_000)?; + for value in values.iter_mut().take(5) { + *value = 100_001; + } + Samples::new(values.clone())?.check_p95(100_000)?; + *values + .get_mut(5) + .ok_or_else(|| anyhow::anyhow!("missing sixth sample"))? = 100_001; + assert!(Samples::new(values)?.check_p95(100_000).is_err()); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_ready_results_at_the_expired_observation_boundary() -> Result<()> { + let error = super::bounded(async { + tokio::time::advance(super::OPERATION_BUDGET).await; + Ok::<_, anyhow::Error>(()) + }) + .await + .err() + .ok_or_else(|| anyhow::anyhow!("accepted late ready result"))?; + assert!(error.to_string().contains("late workload result")); + super::bounded(async { Ok::<_, anyhow::Error>(()) }).await?; + Ok(()) +} + +#[test] +fn test_should_reject_exhaustion_readiness_at_the_two_second_boundary() -> Result<()> { + for worst in [1_999_999, 2_000_000, 2_000_001] { + let mut measured = ExhaustionProbes::default(); + measured.record(&ExecutionStatus::Running, &ExecutionStatus::Running, worst)?; + assert_eq!(measured.check().is_ok(), worst < 2_000_000); + } + Ok(()) +} + +fn event(kind: HistoryEventType, history: &[HistoryEvent]) -> Result { + let previous = i64::try_from(history.len())?; + Ok(HistoryEvent::builder() + .timestamp(DateTime::from_secs(0)) + .r#type(kind) + .id(previous.checked_add(1).context("event ID overflow")?) + .previous_event_id(if history.len() == 1 { 0 } else { previous })) +} +fn complete_history() -> Result> { + let mut events = Vec::new(); + events.push(event(HistoryEventType::ExecutionStarted, &events)?.build()?); + let alternating = [ + (PassStateEntered, PassStateExited), + (ChoiceStateEntered, ChoiceStateExited), + ]; + let states = alternating.into_iter().cycle().take(48).chain([ + (PassStateEntered, PassStateExited), + (SucceedStateEntered, SucceedStateExited), + ]); + for (index, (entered, exited)) in states.enumerate() { + let name = format!("S{index:02}"); + events.push( + event(entered, &events)? + .state_entered_event_details( + StateEnteredEventDetails::builder().name(&name).build()?, + ) + .build()?, + ); + events.push( + event(exited, &events)? + .state_exited_event_details(StateExitedEventDetails::builder().name(name).build()?) + .build()?, + ); + } + events.push(event(HistoryEventType::ExecutionSucceeded, &events)?.build()?); + Ok(events) +} +#[test] +fn test_should_reject_broken_predecessors_and_wrong_or_missing_state_exits() -> Result<()> { + let complete = complete_history()?; + validate_history(&complete)?; + for mutation in [ + "predecessor", + "root-predecessor", + "missing-exit", + "exit-kind", + "exit-name", + "first", + "last", + ] { + let mut broken = complete.clone(); + match mutation { + "root-predecessor" => { + broken + .get_mut(1) + .context("missing first state entry")? + .previous_event_id = 1; + } + "predecessor" => { + broken + .get_mut(50) + .context("missing causal event")? + .previous_event_id = 0; + } + "missing-exit" => broken.retain(|event| event.id() != 3), + "exit-kind" => { + broken.get_mut(2).context("missing exit")?.r#type = + HistoryEventType::ChoiceStateExited; + } + "exit-name" => { + broken + .get_mut(2) + .context("missing exit")? + .state_exited_event_details + .as_mut() + .context("missing exit details")? + .name = "Wrong".to_owned(); + } + "first" => { + broken.first_mut().context("missing first")?.r#type = + HistoryEventType::ExecutionFailed; + } + "last" => { + broken.last_mut().context("missing last")?.r#type = + HistoryEventType::ExecutionFailed; + } + _ => anyhow::bail!("unknown mutation"), + } + assert!(validate_history(&broken).is_err(), "accepted {mutation}"); + } + Ok(()) +} + +#[test] +fn test_should_refuse_post_terminal_probes_as_live_load_evidence() -> Result<()> { + let mut probes = ExhaustionProbes::default(); + probes.record(&ExecutionStatus::Failed, &ExecutionStatus::Failed, 1)?; + assert!(probes.check().is_err()); + probes.record(&ExecutionStatus::Running, &ExecutionStatus::Failed, 1)?; + assert!(probes.check().is_err()); + probes.record(&ExecutionStatus::Running, &ExecutionStatus::Running, 1)?; + probes.check() +} diff --git a/apps/rustack/tests/workflows_sdk/persistence.rs b/apps/rustack/tests/workflows_sdk/persistence.rs new file mode 100644 index 00000000..9a0a7709 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/persistence.rs @@ -0,0 +1,687 @@ +//! Real named binary generations, fresh handle epochs and zero side-effect replay. + +use std::{collections::BTreeMap, fmt, io::Read, path::Path, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use aws_sdk_lambda::{ + Client as Lambda, + primitives::Blob, + types::{Architecture, Environment, FunctionCode, Runtime}, +}; +use aws_sdk_sfn::{ + Client, + error::ProvideErrorMetadata, + types::{ExecutionStatus, HistoryEvent, Tag}, +}; +use rustack_stepfunctions_core::CallbackToken; +use serde_json::{Value, json}; +use tempfile::TempDir; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + task::JoinHandle, + time::{Instant, sleep, timeout_at}, +}; + +use super::{ + process::{SECRET, Server}, + rust::{archive, create, history, lambda, sfn, terminal}, +}; + +const ROLE: &str = "arn:aws:iam::000000000000:role/Workflows"; +const UPDATED_ROLE: &str = "arn:aws:iam::000000000000:role/Updated"; + +#[derive(Debug)] +struct EvidenceDirectory { + directory: Option, +} +impl EvidenceDirectory { + fn path(&self) -> Result<&Path> { + Ok(self + .directory + .as_ref() + .context("missing owned restart directory")? + .path()) + } +} +impl Drop for EvidenceDirectory { + fn drop(&mut self) { + if let Some(directory) = self.directory.take() { + eprintln!( + "restart evidence retained at {}", + directory.keep().display() + ); + } + } +} +#[derive(Debug)] +struct Collector { + port: u16, + join: Option>>, +} +impl Collector { + async fn start() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let join = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await?; + let mut headers = Vec::new(); + while !headers.ends_with(b"\r\n\r\n") { + ensure!(headers.len() < 4096, "oversized local collector header"); + headers.push(stream.read_u8().await?); + } + let headers = String::from_utf8(headers)?; + let length = headers + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .context("collector request length")? + .1 + .trim() + .parse::()?; + ensure!(length == 64, "invalid local token length"); + let mut body = vec![0; length]; + stream.read_exact(&mut body).await?; + let token = CallbackToken::try_from(String::from_utf8(body)?)?; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + Ok(token) + }); + Ok(Self { + port, + join: Some(join), + }) + } + async fn token(&mut self) -> Result { + let result = timeout_at( + deadline(), + self.join.as_mut().context("collector already consumed")?, + ) + .await?; + self.join = None; + result? + } + async fn close(&mut self) -> Result<()> { + if let Some(join) = &mut self.join { + join.abort(); + let result = timeout_at(deadline(), join).await?; + self.join = None; + if let Err(error) = result { + ensure!(error.is_cancelled(), "collector worker panicked"); + } + } + Ok(()) + } +} +impl Drop for Collector { + fn drop(&mut self) { + if let Some(join) = &self.join { + join.abort(); + } + } +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(40) +} +async fn deploy(server: &Server, root: &Path, collector: &Collector) -> Result<()> { + let bytes = fs::read(server.bootstrap()?).await?; + let zip = tokio::task::spawn_blocking(move || archive(&bytes)).await??; + fs::create_dir_all(root.join("invocations")).await?; + let client: Lambda = lambda(server); + for name in ["RestartCompute", "RestartToken", "RestartRetry"] { + let callback = name == "RestartToken"; + let mut environment = Environment::builder() + .variables("WORKFLOW_MODE", "1") + .variables( + "PROCESS_MARKER_DIR", + server.markers.to_str().context("PID marker path")?, + ); + if callback { + environment = environment.variables("WORKFLOW_TOKEN_PORT", collector.port.to_string()); + } else { + environment = environment.variables( + "WORKFLOW_INVOCATION_DIR", + root.join("invocations") + .to_str() + .context("invocation marker path")?, + ); + } + if name == "RestartRetry" { + environment = environment.variables("WORKFLOW_FAIL_ONCE", "1"); + } + client + .create_function() + .function_name(name) + .role(ROLE) + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .architectures(if cfg!(target_arch = "aarch64") { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .timeout(10) + .environment(environment.build()) + .code( + FunctionCode::builder() + .zip_file(Blob::new(zip.clone())) + .build(), + ) + .send() + .await?; + } + Ok(()) +} +fn pass(value: i64) -> Value { + json!({"StartAt":"P","States":{"P":{"Type":"Pass","Result":value,"End":true}}}) +} +async fn start(client: &Client, machine: &str, name: &str, input: &str) -> Result { + Ok(client + .start_execution() + .state_machine_arn(machine) + .name(name) + .input(input) + .send() + .await? + .execution_arn() + .to_owned()) +} +struct Saved { + pure: String, + pure_run: String, + compute: String, + compute_run: String, + waiting: String, + callback: String, + interrupted: Vec, + old_page: String, + token: CallbackToken, + output: String, + events: Vec, + definition: String, + current_definition: String, +} +impl fmt::Debug for Saved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SavedRestartEvidence()") + } +} +async fn seed(server: &Server, root: &Path, collector: &mut Collector) -> Result { + deploy(server, root, collector).await?; + let client = sfn(server, SECRET); + let definition = pass(7).to_string(); + let pure = create(&client, "RestartPure", &pass(7)).await?; + let pure_run = start(&client, &pure, "old", " {\"raw\":1}\n").await?; + ensure!( + terminal(&client, &pure_run).await?.output() == Some("7"), + "pure source did not execute" + ); + let current_definition = pass(8).to_string(); + client + .update_state_machine() + .state_machine_arn(&pure) + .definition(¤t_definition) + .role_arn(UPDATED_ROLE) + .send() + .await?; + client + .tag_resource() + .resource_arn(&pure) + .tags(Tag::builder().key("purpose").value("snapshot").build()) + .send() + .await?; + let compute = create(&client, "RestartCompute", &json!({"StartAt":"T","States":{"T":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload.$":"$"},"End":true}}})).await?; + let compute_run = start(&client, &compute, "old", "{\"n\":21}").await?; + let completed = terminal(&client, &compute_run).await?; + let output = completed + .output() + .context("missing source Native output")? + .to_owned(); + ensure!( + serde_json::from_str::(&output)?.pointer("/Payload/computed") == Some(&json!(42)), + "source Native computation missing" + ); + let events = history(&client, &compute_run, false, true).await?; + let wait = create( + &client, + "RestartWait", + &json!({"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":600,"Next":"After"},"After":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload":{"n":999}},"End":true}}}), + ) + .await?; + let waiting = start(&client, &wait, "pending", "{}").await?; + entered(&client, &waiting, "WaitStateEntered").await?; + let callback_machine = create(&client, "RestartCallback", &json!({"StartAt":"T","States":{"T":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken","Parameters":{"FunctionName":"RestartToken","Payload":{"token.$":"$$.Task.Token","n":21}},"Next":"After"},"After":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload":{"n":999}},"End":true}}})).await?; + let callback = start(&client, &callback_machine, "pending", "{}").await?; + let token = collector.token().await?; + client + .send_task_heartbeat() + .task_token(token.expose()) + .send() + .await?; + let interrupted = structured_pending(&client, &wait, &waiting).await?; + let page = client.list_state_machines().max_results(1).send().await?; + let old_page = page + .next_token() + .context("missing source page handle")? + .to_owned(); + Ok(Saved { + pure, + pure_run, + compute, + compute_run, + waiting, + callback, + interrupted, + old_page, + token, + output, + events, + definition, + current_definition, + }) +} +async fn structured_pending(client: &Client, wait: &str, waiting: &str) -> Result> { + let retry = create(client, "RestartRetry", &json!({"StartAt":"T","States":{ + "T":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke", + "Parameters":{"FunctionName":"RestartRetry","Payload":{"n":21}}, + "Retry":[{"ErrorEquals":["States.ALL"],"IntervalSeconds":600,"MaxAttempts":2}],"Next":"After"},"After":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload":{"n":999}},"End":true}}})).await?; + let retry_run = start(client, &retry, "pending", "{}").await?; + entered(client, &retry_run, "TaskFailed").await?; + let map = create(client, "RestartMap", &json!({"StartAt":"M","States":{ + "M":{"Type":"Map","ItemsPath":"$","Iterator":{"StartAt":"W","States":{ + "W":{"Type":"Wait","Seconds":600,"Next":"After"},"After":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload":{"n":999}},"End":true}}},"End":true}}})).await?; + let map_run = start(client, &map, "pending", "[1,2]").await?; + entered(client, &map_run, "MapIterationStarted").await?; + entered(client, &map_run, "WaitStateEntered").await?; + let nested = create(client, "RestartSync", &json!({"StartAt":"T","States":{ + "T":{"Type":"Task","Resource":"arn:aws:states:::states:startExecution.sync:2", + "Parameters":{"StateMachineArn":wait,"Input":{}},"Next":"After"},"After":{"Type":"Task","Resource":"arn:aws:states:::lambda:invoke","Parameters":{"FunctionName":"RestartCompute","Payload":{"n":999}},"End":true}}})).await?; + let nested_run = start(client, &nested, "pending", "{}").await?; + // The child registry, not a timer, proves the owned child was created. + let child_run = timeout_at(deadline(), async { + loop { + let children = client + .list_executions() + .state_machine_arn(wait) + .send() + .await?; + if let Some(child) = children + .executions() + .iter() + .find(|entry| entry.execution_arn() != waiting) + { + let arn = child.execution_arn().to_owned(); + entered(client, &arn, "WaitStateEntered").await?; + return Ok::<_, anyhow::Error>(arn); + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .context("owned nested child did not enter")??; + Ok(vec![retry_run, map_run, nested_run, child_run]) +} +async fn entered(client: &Client, arn: &str, kind: &str) -> Result<()> { + timeout_at(deadline(), async { + loop { + let events = history(client, arn, false, true).await?; + if events.iter().any(|event| event.r#type().as_str() == kind) { + return Ok::<(), anyhow::Error>(()); + } + ensure!( + events.last().is_none_or(|event| !matches!( + event.r#type().as_str(), + "ExecutionSucceeded" + | "ExecutionFailed" + | "ExecutionAborted" + | "ExecutionTimedOut" + )), + "execution terminated before entering {kind}" + ); + sleep(Duration::from_millis(25)).await; + } + }) + .await + .context("pending shutdown state did not enter")? +} +async fn markers(root: &Path) -> Result>> { + let mut entries = fs::read_dir(root.join("invocations")).await?; + let mut markers = BTreeMap::new(); + while let Some(entry) = entries.next_entry().await? { + ensure!( + markers.len() < 8 + && entry.file_type().await?.is_file() + && entry.metadata().await?.len() <= 1024, + "invalid invocation evidence" + ); + markers.insert( + entry.file_name().to_string_lossy().into_owned(), + fs::read(entry.path()).await?, + ); + } + Ok(markers) +} +async fn old_state(server: &Server, saved: &Saved) -> Result<()> { + let client = sfn(server, SECRET); + let old = client + .describe_execution() + .execution_arn(&saved.compute_run) + .send() + .await?; + ensure!( + old.output() == Some(saved.output.as_str()), + "old Native result changed or replayed" + ); + ensure!( + history(&client, &saved.compute_run, false, true).await? == saved.events, + "old full Native history changed" + ); + let pure = client + .describe_execution() + .execution_arn(&saved.pure_run) + .send() + .await?; + ensure!( + pure.input() == Some(" {\"raw\":1}\n") && pure.output() == Some("7"), + "original pure input/output changed" + ); + let captured = client + .describe_state_machine_for_execution() + .execution_arn(&saved.pure_run) + .send() + .await?; + ensure!( + captured.definition() == saved.definition && captured.role_arn() == ROLE, + "captured revision/role changed" + ); + let current = client + .describe_state_machine() + .state_machine_arn(&saved.pure) + .send() + .await?; + ensure!( + current.definition() == saved.current_definition && current.role_arn() == UPDATED_ROLE, + "current revision/role not restored" + ); + let tags = client + .list_tags_for_resource() + .resource_arn(&saved.pure) + .send() + .await?; + ensure!( + tags.tags() + .iter() + .any(|tag| tag.key() == Some("purpose") && tag.value() == Some("snapshot")), + "tags not restored" + ); + for arn in [&saved.waiting, &saved.callback] + .into_iter() + .chain(saved.interrupted.iter()) + { + let stopped = client + .describe_execution() + .execution_arn(arn) + .send() + .await?; + ensure!( + stopped.status() == &ExecutionStatus::Aborted + && stopped.error() == Some("Rustack.Shutdown"), + "shutdown did not persist terminal interruption" + ); + } + stale_handles(&client, saved).await +} +async fn stale_handles(client: &Client, saved: &Saved) -> Result<()> { + let duplicate = client + .start_execution() + .state_machine_arn(&saved.compute) + .name("old") + .input("{\"n\":21}") + .send() + .await + .err() + .context("lost closed execution name")?; + ensure!( + duplicate + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("ExecutionAlreadyExists"), + "closed name reservation changed" + ); + let page = client + .list_state_machines() + .max_results(1) + .next_token(&saved.old_page) + .send() + .await + .err() + .context("old pagination handle survived restart")?; + ensure!( + page.as_service_error().and_then(ProvideErrorMetadata::code) == Some("InvalidToken"), + "old pagination handle classification changed" + ); + let callback = client + .send_task_heartbeat() + .task_token(saved.token.expose()) + .send() + .await + .err() + .context("old callback token survived restart")?; + ensure!( + callback + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("InvalidToken"), + "old callback root epoch was restored" + ); + Ok(()) +} +async fn assert_no_token_archive(root: &Path, saved: &Saved) -> Result<()> { + for relative in [ + "services/stepfunctions/meta.ss.zst", + "services/stepfunctions/data.ss.zst", + "services/lambda/meta.ss.zst", + ] { + let compressed = fs::read(root.join("snapshots/restart").join(relative)).await?; + let mut decoded = tokio::task::spawn_blocking(move || -> Result> { + let decoder = zstd::stream::read::Decoder::new(compressed.as_slice())?; + let mut bytes = Vec::new(); + decoder.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes)?; + Ok(bytes) + }) + .await??; + ensure!( + decoded.len() <= 16 * 1024 * 1024, + "oversized fixture archive" + ); + ensure!( + !decoded + .windows(saved.token.expose().len()) + .any(|window| window == saved.token.expose().as_bytes()), + "callback credential persisted" + ); + decoded.fill(0); + } + Ok(()) +} +async fn reject_corruption(root: &Path, saved: &Saved) -> Result<()> { + let meta = root.join("snapshots/restart/services/stepfunctions/meta.ss.zst"); + let original = fs::read(&meta).await?; + let mut corrupt = original.clone(); + let first = corrupt.first_mut().context("missing workflow archive")?; + *first ^= 1; + fs::write(&meta, &corrupt).await?; + let manifest = fs::read(root.join("snapshots/restart/manifest.ss.zst")).await?; + Server::reject_named(root).await?; + ensure!( + fs::read(&meta).await? == corrupt + && fs::read(root.join("snapshots/restart/manifest.ss.zst")).await? == manifest, + "failed startup replaced the generation" + ); + fs::write(&meta, original).await?; + assert_no_token_archive(root, saved).await?; + Ok(()) +} + +async fn graceful_snapshot( + source: &mut Server, + root: &Path, + saved: &Saved, +) -> Result>> { + let client = sfn(source, SECRET); + for arn in [&saved.waiting, &saved.callback] + .into_iter() + .chain(saved.interrupted.iter()) + { + ensure!( + client + .describe_execution() + .execution_arn(arn) + .send() + .await? + .status() + == &ExecutionStatus::Running, + "shutdown fixture advanced before cutoff" + ); + } + source.shutdown(deadline()).await?; + assert_no_token_archive(root, saved).await?; + let before = markers(root).await?; + ensure!(before.len() == 2, "source invocation marker count"); + Ok(before) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires actual host bootstrap; run make test-workflows-persistence"] +async fn test_should_restart_named_native_snapshots_without_replay_and_lose_only_unsaved_work() +-> Result<()> { + let mut evidence = EvidenceDirectory { + directory: Some(tempfile::tempdir()?), + }; + let root = evidence.path()?.to_owned(); + let mut collector = Collector::start().await?; + let mut source = Server::start_named(&root).await?; + let mut target = None; + let mut crashing = None; + let mut final_server = None; + let result = async { + let saved = seed(&source, &root, &mut collector).await?; + let before = graceful_snapshot(&mut source, &root, &saved).await?; + target = Some(Server::start_named(&root).await?); + let active = target.as_mut().context("missing restored server")?; + old_state(active, &saved).await?; + ensure!( + markers(&root).await? == before, + "restore replayed a physical Lambda call" + ); + let client = sfn(active, SECRET); + let next = start(&client, &saved.compute, "after", "{\"n\":22}").await?; + let outcome = terminal(&client, &next).await?; + ensure!( + serde_json::from_str::(outcome.output().context("missing new result")?)? + .pointer("/Payload/computed") + == Some(&json!(44)), + "new restored Lambda execution failed" + ); + ensure!( + markers(&root).await?.len() == before.len() + 1, + "new Lambda invocation was not observed" + ); + active.shutdown(deadline()).await?; + let committed = fs::read(root.join("snapshots/restart/manifest.ss.zst")).await?; + crashing = Some(Server::start_named(&root).await?); + let active = crashing.as_mut().context("missing crash server")?; + let client = sfn(active, SECRET); + let unsaved = create(&client, "Unsaved", &pass(99)).await?; + terminal(&client, &start(&client, &unsaved, "unsaved", "{}").await?).await?; + active.crash(deadline()).await?; + ensure!( + fs::read(root.join("snapshots/restart/manifest.ss.zst")).await? == committed, + "SIGKILL wrote a new generation" + ); + final_server = Some(Server::start_named(&root).await?); + let active = final_server.as_mut().context("missing final server")?; + old_state(active, &saved).await?; + let missing = sfn(active, SECRET) + .describe_state_machine() + .state_machine_arn(unsaved) + .send() + .await + .err() + .context("unsaved work survived SIGKILL")?; + ensure!( + missing + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("StateMachineDoesNotExist"), + "wrong unsaved resource error" + ); + ensure!( + markers(&root).await?.len() == before.len() + 1, + "restart replayed Lambda work" + ); + active.shutdown(deadline()).await?; + reject_corruption(&root, &saved).await?; + Ok(()) + } + .await; + let one = source.shutdown(deadline()).await; + let two = if let Some(server) = &mut target { + server.shutdown(deadline()).await.map(|_| ()) + } else { + Ok(()) + }; + let three = if let Some(server) = &mut crashing { + server.shutdown(deadline()).await.map(|_| ()) + } else { + Ok(()) + }; + let four = if let Some(server) = &mut final_server { + server.shutdown(deadline()).await.map(|_| ()) + } else { + Ok(()) + }; + let collected = collector.close().await; + one?; + two?; + three?; + four?; + collected?; + result?; + drop(evidence.directory.take()); + Ok(()) +} + +#[tokio::test] +async fn test_should_close_failed_token_collector_without_repolling_completed_task() -> Result<()> { + for early_eof in [false, true] { + let mut collector = Collector::start().await?; + let result = async { + let mut client = tokio::net::TcpStream::connect(("127.0.0.1", collector.port)).await?; + client + .write_all(b"POST /token HTTP/1.1\r\nHost: localhost\r\nContent-Length: 64\r\n\r\n") + .await?; + client + .write_all(if early_eof { b"early EOF" } else { &[b'z'; 64] }) + .await?; + client.shutdown().await?; + ensure!( + collector.token().await.is_err(), + "malformed token was accepted" + ); + ensure!( + collector.join.is_none(), + "completed failure join was retained" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + collector.close().await?; + result?; + collector.close().await?; + } + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk/process.rs b/apps/rustack/tests/workflows_sdk/process.rs new file mode 100644 index 00000000..85bc2c39 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/process.rs @@ -0,0 +1,485 @@ +//! Retained real server ownership; all SDK observers are separate from process life. +#![warn( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::{ + env, + ffi::OsString, + os::unix::process::ExitStatusExt, + path::{Path, PathBuf}, + process::Stdio, + sync::Arc, + time::Duration, +}; + +use anyhow::{Context, Result, bail, ensure}; +use tempfile::TempDir; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + process::{Child, Command}, + sync::watch, + task::JoinHandle, + time::{Instant, sleep, timeout, timeout_at}, +}; + +use super::runner::Runner; + +pub(super) const ACCESS: &str = "local-key"; +pub(super) const SECRET: &str = "local-secret"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Stop { + Running, + Graceful, + Kill, +} + +#[derive(Debug)] +struct Resources { + child: Child, + directory: Option, + markers: PathBuf, + cleanup_proven: bool, + expect_startup_failure: bool, +} +#[derive(Debug)] +struct Owner { + state: Option, + stop: watch::Receiver, +} +#[derive(Debug, Clone, Copy)] +enum Profile { + Semantic, + Performance, + Pure, +} +impl Profile { + fn settings(self, dispatch: usize) -> String { + let limits = match self { + Self::Pure => { + return include_str!("../../../../examples/workflows/rustack.yaml").to_owned(); + } + Self::Semantic => { + "workflowTransitions: 80\n workflowHistoryBytes: 1048576\n \ + workflowTotalHistoryBytes: 8388608" + } + Self::Performance => { + "workflowOpenExecutions: 1\n workflowTransitions: 1000000\n \ + workflowHistoryBytes: 33554432\n workflowTotalHistoryBytes: 134217728" + } + }; + format!( + "budgets:\n {limits}\n workflowDrainSeconds: 0\n workflowDispatches: {dispatch}\n \ + workflowCallbackTokens: 2\n workflowCallbackTombstones: 64\n \ + workflowCallbackCommands: 8\n workflowCallbackWorkers: 1\n" + ) + } +} + +#[derive(Debug)] +enum ExecutionFixture { + Pure, + Native(PathBuf), +} +impl ExecutionFixture { + fn services(&self) -> &'static str { + match self { + Self::Pure => "stepfunctions", + Self::Native(_) => "lambda,stepfunctions", + } + } + fn executor(&self) -> &'static str { + match self { + Self::Pure => "disabled", + Self::Native(_) => "native", + } + } +} + +#[derive(Debug)] +pub(super) struct Server { + pub pid: u32, + pub endpoint: String, + pub markers: PathBuf, + execution: ExecutionFixture, + stop: watch::Sender, + root: Option>>, + joined: Option>>, +} +impl Server { + pub async fn capabilities(&self) -> Result { + let address = self + .endpoint + .strip_prefix("http://") + .context("owned server endpoint")?; + timeout(Duration::from_secs(2),async { + let mut stream=TcpStream::connect(address).await?; + stream.write_all(b"GET /_rustack/capabilities HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n").await?; + let mut response=Vec::new();stream.take(16_385).read_to_end(&mut response).await?; + ensure!(response.len()<=16_384,"oversized capability response"); + let text=std::str::from_utf8(&response)?; + ensure!(text.starts_with("HTTP/1.1 200"),"capability response failed"); + let (_,body)=text.split_once("\r\n\r\n").context("capability envelope")?; + Ok(serde_json::from_str(body)?) + }).await.context("capability observer expired")? + } + pub async fn start() -> Result { + Self::start_with_dispatch(32).await + } + pub async fn start_with_dispatch(dispatch: usize) -> Result { + Self::start_options(dispatch, None, false, Profile::Semantic).await + } + pub fn bootstrap(&self) -> Result<&Path> { + match &self.execution { + ExecutionFixture::Native(path) => Ok(path), + ExecutionFixture::Pure => bail!("pure fixture has no Lambda bootstrap"), + } + } + pub async fn start_pure() -> Result { + Self::start_options(1, None, false, Profile::Pure).await + } + pub async fn start_performance() -> Result { + Self::start_options(1, None, false, Profile::Performance).await + } + pub async fn readiness(&self) -> Result<()> { + let address = self + .endpoint + .strip_prefix("http://") + .context("owned server endpoint")?; + let began = Instant::now(); + let result = timeout(Duration::from_secs(2), health(address)) + .await + .context("readiness observation expired")?; + ensure!( + began.elapsed() < Duration::from_secs(2), + "late readiness result" + ); + result + } + pub async fn start_named(root: &Path) -> Result { + Self::start_options(32, Some(root), false, Profile::Semantic).await + } + pub async fn reject_named(root: &Path) -> Result<()> { + let mut server = Self::start_options(32, Some(root), true, Profile::Semantic).await?; + server + .observe(Instant::now() + Duration::from_secs(40)) + .await?; + let address = server + .endpoint + .strip_prefix("http://") + .context("owned endpoint")?; + ensure!( + TcpStream::connect(address).await.is_err(), + "rejected startup opened a listener" + ); + Ok(()) + } + async fn start_options( + dispatch: usize, + named: Option<&Path>, + expect_startup_failure: bool, + profile: Profile, + ) -> Result { + ensure!( + (1..=32).contains(&dispatch), + "invalid fixture dispatch capacity" + ); + let execution = if matches!(profile, Profile::Pure | Profile::Performance) { + ExecutionFixture::Pure + } else { + let bootstrap = PathBuf::from( + env::var_os("RUSTACK_WORKFLOWS_BOOTSTRAP") + .context("run make test-stepfunctions-sdk")?, + ); + ensure!( + (1..=32 * 1024 * 1024).contains(&fs::metadata(&bootstrap).await?.len()), + "invalid bootstrap size" + ); + ExecutionFixture::Native(bootstrap) + }; + let directory = tempfile::tempdir()?; + let fixture_root = named.unwrap_or_else(|| directory.path()); + let markers = fixture_root.join("processes"); + for path in [ + &markers, + &fixture_root.join("gates"), + &fixture_root.join("callbacks"), + &fixture_root.join("callback-gates"), + ] { + fs::create_dir_all(path).await?; + } + let config = directory.path().join("rustack.yaml"); + fs::write(&config, profile.settings(dispatch)).await?; + let reservation = TcpListener::bind("127.0.0.1:0").await?; + let address = reservation.local_addr()?; + drop(reservation); + let mut command = Command::new(env!("CARGO_BIN_EXE_rustack")); + command.env_clear(); + if let Some(named) = named { + command.args(["--snapshot", "restart"]); + command.env("RUSTACK_SNAPSHOT_DIR", named.join("snapshots")); + } + let child = command + .env("TMPDIR", directory.path()) + .env("RUSTACK_CONFIG", &config) + .env("GATEWAY_LISTEN", address.to_string()) + .env("SERVICES", execution.services()) + .env("LAMBDA_EXECUTOR", execution.executor()) + .env("LAMBDA_SKIP_SIGNATURE_VALIDATION", "false") + .env("STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION", "false") + .env("ACCESS_KEY", ACCESS) + .env("SECRET_KEY", SECRET) + .env("LOG_LEVEL", "error") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(if expect_startup_failure { + Stdio::piped() + } else { + Stdio::null() + }) + .kill_on_drop(true) + .spawn() + .context("spawn actual Rustack binary")?; + let pid = child.id().context("owned server PID")?; + let (stop, receiver) = watch::channel(Stop::Running); + let owner = Owner { + state: Some(Resources { + child, + directory: Some(directory), + markers: markers.clone(), + cleanup_proven: false, + expect_startup_failure, + }), + stop: receiver, + }; + let mut server = Self { + pid, + endpoint: format!("http://{address}"), + markers, + execution, + stop, + root: Some(tokio::spawn(owner.run())), + joined: None, + }; + if expect_startup_failure { + return Ok(server); + } + let ready = server.await_ready(&address.to_string()).await; + if let Err(error) = ready { + let cleanup = server + .shutdown(Instant::now() + Duration::from_secs(40)) + .await; + cleanup?; + return Err(error); + } + Ok(server) + } + async fn await_ready(&self, address: &str) -> Result<()> { + timeout(Duration::from_secs(20), async { + loop { + ensure!( + !self.root.as_ref().is_none_or(JoinHandle::is_finished), + "server exited before readiness" + ); + if let Ok(Ok(())) = timeout(Duration::from_secs(1), health(address)).await { + return Ok(()); + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .context("server startup deadline")? + } + pub async fn shutdown(&mut self, deadline: Instant) -> Result { + if *self.stop.borrow() == Stop::Running { + self.stop.send_replace(Stop::Graceful); + } + self.observe(deadline).await + } + pub async fn crash(&mut self, deadline: Instant) -> Result { + self.stop.send_replace(Stop::Kill); + self.observe(deadline).await + } + async fn observe(&mut self, deadline: Instant) -> Result { + if self.joined.is_none() { + let root = self.root.as_mut().context("missing retained server root")?; + let result = timeout_at(deadline, root) + .await + .context("server shutdown observer expired; root retained")?; + self.root = None; + self.joined = Some(match result { + Ok(result) => result.map_err(Arc::new), + Err(error) => Err(Arc::new(error.into())), + }); + } + match &self.joined { + Some(Ok(observed)) => Ok(*observed), + Some(Err(error)) => bail!("server cleanup failed: {error:#}"), + None => bail!("missing server outcome"), + } + } +} +impl Drop for Server { + fn drop(&mut self) { + if *self.stop.borrow() == Stop::Running { + self.stop.send_replace(Stop::Graceful); + } + // Intentional independent root: an SDK/cleanup observer does not own it. + // Its Drop guard preserves files if the Tokio runtime itself is lost. + if let Some(root) = self.root.take() { + drop(root); + } + } +} +impl Owner { + async fn run(mut self) -> Result { + let state = self.state.as_mut().context("missing owned server state")?; + let mut gate_failure = None; + let status = tokio::select! { + result = state.child.wait() => { + let status = result?; + ensure!(*self.stop.borrow() != Stop::Running || state.expect_startup_failure, "server exited without requested shutdown"); + status + }, + _ = self.stop.changed() => { + if *self.stop.borrow() == Stop::Graceful { + if let Err(error)=release_fixture_gates(&state.markers).await {gate_failure=Some(error);} + } + let pid = state.child.id().context("missing owned server PID")?; + let signal=Runner::spawn(Path::new("/bin/kill"), &[OsString::from(if *self.stop.borrow() == Stop::Kill {"-KILL"} else {"-TERM"}), OsString::from(pid.to_string())])? + .wait(Instant::now()+Duration::from_secs(5)).await?; + ensure!(!signal.forced && signal.status.success(), "cannot signal owned server"); + if let Ok(status) = timeout(Duration::from_secs(35), state.child.wait()).await { status? } else { + state.child.start_kill()?; + let _reaped = timeout(Duration::from_secs(5), state.child.wait()).await; + bail!("server required forced termination; ownership evidence retained"); + } + } + }; + if state.expect_startup_failure { + ensure!( + !status.success(), + "invalid archive startup unexpectedly succeeded" + ); + let stderr = state + .child + .stderr + .take() + .context("missing startup rejection diagnostic")?; + let mut diagnostic = Vec::new(); + timeout( + Duration::from_secs(2), + stderr.take(16_385).read_to_end(&mut diagnostic), + ) + .await??; + ensure!( + diagnostic.len() <= 16_384 + && String::from_utf8_lossy(&diagnostic).contains("snapshot transaction failed"), + "startup did not reject the workflow snapshot" + ); + } else if *self.stop.borrow() == Stop::Kill { + ensure!( + status.signal() == Some(9), + "SIGKILL did not interrupt the server" + ); + } else { + ensure!(status.success(), "server coordinated shutdown failed"); + } + let observed = timeout(Duration::from_secs(10), verify_absent(&state.markers)) + .await + .context("PID evidence deadline")??; + state.cleanup_proven = true; + if let Some(error) = gate_failure { + return Err(error.context("fixture gates failed after coordinated process cleanup")); + } + self.state = None; + Ok(observed) + } +} +async fn release_fixture_gates(markers: &Path) -> Result<()> { + let root = markers.parent().context("owned fixture directory")?; + timeout( + Duration::from_secs(2), + fs::write(root.join("callback-gates/all.release"), b"cleanup"), + ) + .await??; + Ok(()) +} +impl Drop for Owner { + fn drop(&mut self) { + if let Some(state) = &mut self.state { + if let Some(directory) = state.directory.take() { + let path = directory.keep(); + let reason = if state.cleanup_proven { + "SDK qualification failed after proven process cleanup" + } else { + "SDK cleanup was not proven" + }; + eprintln!("{reason}; retained evidence at {}", path.display()); + } + } + } +} +async fn verify_absent(markers: &Path) -> Result { + let mut entries = fs::read_dir(markers).await?; + let mut count = 0; + while let Some(entry) = entries.next_entry().await? { + count += 1; + ensure!( + count <= 64 && entry.file_type().await?.is_file(), + "invalid PID evidence collection" + ); + ensure!( + entry.metadata().await?.len() <= 11, + "oversized PID evidence" + ); + let raw = fs::read_to_string(entry.path()).await?; + let pid: u32 = raw.parse()?; + ensure!( + pid > 1 && entry.file_name() == format!("pid-{pid}").as_str(), + "invalid PID identity" + ); + assert_absent(pid).await?; + } + Ok(count) +} +async fn health(address: &str) -> Result<()> { + let mut stream = TcpStream::connect(address).await?; + stream + .write_all(b"GET /_health/ready HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await?; + let mut response = Vec::new(); + stream.take(16_385).read_to_end(&mut response).await?; + ensure!(response.len() <= 16_384, "oversized health response"); + let text = std::str::from_utf8(&response)?; + let (_, body) = text.split_once("\r\n\r\n").context("health envelope")?; + let value: serde_json::Value = serde_json::from_str(body)?; + ensure!( + text.starts_with("HTTP/1.1 200") + && value.get("ready") == Some(&serde_json::Value::Bool(true)), + "server not ready" + ); + Ok(()) +} + +pub(super) async fn assert_absent(pid: u32) -> Result<()> { + let probe = Runner::spawn( + Path::new("/bin/kill"), + &[OsString::from("-0"), OsString::from(pid.to_string())], + )? + .wait(Instant::now() + Duration::from_secs(3)) + .await?; + ensure!( + !probe.forced + && !probe.status.success() + && String::from_utf8_lossy(&probe.stderr).contains("No such process"), + "fixture PID absence was not proven" + ); + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk/runner.rs b/apps/rustack/tests/workflows_sdk/runner.rs new file mode 100644 index 00000000..882cb693 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/runner.rs @@ -0,0 +1,136 @@ +//! Direct child ownership with bounded output and joined timeout cleanup. + +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, + process::{ExitStatus, Stdio}, +}; + +use anyhow::{Context, Result, ensure}; +use tokio::{ + io::{AsyncRead, AsyncReadExt}, + process::{Child, Command}, + time::{Duration, Instant, timeout, timeout_at}, +}; + +#[derive(Debug)] +pub(super) struct Runner { + child: Child, +} +#[derive(Debug)] +pub(super) struct Report { + pub pid: u32, + pub forced: bool, + pub status: ExitStatus, + pub stdout: Vec, + pub stderr: Vec, +} +pub(super) fn interpreter() -> Result { + let value = PathBuf::from( + env::var_os("RUSTACK_WORKFLOWS_PYTHON") + .context("run make test-stepfunctions-sdk to provision pinned Python")?, + ); + ensure!( + value.is_absolute(), + "missing absolute pinned Python executable" + ); + Ok(value) +} +impl Runner { + pub fn spawn(program: &Path, arguments: &[OsString]) -> Result { + let child = Command::new(program) + .args(arguments) + .env_clear() + .env("PYTHONDONTWRITEBYTECODE", "1") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .context("spawn directly owned fixture process")?; + Ok(Self { child }) + } + pub async fn await_pid(&mut self) -> Result { + self.await_pid_before(Instant::now() + Duration::from_secs(5)) + .await + } + pub async fn await_pid_before(&mut self, deadline: Instant) -> Result { + ensure!( + Instant::now() < deadline, + "PID barrier deadline already expired" + ); + let expected = self.child.id().context("owned Python PID")?; + let observed = timeout_at(deadline, async { + let output = self.child.stdout.as_mut().context("PID barrier stdout")?; + let mut text = Vec::new(); + loop { + let byte = output.read_u8().await?; + if byte == b'\n' { + break; + } + ensure!(text.len() < 16, "oversized PID barrier"); + text.push(byte); + } + let observed: u32 = std::str::from_utf8(&text)?.parse()?; + ensure!( + observed == expected, + "Python PID differs from directly owned child" + ); + Ok(observed) + }) + .await + .context("Python execution barrier deadline")??; + ensure!(Instant::now() < deadline, "late PID barrier observation"); + Ok(observed) + } + pub async fn wait(mut self, deadline: Instant) -> Result { + let pid = self.child.id().context("owned Python PID")?; + let stdout = self.child.stdout.take().context("Python stdout")?; + let stderr = self.child.stderr.take().context("Python stderr")?; + let (status, stdout, stderr) = tokio::join!( + wait_child(&mut self.child, deadline), + capture(stdout, deadline + Duration::from_secs(10)), + capture(stderr, deadline + Duration::from_secs(10)) + ); + let (status, forced) = status?; + Ok(Report { + pid, + forced, + status, + stdout: stdout?, + stderr: stderr?, + }) + } +} +async fn wait_child(child: &mut Child, deadline: Instant) -> Result<(ExitStatus, bool)> { + if let Ok(status) = timeout_at(deadline, child.wait()).await { + return Ok((status?, false)); + } + child + .start_kill() + .context("kill exact owned fixture process after deadline")?; + let status = timeout(Duration::from_secs(5), child.wait()) + .await + .context("fixture process forced reap deadline")??; + Ok((status, true)) +} +async fn capture(mut input: R, deadline: Instant) -> Result> { + let mut output = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let count = timeout_at(deadline, input.read(&mut buffer)) + .await + .context("fixture pipe drain deadline")??; + if count == 0 { + return Ok(output); + } + ensure!( + output.len().saturating_add(count) <= 32768, + "fixture output exceeded32KiB cap" + ); + output.extend_from_slice(buffer.get(..count).context("invalid pipe read count")?); + } +} diff --git a/apps/rustack/tests/workflows_sdk/rust.rs b/apps/rustack/tests/workflows_sdk/rust.rs new file mode 100644 index 00000000..f4b73361 --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/rust.rs @@ -0,0 +1,1055 @@ +//! Ordinary signed Rust SDK requests against the actual executable. + +use std::{ + collections::BTreeSet, + io::{Cursor, Write}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, ensure}; +use aws_credential_types::Credentials; +use aws_sdk_lambda::{ + Client as Lambda, + config::{BehaviorVersion, Region, retry::RetryConfig, timeout::TimeoutConfig}, + primitives::Blob, + types::{Architecture, Environment, FunctionCode, Runtime}, +}; +use aws_sdk_sfn::{ + Client, + error::ProvideErrorMetadata, + operation::{ + describe_execution::DescribeExecutionOutput, + describe_state_machine::DescribeStateMachineOutput, start_execution::StartExecutionOutput, + }, + types::{ExecutionStatus, HistoryEvent, Tag, ValidateStateMachineDefinitionSeverity}, +}; +use serde_json::{Value, json}; +use tokio::{ + fs, + time::{sleep, timeout}, +}; +use uuid::Uuid; +use zip::{ZipWriter, write::SimpleFileOptions}; + +use super::process::{ACCESS, SECRET, Server}; + +#[path = "rust/callback.rs"] +mod callback; +#[path = "rust/nested.rs"] +mod nested; +#[path = "rust/structured.rs"] +mod structured; + +const ROLE: &str = "arn:aws:iam::000000000000:role/Workflows"; +const PREFIX: &str = "arn:aws:states:us-east-1:000000000000:"; +/// Initial root entry has an independent predecessor; later sequential events chain. +pub(super) fn sequential_predecessor(offset: usize) -> Result { + i64::try_from(if offset == 1 { 0 } else { offset }).context("history predecessor overflow") +} + +fn credentials() -> Credentials { + Credentials::new(ACCESS, SECRET, None, None, "workflow-local") +} +fn timeouts() -> TimeoutConfig { + TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(2)) + .read_timeout(Duration::from_secs(15)) + .operation_timeout(Duration::from_secs(15)) + .build() +} +pub(super) fn sfn(server: &Server, secret: &str) -> Client { + Client::from_conf( + aws_sdk_sfn::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(&server.endpoint) + .credentials_provider(Credentials::new( + ACCESS, + secret, + None, + None, + "workflow-local", + )) + .retry_config(RetryConfig::disabled()) + .timeout_config(timeouts()) + .build(), + ) +} +pub(super) fn lambda(server: &Server) -> Lambda { + Lambda::from_conf( + aws_sdk_lambda::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(&server.endpoint) + .credentials_provider(credentials()) + .retry_config(RetryConfig::disabled()) + .timeout_config(timeouts()) + .build(), + ) +} +pub(super) fn archive(bytes: &[u8]) -> Result> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + )?; + writer.write_all(bytes)?; + Ok(writer.finish()?.into_inner()) +} +async fn deploy(server: &Server, client: &Lambda) -> Result<()> { + let bytes = fs::read(server.bootstrap()?).await?; + let archive = tokio::task::spawn_blocking(move || archive(&bytes)).await??; + for (name, variables, seconds) in [ + ("SdkCompute", vec![("WORKFLOW_MODE", "1")], 5), + ( + "SdkRetry", + vec![("WORKFLOW_MODE", "1"), ("WORKFLOW_FAIL_ONCE", "1")], + 5, + ), + ("SdkFail", vec![("FAIL_MODE", "panic")], 5), + ("SdkSlow", vec![("SLEEP_SECS", "5")], 5), + ("SdkFunctionTimeout", vec![("SLEEP_SECS", "5")], 1), + ] { + let mut environment = Environment::builder().variables( + "PROCESS_MARKER_DIR", + server.markers.to_str().context("marker path")?, + ); + for (key, value) in variables { + environment = environment.variables(key, value); + } + client + .create_function() + .function_name(name) + .role(ROLE) + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .architectures(if cfg!(target_arch = "aarch64") { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .timeout(seconds) + .environment(environment.build()) + .code( + FunctionCode::builder() + .zip_file(Blob::new(archive.clone())) + .build(), + ) + .send() + .await?; + } + let gates = server + .markers + .parent() + .context("private root")? + .join("gates"); + client + .create_function() + .function_name("SdkGate") + .role(ROLE) + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .architectures(if cfg!(target_arch = "aarch64") { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .timeout(900) + .environment( + Environment::builder() + .variables("WORKFLOW_MODE", "1") + .variables( + "PROCESS_MARKER_DIR", + server.markers.to_str().context("marker path")?, + ) + .variables( + "INVOCATION_FILE", + gates.join("entered").to_str().context("gate path")?, + ) + .variables( + "WORKFLOW_GATE_FILE", + gates.join("release").to_str().context("gate path")?, + ) + .build(), + ) + .code( + FunctionCode::builder() + .zip_file(Blob::new(archive.clone())) + .build(), + ) + .send() + .await?; + check_lambda_compute(client).await +} +async fn check_lambda_compute(client: &Lambda) -> Result<()> { + let invoked = client + .invoke() + .function_name("SdkCompute") + .payload(Blob::new(br#"{"n":21}"#.to_vec())) + .send() + .await?; + ensure!( + invoked.status_code() == 200 && invoked.function_error().is_none(), + "actual Lambda SDK invoke failed" + ); + let result: Value = + serde_json::from_slice(invoked.payload().context("Lambda output")?.as_ref())?; + ensure!( + result["computed"] == 42 && result["pid"].as_u64().is_some_and(|pid| pid > 1), + "real bootstrap computation missing" + ); + Ok(()) +} + +pub(super) async fn create(client: &Client, name: &str, definition: &Value) -> Result { + let response = client + .create_state_machine() + .name(name) + .role_arn(ROLE) + .definition(definition.to_string()) + .send() + .await?; + ensure!( + response + .state_machine_arn() + .starts_with(&format!("{PREFIX}stateMachine:")), + "wrong machine scope" + ); + let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs())?; + ensure!( + (response.creation_date().secs() - now).abs() < 60, + "incorrect native SDK timestamp unit" + ); + Ok(response.state_machine_arn().to_owned()) +} +pub(super) async fn terminal(client: &Client, arn: &str) -> Result { + timeout(Duration::from_secs(20), async { + loop { + let value = client + .describe_execution() + .execution_arn(arn) + .send() + .await?; + if value.status() != &ExecutionStatus::Running { + return Ok(value); + } + sleep(Duration::from_millis(15)).await; + } + }) + .await + .context("execution did not terminate")? +} +pub(super) async fn history( + client: &Client, + arn: &str, + reverse: bool, + data: bool, +) -> Result> { + let mut pages = client + .get_execution_history() + .execution_arn(arn) + .max_results(7) + .reverse_order(reverse) + .include_execution_data(data) + .into_paginator() + .send(); + let mut events = Vec::new(); + let mut count = 0; + while let Some(page) = pages.try_next().await? { + count += 1; + ensure!(count <= 200, "unbounded history page count"); + ensure!(events.len() < 1000, "unbounded history paginator"); + events.extend(page.events); + } + Ok(events) +} +fn validated_catalog() -> Result { + let value: Value = + serde_json::from_str(include_str!("../../../../tests/workflows/sdk-cases.json"))?; + ensure!( + value["schemaVersion"] == 1 && value["profile"] == "rustack-standard-jsonpath-v1", + "unsupported SDK catalog" + ); + let cases = value["cases"].as_array().context("SDK case collection")?; + ensure!(cases.len() == 17, "required SDK case manifest changed"); + let mut names = BTreeSet::new(); + for case in cases { + let object = case.as_object().context("SDK case object")?; + for key in object.keys() { + ensure!( + [ + "id", + "definition", + "input", + "expectedOutput", + "uuidField", + "optimizedComputed", + "taskAttempts", + "expectedStatus", + "expectedError", + "expectedCause", + "largeInputBytes", + "evidence" + ] + .contains(&key.as_str()), + "unknown SDK expectation field" + ); + } + let id = case["id"].as_str().context("case ID")?; + ensure!( + !id.is_empty() && id.len() <= 48 && names.insert(id), + "missing/duplicate case ID" + ); + ensure!( + case["definition"].is_object() && object.contains_key("input"), + "missing SDK definition/input" + ); + let outputs = usize::from(object.contains_key("expectedOutput")) + + usize::from(object.contains_key("optimizedComputed")) + + usize::from(object.contains_key("expectedError")); + ensure!( + outputs == 1, + "every SDK case requires exactly one outcome oracle" + ); + if object.contains_key("expectedError") { + ensure!( + ["FAILED", "TIMED_OUT"] + .contains(&case["expectedStatus"].as_str().context("failure status")?), + "invalid failure oracle" + ); + } + if let Some(number) = case.get("optimizedComputed") { + ensure!(number.as_i64().is_some(), "invalid optimized oracle"); + } + if let Some(attempts) = case.get("taskAttempts") { + ensure!( + attempts.as_u64().is_some_and(|n| (1..=10).contains(&n)), + "invalid attempt oracle" + ); + } + for key in ["uuidField", "expectedError", "expectedCause"] { + if let Some(value) = case.get(key) { + ensure!( + value.is_string() + || (key == "expectedError" + && value.is_null() + && case["expectedStatus"] == "TIMED_OUT"), + "invalid string expectation" + ); + } + } + if let Some(size) = case.get("largeInputBytes") { + ensure!( + size.as_u64().is_some_and(|size| size <= 200_000) && case["input"].is_object(), + "invalid large-input fixture" + ); + } + let sources = case["evidence"]["sources"] + .as_array() + .context("case provenance")?; + ensure!(!sources.is_empty(), "missing independent source"); + for source in sources { + ensure!( + value["sources"] + .get(source.as_str().context("source identifier")?) + .is_some(), + "unknown fixture source" + ); + } + } + Ok(value) +} +#[test] +fn test_should_require_independent_expected_outcomes_for_all_sdk_cases() -> Result<()> { + validated_catalog().map(|_| ()) +} +async fn catalog(client: &Client) -> Result<()> { + let catalog = validated_catalog()?; + for case in catalog["cases"].as_array().context("SDK cases")? { + let id = case["id"].as_str().context("case id")?; + let mut input = case["input"].clone(); + if let Some(size) = case["largeInputBytes"].as_u64() { + input["large"] = Value::String("x".repeat(usize::try_from(size)?)); + } + let machine = create(client, &format!("SdkCase-{id}"), &case["definition"]).await?; + let raw = input.to_string(); + let started = client + .start_execution() + .state_machine_arn(&machine) + .name("Run") + .input(&raw) + .send() + .await?; + let result = terminal(client, started.execution_arn()) + .await + .with_context(|| format!("case {id}"))?; + ensure!( + result.status().as_str() == case["expectedStatus"].as_str().unwrap_or("SUCCEEDED"), + "case {id}: status {} error {:?}", + result.status().as_str(), + result.error() + ); + ensure!( + result.input() == Some(raw.as_str()), + "case {id}: raw input changed" + ); + ensure!( + result.start_date() == started.start_date() + && result + .stop_date() + .is_some_and(|date| date >= result.start_date()), + "case {id}: SDK dates inconsistent" + ); + verify_output(case, &result)?; + let events = history(client, started.execution_arn(), false, true).await?; + ensure!(!events.is_empty(), "case {id}: no actual history"); + for (offset, event) in events.iter().enumerate() { + ensure!( + event.id() == i64::try_from(offset + 1)? + && event.previous_event_id() == sequential_predecessor(offset)? + && event.timestamp() >= result.start_date(), + "case {id}: broken causal history" + ); + } + let terminal_kind = match result.status().as_str() { + "SUCCEEDED" => "ExecutionSucceeded", + "TIMED_OUT" => "ExecutionTimedOut", + _ => "ExecutionFailed", + }; + ensure!( + events + .last() + .is_some_and(|event| event.r#type().as_str() == terminal_kind), + "missing terminal history event" + ); + if let Some(attempts) = case["taskAttempts"].as_u64() { + ensure!( + u64::try_from( + events + .iter() + .filter(|event| event.r#type().as_str().ends_with("Scheduled")) + .count() + )? == attempts, + "hidden or missing business retries" + ); + } + if id == "optimized" { + pagination(client, started.execution_arn(), &events).await?; + } + } + Ok(()) +} +fn verify_output(case: &Value, result: &DescribeExecutionOutput) -> Result<()> { + let id = case["id"].as_str().context("case id")?; + if let Some(expected) = case.get("expectedOutput") { + let mut output: Value = + serde_json::from_str(result.output().context("successful output")?)?; + if let Some(field) = case["uuidField"].as_str() { + let uuid = output + .as_object_mut() + .context("UUID output object")? + .remove(field) + .context("UUID output")?; + ensure!( + Uuid::parse_str(uuid.as_str().context("UUID string")?)?.get_version_num() == 4, + "non-v4 UUID" + ); + } + ensure!(&output == expected, "case {id}: wrong output {output}"); + } + if let Some(expected) = case["optimizedComputed"].as_i64() { + let output: Value = serde_json::from_str(result.output().context("optimized output")?)?; + ensure!( + output["Payload"]["computed"] == expected + && output["StatusCode"] == 200 + && output["SdkHttpMetadata"]["HttpStatusCode"] == 200 + && output["ExecutedVersion"] == "$LATEST", + "bad optimized envelope" + ); + ensure!( + output["SdkResponseMetadata"]["RequestId"] == output["Payload"]["request_id"], + "invented invocation request ID" + ); + Uuid::parse_str( + output["Payload"]["request_id"] + .as_str() + .context("actual request ID")?, + )?; + ensure!( + output["Payload"]["pid"].as_u64().is_some_and(|pid| pid > 1), + "no actual process" + ); + } + if let Some(error) = case["expectedError"].as_str() { + ensure!( + result.error() == Some(error), + "case {id}: wrong error {:?}", + result.error() + ); + } + if case.get("expectedError") == Some(&Value::Null) { + ensure!( + result.error().is_none() && result.cause().is_none() && result.output().is_none(), + "case {id}: overall timeout manufactured outcome fields" + ); + } + if let Some(cause) = case["expectedCause"].as_str() { + ensure!( + result.cause() == Some(cause), + "case {id}: changed Unicode cause" + ); + } + Ok(()) +} +fn assert_omitted(events: &[HistoryEvent]) -> Result<()> { + for event in events { + let inputs = [ + event + .execution_started_event_details() + .and_then(|d| d.input()), + event.state_entered_event_details().and_then(|d| d.input()), + event + .lambda_function_scheduled_event_details() + .and_then(|d| d.input()), + ]; + let outputs = [ + event + .task_submitted_event_details() + .and_then(|d| d.output()), + event + .execution_succeeded_event_details() + .and_then(|d| d.output()), + event.state_exited_event_details().and_then(|d| d.output()), + event + .lambda_function_succeeded_event_details() + .and_then(|d| d.output()), + event + .task_succeeded_event_details() + .and_then(|d| d.output()), + ]; + let metadata_present = [ + event + .task_submitted_event_details() + .and_then(|d| d.output_details()) + .is_some(), + event + .execution_started_event_details() + .and_then(|d| d.input_details()) + .is_some(), + event + .state_entered_event_details() + .and_then(|d| d.input_details()) + .is_some(), + event + .lambda_function_scheduled_event_details() + .and_then(|d| d.input_details()) + .is_some(), + event + .execution_succeeded_event_details() + .and_then(|d| d.output_details()) + .is_some(), + event + .state_exited_event_details() + .and_then(|d| d.output_details()) + .is_some(), + event + .lambda_function_succeeded_event_details() + .and_then(|d| d.output_details()) + .is_some(), + event + .task_succeeded_event_details() + .and_then(|d| d.output_details()) + .is_some(), + ]; + ensure!( + inputs.iter().all(Option::is_none) + && outputs.iter().all(Option::is_none) + && metadata_present.iter().all(|present| !present) + && event.task_scheduled_event_details().is_none(), + "execution data was not omitted" + ); + } + Ok(()) +} +async fn pagination(client: &Client, arn: &str, events: &[HistoryEvent]) -> Result<()> { + let reverse = history(client, arn, true, false).await?; + ensure!( + reverse + .iter() + .map(HistoryEvent::id) + .eq(events.iter().rev().map(HistoryEvent::id)), + "reverse traversal changed IDs" + ); + ensure!( + reverse + .iter() + .zip(events.iter().rev()) + .all(|(left, right)| left.previous_event_id() == right.previous_event_id()), + "reverse traversal changed causal links" + ); + assert_omitted(&reverse)?; + let first = client + .get_execution_history() + .execution_arn(arn) + .max_results(1) + .send() + .await?; + let token = first.next_token().context("history token")?; + let next = client + .get_execution_history() + .execution_arn(arn) + .max_results(1) + .next_token(token) + .send() + .await?; + let replay = client + .get_execution_history() + .execution_arn(arn) + .max_results(1) + .next_token(token) + .send() + .await?; + ensure!( + next.events().first().map(HistoryEvent::id) + == replay.events().first().map(HistoryEvent::id), + "token was destructive" + ); + let error = client + .get_execution_history() + .execution_arn(arn) + .max_results(2) + .next_token(token) + .send() + .await + .err() + .context("query-bound token accepted")?; + ensure!( + error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("InvalidToken"), + "wrong cursor query error" + ); + let error = client + .get_execution_history() + .execution_arn(arn) + .next_token("0".repeat(64)) + .send() + .await + .err() + .context("forged token accepted")?; + ensure!( + error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("InvalidToken"), + "wrong forged token error" + ); + Ok(()) +} +fn pass(value: &str) -> Value { + json!({"StartAt":"Done","States":{"Done":{"Type":"Pass","Result":{"revision":value},"End":true}}}) +} +async fn tags(client: &Client, machine: &str) -> Result<()> { + client + .tag_resource() + .resource_arn(machine) + .tags(Tag::builder().key("owner").value("sdk").build()) + .send() + .await?; + let tags = client + .list_tags_for_resource() + .resource_arn(machine) + .send() + .await?; + ensure!( + tags.tags() + .iter() + .any(|tag| tag.key() == Some("owner") && tag.value() == Some("sdk")), + "tag missing" + ); + client + .untag_resource() + .resource_arn(machine) + .tag_keys("owner") + .send() + .await?; + ensure!( + client + .list_tags_for_resource() + .resource_arn(machine) + .send() + .await? + .tags() + .is_empty(), + "tag removal failed" + ); + Ok(()) +} +const RAW: &str = " {\"n\":1}"; +async fn duplicate(client: &Client, machine: &str) -> Result { + let start = || { + client + .start_execution() + .state_machine_arn(machine) + .name("Same") + .input(RAW) + .send() + }; + let (left, right) = tokio::join!(start(), start()); + let left = left?; + let right = right?; + ensure!( + left.execution_arn() == right.execution_arn() && left.start_date() == right.start_date(), + "RUNNING start not idempotent" + ); + for (input, code) in [ + ("{}", "ExecutionAlreadyExists"), + ("{", "InvalidExecutionInput"), + ] { + let error = client + .start_execution() + .state_machine_arn(machine) + .name("Same") + .input(input) + .send() + .await + .err() + .context("invalid duplicate accepted")?; + ensure!( + error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some(code), + "wrong duplicate validation error" + ); + } + Ok(left) +} +async fn consumed_gate(server: &Server) -> Result<()> { + timeout(Duration::from_secs(15), async { + let path = server + .markers + .parent() + .context("private root")? + .join("gates/entered"); + loop { + if let Ok(bytes) = fs::read(&path).await { + ensure!(bytes.len() <= 512, "oversized consumed-event marker"); + if let Ok(value) = serde_json::from_slice::(&bytes) { + ensure!( + value["pid"].as_u64().is_some_and(|pid| pid > 1), + "invalid consumed PID" + ); + Uuid::parse_str(value["request_id"].as_str().context("actual request ID")?)?; + return Ok(()); + } + } + sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("handler consumption barrier expired")? +} +async fn revisions( + client: &Client, + server: &Server, + machine: &str, + original: &DescribeStateMachineOutput, + first: &StartExecutionOutput, +) -> Result<()> { + consumed_gate(server).await?; + let original_id = original + .revision_id() + .context("missing original revision UUID")?; + Uuid::parse_str(original_id)?; + let update = client + .update_state_machine() + .state_machine_arn(machine) + .definition(pass("new").to_string()) + .send() + .await?; + let update_id = update + .revision_id() + .context("missing update revision UUID")?; + Uuid::parse_str(update_id)?; + ensure!(update_id != original_id, "update did not advance revision"); + let captured = client + .describe_state_machine_for_execution() + .execution_arn(first.execution_arn()) + .send() + .await?; + let captured_id = captured + .revision_id() + .context("missing captured revision UUID")?; + Uuid::parse_str(captured_id)?; + ensure!( + captured.definition() == original.definition() + && captured_id == original_id + && captured.update_date() == original.creation_date(), + "captured revision/date drifted" + ); + let new = client + .start_execution() + .state_machine_arn(machine) + .name("New") + .send() + .await?; + ensure!( + terminal(client, new.execution_arn()).await?.output() == Some("{\"revision\":\"new\"}"), + "new definition not used" + ); + fs::write( + server + .markers + .parent() + .context("private root")? + .join("gates/release"), + b"release", + ) + .await?; + ensure!( + terminal(client, first.execution_arn()).await?.output() == Some("{\"revision\":\"old\"}"), + "running definition changed" + ); + let error = client + .start_execution() + .state_machine_arn(machine) + .name("Same") + .input(RAW) + .send() + .await + .err() + .context("closed name reused")?; + ensure!( + error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("ExecutionAlreadyExists"), + "wrong closed-name error" + ); + Ok(()) +} +async fn stop(client: &Client, machine: &str) -> Result<()> { + let wait = + json!({"StartAt":"Wait","States":{"Wait":{"Type":"Wait","Seconds":3600,"End":true}}}); + client + .update_state_machine() + .state_machine_arn(machine) + .definition(wait.to_string()) + .send() + .await?; + let started = client + .start_execution() + .state_machine_arn(machine) + .name("Stop") + .send() + .await?; + client + .stop_execution() + .execution_arn(started.execution_arn()) + .error("") + .cause("quoted \"stop\" 🙂") + .send() + .await?; + let result = terminal(client, started.execution_arn()).await?; + ensure!( + result.status() == &ExecutionStatus::Aborted + && result.error() == Some("") + && result.cause() == Some("quoted \"stop\" 🙂"), + "Stop fields changed" + ); + Ok(()) +} +async fn retained_lists(client: &Client, machine: &str, first: &str) -> Result<()> { + let page = client + .list_executions() + .state_machine_arn(machine) + .max_results(1) + .send() + .await?; + let mut arns: BTreeSet = page + .executions() + .iter() + .map(|item| item.execution_arn().to_owned()) + .collect(); + let mut token = page.next_token().map(str::to_owned); + let mut seen = BTreeSet::new(); + client + .update_state_machine() + .state_machine_arn(machine) + .definition(pass("late").to_string()) + .send() + .await?; + let late = client + .start_execution() + .state_machine_arn(machine) + .name("Late") + .send() + .await?; + terminal(client, late.execution_arn()).await?; + let first_history = client + .get_execution_history() + .execution_arn(first) + .max_results(1) + .send() + .await?; + client + .delete_state_machine() + .state_machine_arn(machine) + .send() + .await?; + while let Some(next) = token { + ensure!( + seen.len() < 10 && seen.insert(next.clone()), + "unbounded execution cursor" + ); + let page = client + .list_executions() + .state_machine_arn(machine) + .max_results(1) + .next_token(next) + .send() + .await?; + arns.extend( + page.executions() + .iter() + .map(|item| item.execution_arn().to_owned()), + ); + token = page.next_token().map(str::to_owned); + } + ensure!( + arns.len() == 3 && !arns.contains(late.execution_arn()), + "execution-list capture drifted after delete" + ); + client + .get_execution_history() + .execution_arn(first) + .max_results(1) + .next_token( + first_history + .next_token() + .context("retained history cursor")?, + ) + .send() + .await?; + ensure!( + client + .describe_execution() + .execution_arn(first) + .send() + .await? + .input() + == Some(RAW), + "execution lost after deletion" + ); + Ok(()) +} +async fn list_and_validate(client: &Client) -> Result<()> { + let mut pages = client + .list_state_machines() + .max_results(2) + .into_paginator() + .send(); + let mut count = 0; + let mut names = BTreeSet::new(); + while let Some(page) = pages.try_next().await? { + count += 1; + ensure!(count <= 64, "unbounded machine pages"); + names.extend( + page.state_machines() + .iter() + .map(|item| item.name().to_owned()), + ); + } + ensure!( + names.contains("SdkCase-optimized") && !names.contains("SdkControl"), + "resource list/delete projection failed" + ); + let valid = client + .validate_state_machine_definition() + .definition(pass("valid").to_string()) + .severity(ValidateStateMachineDefinitionSeverity::Warning) + .send() + .await?; + ensure!(valid.result().as_str() == "OK", "valid definition rejected"); + for definition in [ + json!({"StartAt":"Missing","States":{}}), + json!({"StartAt":"Run","States":{"Run":{"Type":"Pass","InputPath":"$[","End":true}}}), + json!({"StartAt":"Run","States":{"Run":{"Type":"Pass","Parameters":{"duplicate":1,"duplicate.$":"$.n"},"End":true}}}), + ] { + let invalid = client + .validate_state_machine_definition() + .definition(definition.to_string()) + .send() + .await?; + ensure!( + invalid.result().as_str() == "FAIL" && !invalid.diagnostics().is_empty(), + "invalid definition accepted" + ); + } + let unsupported = client + .create_activity() + .name("unsupported") + .send() + .await + .err() + .context("unsupported operation accepted")?; + ensure!( + unsupported + .as_service_error() + .and_then(ProvideErrorMetadata::code) + == Some("UnsupportedOperation"), + "unsupported operation disguised" + ); + Ok(()) +} +async fn controls(client: &Client, server: &Server) -> Result<()> { + let old = json!({"StartAt":"Gate","States":{"Gate":{"Type":"Task","Resource":"arn:aws:lambda:us-east-1:000000000000:function:SdkGate","Parameters":{"n":21},"ResultSelector":{"revision":"old"},"End":true}}}); + let machine = create(client, "SdkControl", &old).await?; + let described = client + .describe_state_machine() + .state_machine_arn(&machine) + .send() + .await?; + let original = old.to_string(); + ensure!(described.definition() == original, "definition changed"); + tags(client, &machine).await?; + let first = duplicate(client, &machine).await?; + revisions(client, server, &machine, &described, &first).await?; + stop(client, &machine).await?; + retained_lists(client, &machine, first.execution_arn()).await?; + list_and_validate(client).await +} +pub(super) async fn exercise(server: &Server) -> Result<()> { + let client = sfn(server, SECRET); + let bad = sfn(server, "wrong-local-secret") + .list_state_machines() + .send() + .await + .err() + .context("strict signing not enforced")?; + ensure!( + bad.as_service_error().and_then(ProvideErrorMetadata::code) + == Some("AccessDeniedException"), + "wrong auth error" + ); + deploy(server, &lambda(server)).await?; + catalog(&client).await?; + controls(&client, server).await?; + nested::exercise(&client, server).await?; + structured::exercise(&client, server).await?; + Ok(()) +} + +pub(super) async fn nested_only(server: &Server) -> Result<()> { + nested::exercise(&sfn(server, SECRET), server).await +} + +pub(super) async fn callbacks_only(server: &Server) -> Result<()> { + callback::exercise(server).await +} + +pub(super) async fn native_probe_only(server: &Server) -> Result<()> { + deploy(server, &lambda(server)).await +} diff --git a/apps/rustack/tests/workflows_sdk/rust/callback.rs b/apps/rustack/tests/workflows_sdk/rust/callback.rs new file mode 100644 index 00000000..bf45ed6a --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/rust/callback.rs @@ -0,0 +1,486 @@ +//! Actual strict-auth service and both handler-side SDKs, using one dispatch slot. + +use std::{collections::BTreeSet, env, path::PathBuf}; + +use serde::Deserialize; + +use super::*; +use crate::{process::assert_absent, runner::interpreter}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Case { + id: String, + mode: String, + status: String, + #[serde(default)] + error: Option, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Catalog { + source: String, + cases: Vec, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Evidence { + request_id: String, + pid: u32, + sdk: String, + stage: String, + retry: u32, + heartbeats: u32, + history_safe: bool, + closed_token: bool, +} + +fn root(server: &Server) -> Result { + Ok(server + .markers + .parent() + .context("private fixture root")? + .to_owned()) +} +async fn markers(server: &Server) -> Result> { + let mut directory = fs::read_dir(root(server)?.join("callbacks")).await?; + let mut events = Vec::new(); + let mut files = 0; + while let Some(entry) = directory.next_entry().await? { + files += 1; + ensure!( + files <= 64 && entry.file_type().await?.is_file(), + "invalid callback evidence collection" + ); + if entry + .path() + .extension() + .is_none_or(|extension| extension != "json") + { + continue; + } + ensure!( + entry.metadata().await?.len() <= 1024, + "oversized callback evidence" + ); + let event: Evidence = serde_json::from_slice(&fs::read(entry.path()).await?)?; + ensure!( + event.request_id.len() == 36 + && Uuid::parse_str(&event.request_id)?.to_string() == event.request_id + && entry.file_name() == format!("{}.json", event.request_id).as_str() + && event.pid > 1 + && event.retry <= 1 + && ["rust", "boto3"].contains(&event.sdk.as_str()) + && ["consumed", "returned", "heartbeat", "callback"] + .contains(&event.stage.as_str()), + "invalid callback evidence" + ); + events.push(event); + } + Ok(events) +} +async fn next(server: &Server, seen: &BTreeSet, sdk: &str) -> Result { + timeout(Duration::from_secs(10), async { + loop { + if let Some(event) = markers(server) + .await? + .into_iter() + .find(|v| !seen.contains(&v.request_id) && v.sdk == sdk) + { + return Ok(event); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("handler did not consume an actual event")? +} +async fn stage(server: &Server, id: &str, expected: &str) -> Result { + timeout(Duration::from_secs(10), async { + loop { + if let Some(event) = markers(server) + .await? + .into_iter() + .find(|v| v.request_id == id && v.stage == expected) + { + return Ok(event); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("handler SDK stage did not complete")? +} +async fn release(server: &Server, id: &str) -> Result<()> { + fs::write( + root(server)? + .join("callback-gates") + .join(format!("{id}.release")), + b"release", + ) + .await?; + Ok(()) +} +async fn submitted(client: &Client, arn: &str) -> Result<()> { + timeout(Duration::from_secs(10), async { + loop { + let events = history(client, arn, false, true).await?; + if events + .iter() + .any(|event| event.r#type().as_str() == "TaskSubmitted") + { + return Ok(()); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("actual Lambda invocation was not submitted")? +} +async fn running(client: &Client, arn: &str) -> Result<()> { + ensure!( + client + .describe_execution() + .execution_arn(arn) + .send() + .await? + .status() + == &ExecutionStatus::Running, + "callback task terminated before its gate" + ); + Ok(()) +} +async fn deploy_callbacks(server: &Server) -> Result<()> { + let rust = PathBuf::from( + env::var_os("RUSTACK_CALLBACK_BOOTSTRAP") + .context("build actual callback bootstrap with make")?, + ); + ensure!( + (1..=64 * 1024 * 1024).contains(&fs::metadata(&rust).await?.len()), + "invalid callback bootstrap size" + ); + let python = interpreter()?; + let python = python.to_str().context("interpreter path")?; + let client = lambda(server); + let private = root(server)?; + let native = fs::read(rust).await?; + for (name, python_handler) in [("CallbackRust", false), ("CallbackPython", true)] { + let bytes = native.clone(); + let archive = + tokio::task::spawn_blocking(move || callback_archive(&bytes, python_handler)).await??; + let mut variables = Environment::builder() + .variables("CALLBACK_ENDPOINT", &server.endpoint) + .variables("CALLBACK_ACCESS_KEY", ACCESS) + .variables("CALLBACK_SECRET_KEY", SECRET) + .variables( + "PROCESS_MARKER_DIR", + server.markers.to_str().context("process path")?, + ) + .variables( + "CALLBACK_MARKER_DIR", + private + .join("callbacks") + .to_str() + .context("callback path")?, + ) + .variables( + "CALLBACK_GATE_DIR", + private + .join("callback-gates") + .to_str() + .context("gate path")?, + ); + if python_handler { + variables = variables.variables("CALLBACK_PYTHON", python); + } + client + .create_function() + .function_name(name) + .role(ROLE) + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .architectures(if cfg!(target_arch = "aarch64") { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .timeout(30) + .environment(variables.build()) + .code(FunctionCode::builder().zip_file(Blob::new(archive)).build()) + .send() + .await?; + } + Ok(()) +} +fn callback_archive(native: &[u8], python: bool) -> Result> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + )?; + writer.write_all(native)?; + if python { + writer.start_file( + "callback.py", + SimpleFileOptions::default().unix_permissions(0o600), + )?; + writer.write_all(include_bytes!( + "../../../../../tests/lambda-fixtures/callback_bootstrap.py" + ))?; + } + Ok(writer.finish()?.into_inner()) +} + +fn definition(function: &str, case: &Case) -> Value { + let mut task = json!({"Type":"Task","Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken","TimeoutSeconds":60, + "Parameters":{"FunctionName":function,"Payload":{"token.$":"$$.Task.Token","executionArn.$":"$$.Execution.Id","mode":case.mode,"n":21,"retry.$":"$$.State.RetryCount"}},"End":true}); + if ["heartbeat", "absoluteTimeout", "heartbeatTimeout"].contains(&case.mode.as_str()) { + task["HeartbeatSeconds"] = json!(1); + } + if case.mode == "absoluteTimeout" { + task["TimeoutSeconds"] = json!(2); + } + if case.mode == "retry" { + task["Retry"] = + json!([{"ErrorEquals":["CallbackRetry"],"IntervalSeconds":1,"MaxAttempts":1}]); + } + json!({"StartAt":"Call","States":{"Call":task}}) +} +async fn unrelated(client: &Client) -> Result<()> { + let definition = json!({"StartAt":"Call","States":{"Call":{"Type":"Task","Resource":"arn:aws:lambda:us-east-1:000000000000:function:SdkCompute","End":true}}}); + let machine = create(client, &format!("Probe{}", Uuid::new_v4()), &definition).await?; + let execution = client + .start_execution() + .state_machine_arn(machine) + .input("{\"n\":21}") + .send() + .await?; + let result = terminal(client, execution.execution_arn()).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "unrelated workflow could not acquire dispatch capacity" + ); + let output: Value = serde_json::from_str(result.output().context("probe output")?)?; + ensure!( + output["computed"] == 42, + "unrelated actual Lambda computation missing" + ); + Ok(()) +} +async fn retired(pid: u32) -> Result<()> { + timeout(Duration::from_secs(5), async { + loop { + if assert_absent(pid).await.is_ok() { + return Ok(()); + } + sleep(Duration::from_millis(15)).await; + } + }) + .await + .context("consumed handler PID survived cancellation")? +} +async fn verify(client: &Client, execution: &str, case: &Case, events: &[Evidence]) -> Result<()> { + let result = terminal(client, execution).await?; + ensure!( + result.status().as_str() == case.status, + "callback case {}: status {:?}, error {:?}", + case.id, + result.status(), + result.error() + ); + ensure!( + result.error() == case.error.as_deref(), + "callback case {}: incorrect error", + case.id + ); + if case.status == "SUCCEEDED" { + let output: Value = serde_json::from_str(result.output().context("callback output")?)?; + let last = events.last().context("consumed callback")?; + ensure!( + output + == json!({"value":42,"sdk":last.sdk,"requestId":last.request_id,"retry":last.retry}), + "actual handler result lost or double encoded" + ); + } + let history = history(client, execution, false, true).await?; + let mut submitted_count = 0; + for (offset, event) in history.iter().enumerate() { + ensure!( + event.id() == i64::try_from(offset + 1)? + && event.previous_event_id() == sequential_predecessor(offset)?, + "broken callback causal history" + ); + if let Some(details) = event.task_scheduled_event_details() { + let parameters: Value = serde_json::from_str(details.parameters())?; + ensure!( + parameters["Payload"]["token"].is_null(), + "injected token exposed in Scheduled history" + ); + } + if let Some(details) = event.task_submitted_event_details() { + submitted_count += 1; + let value: Value = + serde_json::from_str(details.output().context("submitted metadata")?)?; + let request = value["SdkResponseMetadata"]["RequestId"] + .as_str() + .context("actual request metadata")?; + ensure!( + events.iter().any(|e| e.request_id == request) + && value["ExecutedVersion"] == "$LATEST" + && value["SdkHttpMetadata"]["HttpStatusCode"] == 200, + "invented callback invocation metadata" + ); + ensure!( + details.output_details().is_some_and(|d| d.truncated), + "missing callback body omission marker" + ); + } + } + let expected = if case.mode == "stop" || case.mode == "invocationFailure" { + 0 + } else if case.mode == "retry" { + 2 + } else { + 1 + }; + ensure!( + submitted_count == expected, + "missing or duplicate callback submission" + ); + Ok(()) +} +async fn exercise_case( + client: &Client, + server: &Server, + case: &Case, + function: &str, + sdk: &str, + seen: &mut BTreeSet, +) -> Result<()> { + let machine = create( + client, + &format!("{function}-{}", case.id), + &definition(function, case), + ) + .await?; + let execution = client + .start_execution() + .state_machine_arn(machine) + .send() + .await?; + let arn = execution.execution_arn(); + let first = next(server, seen, sdk).await?; + seen.insert(first.request_id.clone()); + let mut events = vec![first]; + let first = events.first().context("first consumed callback")?; + if case.mode == "stop" { + running(client, arn).await?; + client + .stop_execution() + .execution_arn(arn) + .error("Stopped") + .send() + .await?; + retired(first.pid).await?; + } else if ["late", "heartbeat", "absoluteTimeout", "heartbeatTimeout"] + .contains(&case.mode.as_str()) + { + submitted(client, arn).await?; + running(client, arn).await?; + if case.mode == "late" { + unrelated(client).await?; + running(client, arn).await?; + } + if case.mode == "heartbeat" { + sleep(Duration::from_millis(1400)).await; + running(client, arn).await?; + ensure!( + stage(server, &first.request_id, "heartbeat") + .await? + .heartbeats + >= 3, + "actual SDK heartbeats missing" + ); + } + if ["absoluteTimeout", "heartbeatTimeout"].contains(&case.mode.as_str()) { + terminal(client, arn).await?; + } + release(server, &first.request_id).await?; + let complete = stage(server, &first.request_id, "callback").await?; + ensure!( + complete.history_safe && complete.closed_token, + "handler token/history checks failed" + ); + } else { + let complete = stage(server, &first.request_id, "callback").await?; + ensure!( + complete.history_safe && complete.closed_token, + "handler SDK evidence missing" + ); + running(client, arn).await?; + release(server, &first.request_id).await?; + if case.mode == "retry" { + let next = next(server, seen, sdk).await?; + seen.insert(next.request_id.clone()); + ensure!( + next.retry == 1 && next.pid == first.pid, + "fresh retry context or retained old-token proof missing" + ); + let complete = stage(server, &next.request_id, "callback").await?; + ensure!( + complete.history_safe && complete.closed_token, + "retry SDK checks failed" + ); + running(client, arn).await?; + release(server, &next.request_id).await?; + events.push(next); + } + } + verify(client, arn, case, &events).await?; + if case.mode == "invocationFailure" { + retired(events.first().context("invocation")?.pid).await?; + } + Ok(()) +} +pub(super) async fn exercise(server: &Server) -> Result<()> { + let capabilities = server.capabilities().await?; + let profile = &capabilities["workflows"]; + ensure!( + capabilities["ready"] == true + && profile["callbacks"] == true + && profile["dockerQualified"] == false + && profile["limits"]["dispatches"] == 1 + && profile["limits"]["callbackTokens"] == 2 + && profile["limits"]["callbackTombstones"] == 64 + && profile["limits"]["callbackCommands"] == 8 + && profile["limits"]["callbackWorkers"] == 1 + && profile["limits"]["callbackRetentionSeconds"] == 86_400, + "actual service callback capability mismatch" + ); + let catalog: Catalog = serde_json::from_str(include_str!( + "../../../../../tests/workflows/callback-cases.json" + ))?; + ensure!( + catalog.cases.len() == 9 + && catalog.source + == "https://docs.aws.amazon.com/step-functions/latest/dg/connect-lambda.html", + "invalid callback catalog" + ); + let client = sfn(server, SECRET); + deploy(server, &lambda(server)).await?; + deploy_callbacks(server).await?; + let mut seen = BTreeSet::new(); + for (function, sdk) in [("CallbackRust", "rust"), ("CallbackPython", "boto3")] { + for case in &catalog.cases { + exercise_case(&client, server, case, function, sdk, &mut seen) + .await + .with_context(|| format!("{sdk} handler callback case {}", case.id))?; + eprintln!("{sdk} handler callback case {} passed", case.id); + } + } + ensure!( + seen.len() == 20, + "missing actual consumed callback attempts" + ); + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk/rust/nested.rs b/apps/rustack/tests/workflows_sdk/rust/nested.rs new file mode 100644 index 00000000..3a5b91da --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/rust/nested.rs @@ -0,0 +1,241 @@ +//! Independent optimized nested envelopes and actual child Native cleanup. + +use super::*; +use crate::process::assert_absent; + +fn catalog() -> Result { + let value: Value = serde_json::from_str(include_str!( + "../../../../../tests/workflows/nested-cases.json" + ))?; + ensure!( + value["schemaVersion"] == 1 && value["profile"] == "rustack-standard-jsonpath-nested-v1", + "invalid nested profile" + ); + let cases = value["cases"].as_array().context("cases")?; + ensure!(cases.len() == 6, "missing nested cases"); + let mut names = BTreeSet::new(); + for case in cases { + ensure!( + names.insert(case["id"].as_str().context("ID")?) + && case["suffix"].is_string() + && case.get("input").is_some() + && case.get("expected").is_some(), + "invalid nested case" + ); + } + Ok(value) +} +#[test] +fn test_should_validate_independent_nested_catalog() -> Result<()> { + catalog().map(|_| ()) +} +fn definition(child: &str, suffix: &str, input: &Value) -> Value { + json!({"StartAt":"T","States":{"T":{"Type":"Task","Resource":format!("arn:aws:states:::states:startExecution{suffix}"),"Parameters":{"StateMachineArn":child,"Input":input},"End":true}}}) +} +async fn check_history(client: &Client, arn: &str, suffix: &str, child: &str) -> Result<()> { + let events = history(client, arn, false, true).await?; + let mut expected = vec![ + "ExecutionStarted", + "TaskStateEntered", + "TaskScheduled", + "TaskStarted", + ]; + if !suffix.is_empty() { + expected.push("TaskSubmitted"); + } + expected.extend(["TaskSucceeded", "TaskStateExited", "ExecutionSucceeded"]); + ensure!( + events.iter().map(|e| e.r#type().as_str()).eq(expected), + "wrong nested history" + ); + for (index, event) in events.iter().enumerate() { + ensure!( + event.id() == i64::try_from(index + 1)? + && event.previous_event_id() == sequential_predecessor(index)?, + "invalid nested predecessor" + ); + if let Some(details) = event.task_submitted_event_details() { + ensure!( + details.resource_type() == "states" + && details.resource() == format!("startExecution{suffix}"), + "wrong submitted resource" + ); + let accepted: Value = + serde_json::from_str(details.output().context("submitted output")?)?; + ensure!(accepted["ExecutionArn"] == child, "invented child identity"); + } + } + let reverse = history(client, arn, true, false).await?; + assert_omitted(&reverse)?; + ensure!( + events + .iter() + .rev() + .map(|e| (e.id(), e.previous_event_id())) + .eq(reverse.iter().map(|e| (e.id(), e.previous_event_id()))), + "changed nested pagination identity" + ); + Ok(()) +} +async fn cases(client: &Client) -> Result<()> { + let child = create( + client, + "SdkNestedEcho", + &json!({"StartAt":"D","States":{"D":{"Type":"Pass","End":true}}}), + ) + .await?; + for case in catalog()?["cases"].as_array().context("cases")? { + let suffix = case["suffix"].as_str().context("suffix")?; + let id = case["id"].as_str().context("id")?; + let machine = create( + client, + &format!("SdkNested-{id}"), + &definition(&child, suffix, &case["input"]), + ) + .await?; + let start = client + .start_execution() + .state_machine_arn(machine) + .name("Run") + .input("{}") + .send() + .await?; + let result = terminal(client, start.execution_arn()).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "nested {id}: {result:?}" + ); + let output: Value = serde_json::from_str(result.output().context("output")?)?; + let child_arn = output["ExecutionArn"].as_str().context("child ARN")?; + let nested = terminal(client, child_arn).await?; + ensure!( + serde_json::from_str::(nested.output().context("child output")?)? + == case["expected"], + "wrong child result" + ); + ensure!( + output["StartDate"].as_i64().is_some_and(|millis| nested + .start_date() + .to_millis() + .is_ok_and(|observed| millis.abs_diff(observed) <= 1)), + "wrong nested millisecond date" + ); + if !suffix.is_empty() { + ensure!( + output["Status"] == "SUCCEEDED" + && output["StateMachineArn"] == child + && output["InputDetails"] == json!({"Included":true}) + && output["OutputDetails"] == json!({"Included":true}), + "wrong optimized envelope" + ); + if suffix == ".sync" { + ensure!( + serde_json::from_str::( + output["Input"].as_str().context("string input")? + )? == case["expected"] + && serde_json::from_str::( + output["Output"].as_str().context("string output")? + )? == case["expected"], + "wrong sync string data" + ); + if case["input"].is_string() { + ensure!( + output["Input"] == case["input"], + "changed serialized input bytes" + ); + } + } else { + ensure!( + output["Input"] == case["expected"] && output["Output"] == case["expected"], + "wrong sync2 JSON data" + ); + } + } + check_history(client, start.execution_arn(), suffix, child_arn).await?; + } + Ok(()) +} +async fn gated(client: &Client, server: &Server, stop: bool) -> Result<()> { + let name = if stop { + "SdkNestedStop" + } else { + "SdkNestedCompute" + }; + let directory = structured::deploy_named(server, &lambda(server), name).await?; + let child = create(client, &format!("{name}Child"), &json!({"StartAt":"L","States":{"L":{"Type":"Task","Resource":format!("arn:aws:lambda:us-east-1:000000000000:function:{name}"),"End":true}}})).await?; + let machine = create( + client, + &format!("{name}Parent"), + &definition(&child, ".sync:2", &json!({"n":21})), + ) + .await?; + let start = client + .start_execution() + .state_machine_arn(machine) + .input("{}") + .send() + .await?; + let markers = structured::consumed(&directory, 1).await?; + let (pid, request) = markers.first().context("consumed handler")?; + let events = history(client, start.execution_arn(), false, true).await?; + let submitted: Vec<_> = events + .iter() + .filter_map(HistoryEvent::task_submitted_event_details) + .collect(); + ensure!(submitted.len() == 1, "missing unique nested submission"); + let accepted: Value = serde_json::from_str( + submitted + .first() + .context("submitted")? + .output() + .context("accepted output")?, + )?; + let child_arn = accepted["ExecutionArn"].as_str().context("child ARN")?; + if stop { + client + .stop_execution() + .execution_arn(start.execution_arn()) + .error("NestedStopped") + .send() + .await?; + ensure!( + terminal(client, start.execution_arn()).await?.status() == &ExecutionStatus::Aborted, + "parent not stopped" + ); + ensure!( + terminal(client, child_arn).await?.status() == &ExecutionStatus::Aborted, + "child not stopped" + ); + timeout(Duration::from_secs(5), async { + while assert_absent(*pid).await.is_err() { + sleep(Duration::from_millis(20)).await; + } + }) + .await + .context("nested handler PID survived")?; + } else { + fs::write(directory.join("release"), b"release").await?; + let result = terminal(client, start.execution_arn()).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "nested compute failed: {result:?}" + ); + let output: Value = serde_json::from_str(result.output().context("nested output")?)?; + ensure!( + output["ExecutionArn"] == child_arn + && output["Output"]["computed"] == 42 + && output["Output"]["pid"].as_u64() == Some(u64::from(*pid)) + && output["Output"]["request_id"] == *request, + "returned identity differs from actual consumed child" + ); + check_history(client, start.execution_arn(), ".sync:2", child_arn).await?; + } + Ok(()) +} +pub(super) async fn exercise(client: &Client, server: &Server) -> Result<()> { + cases(client).await?; + for stop in [false, true] { + gated(client, server, stop).await?; + } + Ok(()) +} diff --git a/apps/rustack/tests/workflows_sdk/rust/structured.rs b/apps/rustack/tests/workflows_sdk/rust/structured.rs new file mode 100644 index 00000000..92e6a84e --- /dev/null +++ b/apps/rustack/tests/workflows_sdk/rust/structured.rs @@ -0,0 +1,375 @@ +//! Independent structured SDK expectations and consumed-event lifecycle barriers. + +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, +}; + +use aws_sdk_sfn::types::MapIterationEventDetails; + +use super::*; +use crate::process::assert_absent; + +fn catalog() -> Result { + let catalog: Value = serde_json::from_str(include_str!( + "../../../../../tests/workflows/structured-cases.json" + ))?; + ensure!( + catalog["schemaVersion"] == 1 + && catalog["profile"] == "rustack-standard-jsonpath-structured-v1", + "invalid structured catalog profile" + ); + let cases = catalog["cases"].as_array().context("structured cases")?; + ensure!(cases.len() == 6, "missing structured cases"); + let mut names = BTreeSet::new(); + for case in cases { + let id = case["id"].as_str().context("case ID")?; + ensure!(names.insert(id), "duplicate structured case"); + ensure!( + case.get("expectedOutput").is_some() + && case["definition"].is_object() + && case.get("input").is_some(), + "incomplete structured expectation" + ); + } + Ok(catalog) +} +#[test] +fn test_should_validate_independent_structured_catalog() -> Result<()> { + catalog().map(|_| ()) +} +async fn check_history(client: &Client, arn: &str, round: &str) -> Result> { + let events = history(client, arn, false, true).await?; + ensure!(!events.is_empty(), "missing structured history"); + for (offset, event) in events.iter().enumerate() { + ensure!( + event.id() == i64::try_from(offset + 1)? + && event.previous_event_id() >= 0 + && event.previous_event_id() < event.id(), + "invalid structured causal predecessor" + ); + } + ensure!( + events.iter().any(|event| event.r#type().as_str() == round), + "missing {round}" + ); + let reverse = history(client, arn, true, false).await?; + ensure!( + events + .iter() + .rev() + .map(|e| (e.id(), e.previous_event_id())) + .eq(reverse.iter().map(|e| (e.id(), e.previous_event_id()))), + "causal links changed under reverse/data-disabled pagination" + ); + assert_omitted(&reverse)?; + for event in &events { + if event.r#type().as_str() == "MapIterationStarted" { + ensure!( + events + .iter() + .find(|parent| parent.id() == event.previous_event_id()) + .is_some_and(|parent| matches!( + parent.r#type().as_str(), + "MapStateStarted" + | "PassStateExited" + | "WaitStateExited" + | "TaskStateExited" + | "ParallelStateExited" + | "MapStateExited" + | "SucceedStateExited" + )), + "Map iteration did not link to its round or completed processor" + ); + } + } + Ok(events) +} +async fn cases(client: &Client) -> Result<()> { + for case in catalog()?["cases"].as_array().context("cases")? { + let id = case["id"].as_str().context("case ID")?; + let machine = create(client, &format!("SdkStructured-{id}"), &case["definition"]).await?; + let started = client + .start_execution() + .state_machine_arn(machine) + .name("Run") + .input(case["input"].to_string()) + .send() + .await?; + let result = terminal(client, started.execution_arn()).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "structured {id}: {result:?}" + ); + let output: Value = serde_json::from_str(result.output().context("structured output")?)?; + ensure!( + output == case["expectedOutput"], + "structured {id}: {output}" + ); + let events = check_history( + client, + started.execution_arn(), + case["round"].as_str().context("round")?, + ) + .await?; + if let Some(count) = case["taskAttempts"].as_u64() { + ensure!( + events + .iter() + .filter(|e| e.r#type().as_str().ends_with("Scheduled")) + .count() + == usize::try_from(count)?, + "wrong fanout invocation count" + ); + } + } + Ok(()) +} +async fn deploy_gate(server: &Server, client: &Lambda, stop: bool) -> Result { + deploy_named( + server, + client, + if stop { + "SdkFanoutStop" + } else { + "SdkFanoutGate" + }, + ) + .await +} +pub(super) async fn deploy_named(server: &Server, client: &Lambda, name: &str) -> Result { + let directory = server.markers.parent().context("root")?.join(name); + fs::create_dir(&directory).await?; + fs::create_dir(directory.join("consumed")).await?; + let bytes = fs::read(server.bootstrap()?).await?; + let archive = tokio::task::spawn_blocking(move || archive(&bytes)).await??; + client + .create_function() + .function_name(name) + .role(ROLE) + .runtime(Runtime::Providedal2023) + .handler("bootstrap") + .architectures(if cfg!(target_arch = "aarch64") { + Architecture::Arm64 + } else { + Architecture::X8664 + }) + .timeout(900) + .environment( + Environment::builder() + .variables("WORKFLOW_MODE", "1") + .variables( + "PROCESS_MARKER_DIR", + server.markers.to_str().context("marker path")?, + ) + .variables( + "WORKFLOW_INVOCATION_DIR", + directory + .join("consumed") + .to_str() + .context("consumed path")?, + ) + .variables( + "WORKFLOW_GATE_FILE", + directory.join("release").to_str().context("release path")?, + ) + .build(), + ) + .code(FunctionCode::builder().zip_file(Blob::new(archive)).build()) + .send() + .await?; + Ok(directory) +} +pub(super) async fn consumed(directory: &Path, count: usize) -> Result> { + timeout(Duration::from_secs(10), async { + loop { + let mut entries = fs::read_dir(directory.join("consumed")).await?; + let mut markers = BTreeSet::new(); + while let Some(entry) = entries.next_entry().await? { + if entry.path().extension().is_none_or(|v| v != "json") { + continue; + } + ensure!( + markers.len() < count && entry.metadata().await?.len() <= 1024, + "unexpected marker count/size" + ); + let marker: Value = serde_json::from_slice(&fs::read(entry.path()).await?)?; + let pid = u32::try_from(marker["pid"].as_u64().context("PID")?)?; + let request = marker["request_id"].as_str().context("request ID")?; + ensure!( + pid > 1 && Uuid::parse_str(request)?.get_version_num() == 4, + "invalid consumed identity" + ); + markers.insert((pid, request.to_owned())); + } + if markers.len() == count { + ensure!( + markers + .iter() + .map(|(pid, _)| pid) + .collect::>() + .len() + == count + && markers + .iter() + .map(|(_, id)| id) + .collect::>() + .len() + == count, + "handlers did not consume independent invocations" + ); + return Ok(markers); + } + sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("two handlers did not consume before release")? +} +async fn gated(client: &Client, server: &Server, stop: bool) -> Result<()> { + let directory = deploy_gate(server, &lambda(server), stop).await?; + let function = if stop { + "SdkFanoutStop" + } else { + "SdkFanoutGate" + }; + let task = json!({"StartAt":"Call","States":{"Call":{"Type":"Task","Resource":format!("arn:aws:lambda:us-east-1:000000000000:function:{function}"),"End":true}}}); + let definition = if stop { + json!({"StartAt":"P","States":{"P":{"Type":"Parallel","Branches":[{"StartAt":"M","States":{"M":{"Type":"Map","MaxConcurrency":2,"ItemProcessor":task,"End":true}}}],"End":true}}}) + } else { + json!({"StartAt":"P","States":{"P":{"Type":"Parallel","Branches":[task.clone(),task],"End":true}}}) + }; + let machine = create( + client, + if stop { + "SdkStructuredStop" + } else { + "SdkStructuredOverlap" + }, + &definition, + ) + .await?; + let started = client + .start_execution() + .state_machine_arn(machine) + .name("Run") + .input(if stop { + r#"[{"n":21},{"n":7}]"# + } else { + r#"{"n":21}"# + }) + .send() + .await?; + let markers = consumed(&directory, 2).await?; + if stop { + finish_stopped(client, started.execution_arn(), markers).await + } else { + fs::write(directory.join("release"), b"release").await?; + finish_parallel(client, started.execution_arn(), markers).await + } +} +async fn finish_stopped( + client: &Client, + arn: &str, + markers: BTreeSet<(u32, String)>, +) -> Result<()> { + client + .stop_execution() + .execution_arn(arn) + .error("SdkStructuredStop") + .cause("consumed nested tasks") + .send() + .await?; + let result = terminal(client, arn).await?; + ensure!( + result.status() == &ExecutionStatus::Aborted && result.error() == Some("SdkStructuredStop"), + "nested stop did not settle" + ); + for (pid, _) in markers { + timeout(Duration::from_secs(5), async { + loop { + if assert_absent(pid).await.is_ok() { + return; + } + sleep(Duration::from_millis(20)).await; + } + }) + .await + .context("stopped handler PID survived")?; + } + let events = check_history(client, arn, "MapIterationAborted").await?; + let aborted: Vec<_> = events + .iter() + .filter(|e| e.r#type().as_str() == "MapIterationAborted") + .collect(); + ensure!( + aborted.len() == 2 + && aborted + .iter() + .filter_map(|e| e + .map_iteration_aborted_event_details() + .map(MapIterationEventDetails::index)) + .collect::>() + == BTreeSet::from([0, 1]), + "missing stopped iteration identities" + ); + ensure!( + events + .last() + .is_some_and(|e| e.r#type().as_str() == "ExecutionAborted"), + "iteration aborts were not committed before root termination" + ); + Ok(()) +} +async fn finish_parallel( + client: &Client, + arn: &str, + markers: BTreeSet<(u32, String)>, +) -> Result<()> { + let result = terminal(client, arn).await?; + ensure!( + result.status() == &ExecutionStatus::Succeeded, + "parallel gate failed" + ); + let output: Value = serde_json::from_str(result.output().context("parallel output")?)?; + let values = output.as_array().context("parallel array")?; + ensure!(values.len() == 2, "wrong parallel result length"); + let mut returned = BTreeSet::new(); + for value in values { + ensure!( + value["computed"] == 42, + "actual handler computation changed" + ); + returned.insert(( + u32::try_from(value["pid"].as_u64().context("PID")?)?, + value["request_id"] + .as_str() + .context("request ID")? + .to_owned(), + )); + } + ensure!( + returned == markers, + "returned invocations differ from consumed handlers" + ); + let events = check_history(client, arn, "ParallelStateSucceeded").await?; + let fork = events + .iter() + .find(|e| e.r#type().as_str() == "ParallelStateStarted") + .context("missing fork")? + .id(); + let entries: Vec<_> = events + .iter() + .filter(|e| e.r#type().as_str() == "TaskStateEntered") + .collect(); + ensure!( + entries.len() == 2 && entries.iter().all(|e| e.previous_event_id() == fork), + "concurrent branches do not share their actual fork predecessor" + ); + Ok(()) +} +pub(super) async fn exercise(client: &Client, server: &Server) -> Result<()> { + cases(client).await?; + gated(client, server, false).await?; + gated(client, server, true).await +} diff --git a/assets/lambda/runtime/Dockerfile b/assets/lambda/runtime/Dockerfile new file mode 100644 index 00000000..4fa08b2a --- /dev/null +++ b/assets/lambda/runtime/Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/lambda/provided:al2023@sha256:612c98f126c2a95e77a016d56fed50a83055e11d636132ee6ebdd461f01b9617 +COPY bootstrap /rustack-lambda-runtime +USER 1000:1000 +ENTRYPOINT ["/rustack-lambda-runtime"] +CMD [] diff --git a/codegen/Cargo.lock b/codegen/Cargo.lock index b8f117e6..cf9526a7 100644 --- a/codegen/Cargo.lock +++ b/codegen/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "equivalent" version = "1.0.2" @@ -48,6 +54,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -74,6 +86,7 @@ dependencies = [ "heck", "serde", "serde_json", + "tokio", "toml", ] @@ -104,7 +117,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -140,6 +153,39 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 018d838c..5da65e54 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -2,6 +2,7 @@ name = "rustack-codegen" version = "0.1.1" edition = "2024" +license = "MIT" [dependencies] serde = { version = "1.0", features = ["derive"] } @@ -9,3 +10,5 @@ serde_json = "1.0" toml = "0.9" heck = "0.5" anyhow = "1.0" + +tokio = { version = "~1.54.1", features = ["rt", "macros", "fs", "time", "io-util"] } diff --git a/codegen/services/stepfunctions.toml b/codegen/services/stepfunctions.toml new file mode 100644 index 00000000..60ce94ec --- /dev/null +++ b/codegen/services/stepfunctions.toml @@ -0,0 +1,67 @@ +[service] +name = "stepfunctions" +display_name = "Step Functions" +rust_prefix = "StepFunctions" +namespace = "com.amazonaws.sfn" +protocol = "awsJson1_0" + +[protocol] +serde_rename = "camelCase" +target_prefix = "AWSStepFunctions" +emit_serde_derives = true + +[operations] +all = [ + "CreateActivity", + "CreateStateMachine", + "CreateStateMachineAlias", + "DeleteActivity", + "DeleteStateMachine", + "DeleteStateMachineAlias", + "DeleteStateMachineVersion", + "DescribeActivity", + "DescribeExecution", + "DescribeMapRun", + "DescribeStateMachine", + "DescribeStateMachineAlias", + "DescribeStateMachineForExecution", + "GetActivityTask", + "GetExecutionHistory", + "ListActivities", + "ListExecutions", + "ListMapRuns", + "ListStateMachineAliases", + "ListStateMachines", + "ListStateMachineVersions", + "ListTagsForResource", + "PublishStateMachineVersion", + "RedriveExecution", + "SendTaskFailure", + "SendTaskHeartbeat", + "SendTaskSuccess", + "StartExecution", + "StartSyncExecution", + "StopExecution", + "TagResource", + "TestState", + "UntagResource", + "UpdateMapRun", + "UpdateStateMachine", + "UpdateStateMachineAlias", + "ValidateStateMachineDefinition", +] + +[errors.custom] +AccessDeniedException = { status = 403, message = "Request scope is not authorized" } +InternalFailure = { status = 500, message = "Request processing failed" } +RequestTimeoutException = { status = 408, message = "Request deadline exceeded" } +ServiceUnavailable = { status = 503, message = "Workflow API unavailable" } +ThrottlingException = { status = 400, message = "Workflow API admission exhausted" } +UnknownOperationException = { status = 404, message = "Unknown Step Functions operation" } +UnsupportedOperation = { status = 400, message = "Operation is outside the supported Step Functions profile" } + +[output] +file_layout = "flat" +always_serialize_arrays = true + +strict_model = true diff --git a/codegen/smithy-model/stepfunctions.json b/codegen/smithy-model/stepfunctions.json new file mode 100644 index 00000000..7fba49cf --- /dev/null +++ b/codegen/smithy-model/stepfunctions.json @@ -0,0 +1,8218 @@ +{ + "smithy": "2.0", + "metadata": { + "suppressions": [ + { + "id": "HttpMethodSemantics", + "namespace": "*" + }, + { + "id": "HttpResponseCodeSemantics", + "namespace": "*" + }, + { + "id": "PaginatedTrait", + "namespace": "*" + }, + { + "id": "HttpHeaderTrait", + "namespace": "*" + }, + { + "id": "HttpUriConflict", + "namespace": "*" + }, + { + "id": "Service", + "namespace": "*" + } + ] + }, + "shapes": { + "com.amazonaws.sfn#AWSStepFunctions": { + "type": "service", + "version": "2016-11-23", + "operations": [ + { + "target": "com.amazonaws.sfn#CreateActivity" + }, + { + "target": "com.amazonaws.sfn#CreateStateMachine" + }, + { + "target": "com.amazonaws.sfn#CreateStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DeleteActivity" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachine" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachineVersion" + }, + { + "target": "com.amazonaws.sfn#DescribeActivity" + }, + { + "target": "com.amazonaws.sfn#DescribeExecution" + }, + { + "target": "com.amazonaws.sfn#DescribeMapRun" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachine" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecution" + }, + { + "target": "com.amazonaws.sfn#GetActivityTask" + }, + { + "target": "com.amazonaws.sfn#GetExecutionHistory" + }, + { + "target": "com.amazonaws.sfn#ListActivities" + }, + { + "target": "com.amazonaws.sfn#ListExecutions" + }, + { + "target": "com.amazonaws.sfn#ListMapRuns" + }, + { + "target": "com.amazonaws.sfn#ListStateMachineAliases" + }, + { + "target": "com.amazonaws.sfn#ListStateMachines" + }, + { + "target": "com.amazonaws.sfn#ListStateMachineVersions" + }, + { + "target": "com.amazonaws.sfn#ListTagsForResource" + }, + { + "target": "com.amazonaws.sfn#PublishStateMachineVersion" + }, + { + "target": "com.amazonaws.sfn#RedriveExecution" + }, + { + "target": "com.amazonaws.sfn#SendTaskFailure" + }, + { + "target": "com.amazonaws.sfn#SendTaskHeartbeat" + }, + { + "target": "com.amazonaws.sfn#SendTaskSuccess" + }, + { + "target": "com.amazonaws.sfn#StartExecution" + }, + { + "target": "com.amazonaws.sfn#StartSyncExecution" + }, + { + "target": "com.amazonaws.sfn#StopExecution" + }, + { + "target": "com.amazonaws.sfn#TagResource" + }, + { + "target": "com.amazonaws.sfn#TestState" + }, + { + "target": "com.amazonaws.sfn#UntagResource" + }, + { + "target": "com.amazonaws.sfn#UpdateMapRun" + }, + { + "target": "com.amazonaws.sfn#UpdateStateMachine" + }, + { + "target": "com.amazonaws.sfn#UpdateStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinition" + } + ], + "traits": { + "aws.api#service": { + "sdkId": "SFN", + "arnNamespace": "states", + "cloudFormationName": "StepFunctions", + "cloudTrailEventSource": "states.amazonaws.com", + "docId": "states-2016-11-23", + "endpointPrefix": "states" + }, + "aws.auth#sigv4": { + "name": "states" + }, + "aws.protocols#awsJson1_0": {}, + "smithy.api#documentation": "Step Functions\n

With Step Functions, you can create workflows, also called state machines, to build distributed applications, automate processes, orchestrate microservices, and create data and machine learning pipelines.

\n

Through the Step Functions API, you can create, list, update, and delete state machines, activities, and other data types. You can start, stop, and redrive your state machines. Your activity workers can send task success, heartbeat, and failure responses.

\n

With API calls, you can also manage other aspects of your workflow, such as tags, versions, and aliases.

\n

For more information about developing solutions with Step Functions, see the \n Step Functions Developer Guide\n .

\n \n

If you use the Step Functions API actions using Amazon Web Services SDK integrations, make sure the API actions are in camel case and parameter names are in Pascal case. For example, you might use Step Functions API action startSyncExecution and specify its parameter as StateMachineArn.

\n
", + "smithy.api#title": "AWS Step Functions", + "smithy.api#xmlNamespace": { + "uri": "http://swf.amazonaws.com/doc/2015-07-20/" + }, + "smithy.rules#endpointBdd": { + "version": "1.1", + "parameters": { + "UseDualStack": { + "builtIn": "AWS::UseDualStack", + "required": true, + "default": false, + "documentation": "When true, use the dual-stack endpoint. If the configured endpoint does not support dual-stack, dispatching the request MAY return an error.", + "type": "boolean" + }, + "UseFIPS": { + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "When true, send this request to the FIPS-compliant regional endpoint. If the configured endpoint does not have a FIPS compliant endpoint, dispatching the request will return an error.", + "type": "boolean" + }, + "Endpoint": { + "builtIn": "SDK::Endpoint", + "required": false, + "documentation": "Override the endpoint used to send this request", + "type": "string" + }, + "Region": { + "builtIn": "AWS::Region", + "required": false, + "documentation": "The AWS region used to dispatch the request.", + "type": "string" + } + }, + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Endpoint" + } + ] + }, + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + }, + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + }, + { + "fn": "stringEquals", + "argv": [ + { + "ref": "Region" + }, + "us-gov-west-1" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + }, + true + ] + }, + { + "fn": "stringEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "name" + ] + }, + "aws-us-gov" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + }, + true + ] + } + ], + "results": [ + { + "conditions": [], + "error": "Invalid Configuration: FIPS and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": { + "ref": "Endpoint" + }, + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "FIPS and DualStack are enabled, but this partition does not support one or both", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "FIPS is enabled but this partition does not support FIPS", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "DualStack is enabled but this partition does not support DualStack", + "type": "error" + }, + { + "conditions": [], + "error": "Invalid Configuration: Missing Region", + "type": "error" + } + ], + "root": 2, + "nodeCount": 15, + "nodes": "/////wAAAAH/////AAAAAAAAAA4AAAADAAAAAQAAAAQF9eELAAAAAgAAAAUF9eELAAAAAwAAAAkAAAAGAAAABAAAAAcF9eEEAAAABwX14QUAAAAIAAAACAX14QkF9eEKAAAABAAAAAwAAAAKAAAABQX14QQAAAALAAAABgX14QcF9eEIAAAABgAAAA0F9eEGAAAACAX14QUF9eEGAAAAAwX14QEAAAAPAAAABAX14QIF9eED" + }, + "smithy.rules#endpointRuleSet": { + "version": "1.0", + "parameters": { + "UseDualStack": { + "builtIn": "AWS::UseDualStack", + "required": true, + "default": false, + "documentation": "When true, use the dual-stack endpoint. If the configured endpoint does not support dual-stack, dispatching the request MAY return an error.", + "type": "boolean" + }, + "UseFIPS": { + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "When true, send this request to the FIPS-compliant regional endpoint. If the configured endpoint does not have a FIPS compliant endpoint, dispatching the request will return an error.", + "type": "boolean" + }, + "Endpoint": { + "builtIn": "SDK::Endpoint", + "required": false, + "documentation": "Override the endpoint used to send this request", + "type": "string" + }, + "Region": { + "builtIn": "AWS::Region", + "required": false, + "documentation": "The AWS region used to dispatch the request.", + "type": "string" + } + }, + "rules": [ + { + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Endpoint" + } + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + } + ], + "error": "Invalid Configuration: FIPS and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": { + "ref": "Endpoint" + }, + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + } + ], + "type": "tree" + }, + { + "conditions": [], + "rules": [ + { + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "stringEquals", + "argv": [ + { + "ref": "Region" + }, + "us-gov-west-1" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + false + ] + } + ], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [ + { + "fn": "stringEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "name" + ] + }, + "aws-us-gov" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + false + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + } + ] + }, + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + } + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "FIPS and DualStack are enabled, but this partition does not support one or both", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + false + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + }, + true + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "FIPS is enabled but this partition does not support FIPS", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + false + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + } + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "DualStack is enabled but this partition does not support DualStack", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "Invalid Configuration: Missing Region", + "type": "error" + } + ], + "type": "tree" + } + ] + }, + "smithy.rules#endpointTests": { + "testCases": [ + { + "documentation": "For custom endpoint with region not set and fips disabled", + "expect": { + "endpoint": { + "url": "https://example.com" + } + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": false + } + }, + { + "documentation": "For custom endpoint with fips enabled", + "expect": { + "error": "Invalid Configuration: FIPS and custom endpoint are not supported" + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": true + } + }, + { + "documentation": "For custom endpoint with fips disabled and dualstack enabled", + "expect": { + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported" + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-gov-west-1.amazonaws.com" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-east-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-east-1.api.aws" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region us-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-east-1.amazonaws.com" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-east-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states.us-east-1.api.aws" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-east-1.amazonaws.com" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.cn-northwest-1.api.amazonwebservices.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.cn-northwest-1.amazonaws.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states.cn-northwest-1.api.amazonwebservices.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.cn-northwest-1.amazonaws.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region eusc-de-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.eusc-de-east-1.amazonaws.eu" + } + }, + "params": { + "Region": "eusc-de-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region eusc-de-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.eusc-de-east-1.amazonaws.eu" + } + }, + "params": { + "Region": "eusc-de-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-iso-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-iso-east-1.c2s.ic.gov" + } + }, + "params": { + "Region": "us-iso-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-iso-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-iso-east-1.c2s.ic.gov" + } + }, + "params": { + "Region": "us-iso-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isob-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-isob-east-1.sc2s.sgov.gov" + } + }, + "params": { + "Region": "us-isob-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isob-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-isob-east-1.sc2s.sgov.gov" + } + }, + "params": { + "Region": "us-isob-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region eu-isoe-west-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.eu-isoe-west-1.cloud.adc-e.uk" + } + }, + "params": { + "Region": "eu-isoe-west-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region eu-isoe-west-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.eu-isoe-west-1.cloud.adc-e.uk" + } + }, + "params": { + "Region": "eu-isoe-west-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isof-south-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-isof-south-1.csp.hci.ic.gov" + } + }, + "params": { + "Region": "us-isof-south-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isof-south-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-isof-south-1.csp.hci.ic.gov" + } + }, + "params": { + "Region": "us-isof-south-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-gov-west-1.api.aws" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-gov-west-1.api.aws" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-gov-west-1.amazonaws.com" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "Missing region", + "expect": { + "error": "Invalid Configuration: Missing Region" + } + } + ], + "version": "1.0" + } + } + }, + "com.amazonaws.sfn#ActivityAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Activity already exists. EncryptionConfiguration may not be updated.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified activity does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity that failed during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of activities has been reached. Existing activities must be deleted\n before a new activity can be created.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ActivityListItem" + } + }, + "com.amazonaws.sfn#ActivityListItem": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the activity.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity.

" + } + }, + "com.amazonaws.sfn#ActivityScheduleFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity schedule failure that occurred during an\n execution.

" + } + }, + "com.amazonaws.sfn#ActivityScheduledEventDetails": { + "type": "structure", + "members": { + "resource": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the scheduled activity.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the activity task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the activity task.

" + } + }, + "heartbeatInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration between two heartbeats for the activity task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityStartedEventDetails": { + "type": "structure", + "members": { + "workerName": { + "target": "com.amazonaws.sfn#Identity", + "traits": { + "smithy.api#documentation": "

The name of the worker that the task is assigned to. These names are provided by the\n workers when calling GetActivityTask.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of an activity during an execution.

" + } + }, + "com.amazonaws.sfn#ActivitySucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the activity task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity that successfully terminated during an\n execution.

" + } + }, + "com.amazonaws.sfn#ActivityTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityWorkerLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of workers concurrently polling for activity tasks has been\n reached.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#AliasDescription": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#Arn": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 256 + } + } + }, + "com.amazonaws.sfn#AssignedVariables": { + "type": "map", + "key": { + "target": "com.amazonaws.sfn#VariableName" + }, + "value": { + "target": "com.amazonaws.sfn#VariableValue" + } + }, + "com.amazonaws.sfn#AssignedVariablesDetails": { + "type": "structure", + "members": { + "truncated": { + "target": "com.amazonaws.sfn#truncated", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether assigned variables were truncated in the response. Always false\n for API calls. In CloudWatch logs, the value will be true if the data is truncated due to size limits.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about assigned variables in an execution history event.

" + } + }, + "com.amazonaws.sfn#BilledDuration": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#BilledMemoryUsed": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#BillingDetails": { + "type": "structure", + "members": { + "billedMemoryUsedInMB": { + "target": "com.amazonaws.sfn#BilledMemoryUsed", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Billed memory consumption of your workflow, in MB.

" + } + }, + "billedDurationInMilliseconds": { + "target": "com.amazonaws.sfn#BilledDuration", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Billed duration of your workflow, in milliseconds.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

An object that describes workflow billing details.

" + } + }, + "com.amazonaws.sfn#CharacterRestrictedName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + }, + "smithy.api#pattern": "^(?=.*[a-zA-Z_\\-\\.])[a-zA-Z0-9_\\-\\.]+$" + } + }, + "com.amazonaws.sfn#ClientToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 64 + }, + "smithy.api#pattern": "^[!-~]+$" + } + }, + "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails": { + "type": "structure", + "members": { + "included": { + "target": "com.amazonaws.sfn#includedDetails", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether input or output was included in the response. Always true\n for API calls.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about execution input or output.

" + } + }, + "com.amazonaws.sfn#CloudWatchLogsLogGroup": { + "type": "structure", + "members": { + "logGroupArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The ARN of the the CloudWatch log group to which you want your logs emitted to. The ARN\n must end with :*\n

" + } + } + }, + "traits": { + "smithy.api#documentation": "

" + } + }, + "com.amazonaws.sfn#ConflictException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Updating or deleting a resource can cause an inconsistent state. This error occurs when there're concurrent requests for DeleteStateMachineVersion, PublishStateMachineVersion, or UpdateStateMachine with the publish parameter set to true.

\n

HTTP Status Code: 409

", + "smithy.api#error": "client", + "smithy.api#httpError": 409 + } + }, + "com.amazonaws.sfn#ConnectorParameters": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 262144 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#CreateActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#ActivityLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + } + ], + "traits": { + "smithy.api#documentation": "

Creates an activity. An activity is a task that you write in any programming language and\n host on any machine that has access to Step Functions. Activities must poll Step Functions using the\n GetActivityTask API action and respond using SendTask* API\n actions. This function lets Step Functions know the existence of your activity and returns an\n identifier for use in a state machine and when polling from the activity.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
\n \n

\n CreateActivity is an idempotent API. Subsequent requests won\u2019t create a\n duplicate resource if it was already created. CreateActivity's idempotency\n check is based on the activity name. If a following request has different\n tags values, Step Functions will ignore these differences and treat it as an\n idempotent request of the previous. In this case, tags will not be updated,\n even if they are different.

\n
", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#CreateActivityInput": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity to create. This name must be unique for your Amazon Web Services account and region.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

The list of tags to add to a resource.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateActivityOutput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created activity.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#CreateStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidLoggingConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#InvalidTracingConfiguration" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates a state machine. A state machine consists of a collection of states that can do\n work (Task states), determine to which states to transition next\n (Choice states), stop an execution with an error (Fail states),\n and so on. State machines are specified using a JSON-based, structured language. For more\n information, see Amazon States\n Language in the Step Functions User Guide.

\n

If you set the publish parameter of this API action to true, it\n publishes version 1 as the first revision of the state machine.

\n

\n For additional control over security, you can encrypt your data using a customer-managed key for Step Functions state machines. You can configure a symmetric KMS key and data key reuse period when creating or updating a State Machine. The execution history and state machine definition will be encrypted with the key applied to the State Machine.\n

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
\n \n

\n CreateStateMachine is an idempotent API. Subsequent requests won\u2019t create a\n duplicate resource if it was already created. CreateStateMachine's idempotency\n check is based on the state machine name, definition,\n type, LoggingConfiguration,\n TracingConfiguration, and EncryptionConfiguration The check is also based on the publish and versionDescription parameters. If a following request has a different\n roleArn or tags, Step Functions will ignore these differences and treat\n it as an idempotent request of the previous. In this case, roleArn and\n tags will not be updated, even if they are different.

\n
", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates an alias for a state machine that points to one or two versions of the same state machine. You can set your application to call StartExecution with an alias and update the version the alias uses without changing the client's code.

\n

You can also map an alias to split StartExecution requests between two\n versions of a state machine. To do this, add a second RoutingConfig object in the\n routingConfiguration parameter. You must also specify the percentage of\n execution run requests each version should receive in both RoutingConfig objects.\n Step Functions randomly chooses which version runs a given execution based on the\n percentage you specify.

\n

To create an alias that points to a single version, specify a single\n RoutingConfig object with a weight set to 100.

\n

You can create up to 100 aliases for each state machine. You must delete unused aliases using the DeleteStateMachineAlias API action.

\n

\n CreateStateMachineAlias is an idempotent API. Step Functions bases the\n idempotency check on the stateMachineArn, description,\n name, and routingConfiguration parameters. Requests that contain\n the same values for these parameters return a successful idempotent response without creating\n a duplicate resource.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#CreateStateMachineAliasInput": { + "type": "structure", + "members": { + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description for the state machine alias.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#CharacterRestrictedName", + "traits": { + "smithy.api#documentation": "

The name of the state machine alias.

\n

To avoid conflict with version ARNs, don't use an integer in the name of the alias.

", + "smithy.api#required": {} + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of a state machine alias. The routing configuration shifts\n execution traffic between two state machine versions. routingConfiguration\n contains an array of RoutingConfig objects that specify up to two state machine\n versions. Step Functions then randomly choses which version to run an execution with based\n on the weight assigned to each RoutingConfig.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineAliasOutput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine alias.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineInput": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role to use for this state machine.

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

Determines whether a Standard or Express state machine is created. The default is\n STANDARD. You cannot update the type of a state machine once it\n has been created.

" + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration", + "traits": { + "smithy.api#documentation": "

Defines what execution history events are logged and where they are logged.

\n \n

By default, the level is set to OFF. For more information see\n Log\n Levels in the Step Functions User Guide.

\n
" + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

Tags to be added when creating a state machine.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "publish": { + "target": "com.amazonaws.sfn#Publish", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Set to true to publish the first version of the state machine during creation. The default is false.

" + } + }, + "versionDescription": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

Sets description about the state machine version. You can only set the description if the publish parameter is set to true. Otherwise, if you set versionDescription, but publish to false, this API action throws ValidationException.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

", + "smithy.api#required": {} + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine version. If you do not set the publish parameter to true, this field returns null value.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#Definition": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 1048576 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#DeleteActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes an activity.

" + } + }, + "com.amazonaws.sfn#DeleteActivityInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteActivityOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine. This is an asynchronous operation. It sets the state machine's\n status to DELETING and begins the deletion process. A state machine is deleted only when all its executions are completed. On the next state transition, the state machine's executions are terminated.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n
    \n
  • \n

    The following qualified state machine ARN refers to a Distributed Map state with a label mapStateLabel in a state machine named myStateMachine.

    \n

    \n arn:partition:states:region:account-id:stateMachine:myStateMachine/mapStateLabel\n

    \n \n

    If you provide a qualified state machine ARN that refers to a Distributed Map state, the request fails with ValidationException.

    \n
    \n
  • \n
  • \n

    The following unqualified state machine ARN refers to a state machine named myStateMachine.

    \n

    \n arn:partition:states:region:account-id:stateMachine:myStateMachine\n

    \n
  • \n
\n

This API action also deletes all versions and aliases associated with a state machine.

\n \n

For EXPRESS state machines, the deletion happens eventually (usually in\n less than a minute). Running executions may emit logs after DeleteStateMachine\n API is called.

\n
" + } + }, + "com.amazonaws.sfn#DeleteStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine alias.

\n

After you delete a state machine alias, you can't use it to start executions. When you\n delete a state machine alias, Step Functions doesn't delete the state machine versions\n that alias references.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DeleteStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineAliasOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersion": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineVersionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineVersionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine version. After\n you delete a version, you can't call StartExecution using that version's ARN\n or use the version with a state machine alias.

\n \n

Deleting a state machine version won't terminate its in-progress executions.

\n
\n \n

You can't delete a state machine version currently referenced by one or more aliases. Before you delete a version, you must either delete the aliases or update them to point to another state machine version.

\n
\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersionInput": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersionOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + } + ], + "traits": { + "smithy.api#documentation": "

Describes an activity.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
" + } + }, + "com.amazonaws.sfn#DescribeActivityInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to describe.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeActivityOutput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the activity.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings for configured server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine execution, such as the state machine associated with the execution, the execution input and output, and relevant execution metadata. If you've redriven an execution, you can use this API action to return information about the redrives of that execution. In addition, you can use this API action to return the Map Run Amazon Resource Name (ARN) if the execution was dispatched by a Map Run.

\n

If you specify a version or alias ARN when you call the StartExecution\n API action, DescribeExecution returns that ARN.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
\n

Executions of an EXPRESS state machine aren't supported by DescribeExecution unless a Map Run dispatched them.

" + } + }, + "com.amazonaws.sfn#DescribeExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to describe.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call DescribeStateMachine API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the executed stated machine.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

" + } + }, + "status": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution ended, the date the execution stopped.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

\n \n

This field is set only if the execution succeeds. If the execution fails, this field is\n null.

\n
" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

The X-Ray trace header that was passed to the execution.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n
" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run, which dispatched this execution.

" + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error string if the state machine execution failed.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

The cause string if the state machine execution failed.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version associated with the execution. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

\n

If you start an execution from a StartExecution request without specifying a\n state machine version or alias ARN, Step Functions returns a null value.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias associated with the execution. The alias ARN is a combination of state machine ARN and the alias name separated by a colon (:). For example, stateMachineARN:PROD.

\n

If you start an execution from a StartExecution request with a\n state machine version ARN, this field will be null.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is only updated if you successfully redrive an execution.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven. If you have not yet redriven an execution, the redriveDate is null.

\n

The redriveDate is unavailable if you redrive a Map Run that starts child workflow executions of type EXPRESS.

" + } + }, + "redriveStatus": { + "target": "com.amazonaws.sfn#ExecutionRedriveStatus", + "traits": { + "smithy.api#documentation": "

Indicates whether or not an execution can be redriven at a given point in time.

\n
    \n
  • \n

    For executions of type STANDARD, redriveStatus is NOT_REDRIVABLE if calling the RedriveExecution API action would return the ExecutionNotRedrivable error.

    \n
  • \n
  • \n

    For a Distributed Map that includes child workflows of type STANDARD, redriveStatus indicates whether or not the Map Run can redrive child workflow executions.

    \n
  • \n
  • \n

    For a Distributed Map that includes child workflows of type EXPRESS, redriveStatus indicates whether or not the Map Run can redrive child workflow executions.

    \n

    You can redrive failed or timed out EXPRESS workflows only if they're a part of a Map Run. When you redrive the Map Run, these workflows are restarted using the StartExecution API action.

    \n
  • \n
" + } + }, + "redriveStatusReason": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

When redriveStatus is NOT_REDRIVABLE, redriveStatusReason specifies the reason why an execution cannot be redriven.

\n
    \n
  • \n

    For executions of type STANDARD, or for a Distributed Map that includes child workflows of type STANDARD, redriveStatusReason can include one of the following reasons:

    \n
      \n
    • \n

      \n State machine is in DELETING status.

      \n
    • \n
    • \n

      \n Execution is RUNNING and cannot be redriven.

      \n
    • \n
    • \n

      \n Execution is SUCCEEDED and cannot be redriven.

      \n
    • \n
    • \n

      \n Execution was started before the launch of RedriveExecution.

      \n
    • \n
    • \n

      \n Execution history event limit exceeded.

      \n
    • \n
    • \n

      \n Execution has exceeded the max execution time.

      \n
    • \n
    • \n

      \n Execution redrivable period exceeded.

      \n
    • \n
    \n
  • \n
  • \n

    For a Distributed Map that includes child workflows of type EXPRESS, redriveStatusReason is only returned if the child workflows are not redrivable. This happens when the child workflow executions have completed successfully.

    \n
  • \n
" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeMapRun": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeMapRunInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeMapRunOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a Map Run's configuration, progress, and results. If you've redriven a Map Run, this API action also returns information about the redrives of that Map Run. For more information, see Examining Map Run in the Step Functions Developer Guide.

" + } + }, + "com.amazonaws.sfn#DescribeMapRunInput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeMapRunOutput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run.

", + "smithy.api#required": {} + } + }, + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution in which the Map Run was started.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#MapRunStatus", + "traits": { + "smithy.api#documentation": "

The current status of the Map Run.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date when the Map Run was started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date when the Map Run was stopped.

" + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#MaxConcurrency", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of child workflow executions configured to run in parallel for the Map Run at the same time.

", + "smithy.api#required": {} + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#ToleratedFailurePercentage", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum percentage of failed child workflow executions before the Map Run fails.

", + "smithy.api#required": {} + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#ToleratedFailureCount", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of failed child workflow executions before the Map Run fails.

", + "smithy.api#required": {} + } + }, + "itemCounts": { + "target": "com.amazonaws.sfn#MapRunItemCounts", + "traits": { + "smithy.api#documentation": "

A JSON object that contains information about the total number of items, and the item count for each processing status, such as pending and failed.

", + "smithy.api#required": {} + } + }, + "executionCounts": { + "target": "com.amazonaws.sfn#MapRunExecutionCounts", + "traits": { + "smithy.api#documentation": "

A JSON object that contains information about the total number of child workflow executions for the Map Run, and the count of child workflow executions for each status, such as failed and succeeded.

", + "smithy.api#required": {} + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven a Map Run. If you have not yet redriven a Map Run, the redriveCount is 0. This count is only updated if you successfully redrive a Map Run.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date a Map Run was last redriven. If you have not yet redriven a Map Run, the redriveDate is null.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine's definition, its IAM role Amazon Resource Name (ARN), and configuration.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n
    \n
  • \n

    The following qualified state machine ARN refers to a Distributed Map state with a label mapStateLabel in a state machine named myStateMachine.

    \n

    \n arn:partition:states:region:account-id:stateMachine:myStateMachine/mapStateLabel\n

    \n \n

    If you provide a qualified state machine ARN that refers to a Distributed Map state, the request fails with ValidationException.

    \n
    \n
  • \n
  • \n

    The following qualified state machine ARN refers to an alias named PROD.

    \n

    \n arn::states:::stateMachine:\n

    \n \n

    If you provide a qualified state machine ARN that refers to a version ARN or an alias ARN, the request starts execution for that version or alias.

    \n
    \n
  • \n
  • \n

    The following unqualified state machine ARN refers to a state machine named myStateMachine.

    \n

    \n arn::states:::stateMachine:\n

    \n
  • \n
\n

This API action returns the details for a state machine version if the\n stateMachineArn you specify is a state machine version ARN.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
" + } + }, + "com.amazonaws.sfn#DescribeStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Returns details about a state machine alias.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DescribeStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineAliasOutput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine alias.

" + } + }, + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description of the alias.

" + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of the alias.

" + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was created.

" + } + }, + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was last updated.

\n

For a newly created state machine, this is the same as the creation date.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine's definition, its execution role ARN, and\n configuration. If a Map Run dispatched the execution, this action returns the Map Run\n Amazon Resource Name (ARN) in the response. The state machine returned is the state machine associated with the\n Map Run.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
\n

This API action is not supported by EXPRESS state machines.

" + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution you want state machine information for.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecutionOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine associated with the execution.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine associated with the execution.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role of the State Machine for the execution.

", + "smithy.api#required": {} + } + }, + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine associated with an execution was updated. For a newly\n created state machine, this is the creation date.

", + "smithy.api#required": {} + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration" + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run that started the child workflow execution. This field is returned only if the executionArn is a child workflow execution that was started by a Distributed Map state.

" + } + }, + "label": { + "target": "com.amazonaws.sfn#MapRunLabel", + "traits": { + "smithy.api#documentation": "

A user-defined or an auto-generated string that identifies a Map state. This field is returned only if the executionArn is a child workflow execution that was started by a Distributed Map state.

" + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the state machine. The first revision ID when you create the state machine is null.

\n

Use the state machine revisionId parameter to compare the revision of a state machine with the configuration of the state machine used for executions without performing a diff of the properties, such as definition and roleArn.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + }, + "variableReferences": { + "target": "com.amazonaws.sfn#VariableReferences", + "traits": { + "smithy.api#documentation": "

A map of state name to a list of variables referenced by that state. States that do not use variable references will not be shown in the response.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine for which you want the information.

\n

If you specify a state machine version ARN, this API returns details about that version. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

\n \n

\n When calling a labelled ARN for an encrypted state machine, the includedData = METADATA_ONLY parameter will not apply because Step Functions needs to decrypt the entire state machine definition to get the Distributed Map state\u2019s definition. In this case, the API caller needs to have kms:Decrypt permission.\n

\n
" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

\n

If you specified a state machine version ARN in your request, the API returns the version ARN. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#StateMachineStatus", + "traits": { + "smithy.api#documentation": "

The current status of the state machine.

" + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

\n

If called with includedData = METADATA_ONLY, the returned definition will be {}.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role used when creating this state machine. (The IAM role\n maintains security by granting Step Functions access to Amazon Web Services resources.)

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

The type of the state machine (STANDARD or\n EXPRESS).

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

\n

For a state machine version, creationDate is the date the version was created.

", + "smithy.api#required": {} + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration" + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "label": { + "target": "com.amazonaws.sfn#MapRunLabel", + "traits": { + "smithy.api#documentation": "

A user-defined or an auto-generated string that identifies a Map state. This parameter is present only if the stateMachineArn specified in input is a qualified state machine ARN.

" + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the state machine.

\n

Use the revisionId parameter to compare between versions of a state machine\n configuration used for executions without performing a diff of the properties, such as\n definition and roleArn.

" + } + }, + "description": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

The description of the state machine version.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + }, + "variableReferences": { + "target": "com.amazonaws.sfn#VariableReferences", + "traits": { + "smithy.api#documentation": "

A map of state name to a list of variables referenced by that state. States that do not use variable references will not be shown in the response.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#Enabled": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#EncryptionConfiguration": { + "type": "structure", + "members": { + "kmsKeyId": { + "target": "com.amazonaws.sfn#KmsKeyId", + "traits": { + "smithy.api#documentation": "

An alias, alias ARN, key ID, or key ARN of a symmetric encryption KMS key to encrypt data. To specify a KMS key in a different Amazon Web Services account, you must use the key ARN or alias ARN.

" + } + }, + "kmsDataKeyReusePeriodSeconds": { + "target": "com.amazonaws.sfn#KmsDataKeyReusePeriodSeconds", + "traits": { + "smithy.api#documentation": "

Maximum duration that Step Functions will reuse data keys. When the period expires, Step Functions will call GenerateDataKey. Only applies to customer managed keys.

" + } + }, + "type": { + "target": "com.amazonaws.sfn#EncryptionType", + "traits": { + "smithy.api#documentation": "

Encryption type

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

\n

\n For additional control over security, you can encrypt your data using a customer-managed key for Step Functions state machines and activities. You can configure a symmetric KMS key and data key reuse period when creating or updating a State Machine, and when creating an Activity. The execution history and state machine definition will be encrypted with the key applied to the State Machine. Activity inputs will be encrypted with the key applied to the Activity.\n

\n \n

Step Functions automatically enables encryption at rest using Amazon Web Services owned keys at no charge. However, KMS charges apply when using a customer managed key. For more information about pricing, see Key Management Service pricing.

\n
\n

For more information on KMS, see What is Key Management Service?\n

" + } + }, + "com.amazonaws.sfn#EncryptionType": { + "type": "enum", + "members": { + "AWS_OWNED_KEY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "AWS_OWNED_KEY" + } + }, + "CUSTOMER_MANAGED_KMS_KEY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CUSTOMER_MANAGED_KMS_KEY" + } + } + } + }, + "com.amazonaws.sfn#ErrorMessage": { + "type": "string" + }, + "com.amazonaws.sfn#EvaluationFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + }, + "location": { + "target": "com.amazonaws.sfn#EvaluationFailureLocation", + "traits": { + "smithy.api#documentation": "

The location of the field in the state in which the evaluation error occurred.

" + } + }, + "state": { + "target": "com.amazonaws.sfn#StateName", + "traits": { + "smithy.api#documentation": "

The name of the state in which the evaluation error occurred.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an evaluation failure that occurred while processing a state, for example, when a JSONata expression throws an error. This event will only be present in state machines that have QueryLanguage set to JSONata, or individual states set to JSONata.

" + } + }, + "com.amazonaws.sfn#EvaluationFailureLocation": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#EventId": { + "type": "long", + "traits": { + "smithy.api#default": 0 + } + }, + "com.amazonaws.sfn#ExceptionHandlerIndex": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ExecutionAbortedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an abort of an execution.

" + } + }, + "com.amazonaws.sfn#ExecutionAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The execution has the same name as another execution (but a different\n input).

\n \n

Executions with the same name and input are considered\n idempotent.

\n
", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified execution does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an execution failure event.

" + } + }, + "com.amazonaws.sfn#ExecutionLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of running executions has been reached. Running executions must end or\n be stopped before a new execution can be started.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ExecutionListItem" + } + }, + "com.amazonaws.sfn#ExecutionListItem": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine that ran the execution.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution already ended, the date the execution stopped.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run. This field is returned only if mapRunArn was specified in the ListExecutions API action. If stateMachineArn was specified in ListExecutions, the mapRunArn isn't returned.

" + } + }, + "itemCount": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The total number of items processed in a child workflow execution. This field is returned only if mapRunArn was specified in the ListExecutions API action. If stateMachineArn was specified in ListExecutions, the itemCount field isn't returned.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version associated with the execution.

\n

If the state machine execution was started with an unqualified ARN, it returns null.

\n

If the execution was started using a stateMachineAliasArn, both the stateMachineAliasArn and stateMachineVersionArn parameters contain the respective values.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias used to start an execution.

\n

If the state machine execution was started with an unqualified ARN or a version ARN, it returns null.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is only updated when you successfully redrive an execution.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an execution.

" + } + }, + "com.amazonaws.sfn#ExecutionNotRedrivable": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The execution Amazon Resource Name (ARN) that you specified for executionArn cannot be redriven.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionRedriveFilter": { + "type": "enum", + "members": { + "REDRIVEN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVEN" + } + }, + "NOT_REDRIVEN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NOT_REDRIVEN" + } + } + } + }, + "com.amazonaws.sfn#ExecutionRedriveStatus": { + "type": "enum", + "members": { + "REDRIVABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVABLE" + } + }, + "NOT_REDRIVABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NOT_REDRIVABLE" + } + }, + "REDRIVABLE_BY_MAP_RUN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVABLE_BY_MAP_RUN" + } + } + } + }, + "com.amazonaws.sfn#ExecutionRedrivenEventDetails": { + "type": "structure", + "members": { + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is not updated for redrives that failed to start or are pending to be redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a redriven execution.

" + } + }, + "com.amazonaws.sfn#ExecutionStartedEventDetails": { + "type": "structure", + "members": { + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role used for executing Lambda tasks.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine alias used for starting the state machine execution.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine version used for starting the state machine execution.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of the execution.

" + } + }, + "com.amazonaws.sfn#ExecutionStatus": { + "type": "enum", + "members": { + "RUNNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RUNNING" + } + }, + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "TIMED_OUT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TIMED_OUT" + } + }, + "ABORTED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ABORTED" + } + }, + "PENDING_REDRIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_REDRIVE" + } + } + } + }, + "com.amazonaws.sfn#ExecutionSucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the successful termination of the execution.

" + } + }, + "com.amazonaws.sfn#ExecutionTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the execution timeout that occurred during the execution.

" + } + }, + "com.amazonaws.sfn#GetActivityTask": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#GetActivityTaskInput" + }, + "output": { + "target": "com.amazonaws.sfn#GetActivityTaskOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ActivityWorkerLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Used by workers to retrieve a task (with the specified activity ARN) which has been\n scheduled for execution by a running state machine. This initiates a long poll, where the\n service holds the HTTP connection open and responds as soon as a task becomes available (i.e.\n an execution of a task of this type is needed.) The maximum time the service holds on to the\n request before responding is 60 seconds. If no task is available within 60 seconds, the poll\n returns a taskToken with a null string.

\n \n

Workers should set their client side socket timeout to at least 65 seconds (5 seconds\n higher than the maximum time the service may hold the poll request).

\n

Polling with GetActivityTask can cause latency in some implementations. See\n Avoid\n Latency When Polling for Activity Tasks in the Step Functions Developer Guide.

\n
" + } + }, + "com.amazonaws.sfn#GetActivityTaskInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to retrieve tasks from (assigned when you create the task\n using CreateActivity.)

", + "smithy.api#required": {} + } + }, + "workerName": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

You can provide an arbitrary name in order to identify the worker that the task is\n assigned to. This name is used when it is logged in the execution history.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#GetActivityTaskOutput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

A token that identifies the scheduled task. This token must be copied and included in\n subsequent calls to SendTaskHeartbeat, SendTaskSuccess or\n SendTaskFailure in order to report the progress or completion of the\n task.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveDataJobInput", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#GetExecutionHistory": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#GetExecutionHistoryInput" + }, + "output": { + "target": "com.amazonaws.sfn#GetExecutionHistoryOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Returns the history of the specified execution as a list of events. By default, the\n results are returned in ascending order of the timeStamp of the events. Use the\n reverseOrder parameter to get the latest events first.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

This API action is not supported by EXPRESS state machines.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "events", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#GetExecutionHistoryInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution.

", + "smithy.api#required": {} + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "reverseOrder": { + "target": "com.amazonaws.sfn#ReverseOrder", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Lists events in descending order of their timeStamp.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "includeExecutionData": { + "target": "com.amazonaws.sfn#IncludeExecutionDataGetExecutionHistory", + "traits": { + "smithy.api#documentation": "

You can select whether execution data (input or output of a history event) is returned.\n The default is true.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#GetExecutionHistoryOutput": { + "type": "structure", + "members": { + "events": { + "target": "com.amazonaws.sfn#HistoryEventList", + "traits": { + "smithy.api#documentation": "

The list of events that occurred in the execution.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#HTTPBody": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPHeaders": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPMethod": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPProtocol": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPStatusCode": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPStatusMessage": { + "type": "string" + }, + "com.amazonaws.sfn#HistoryEvent": { + "type": "structure", + "members": { + "timestamp": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the event occurred, expressed in seconds and fractional milliseconds since the Unix epoch, which is defined as January 1, 1970, at 00:00:00 Coordinated Universal Time (UTC).

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#HistoryEventType", + "traits": { + "smithy.api#documentation": "

The type of the event.

", + "smithy.api#required": {} + } + }, + "id": { + "target": "com.amazonaws.sfn#EventId", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The id of the event. Events are numbered sequentially, starting at one.

", + "smithy.api#required": {} + } + }, + "previousEventId": { + "target": "com.amazonaws.sfn#EventId", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The id of the previous event.

" + } + }, + "activityFailedEventDetails": { + "target": "com.amazonaws.sfn#ActivityFailedEventDetails" + }, + "activityScheduleFailedEventDetails": { + "target": "com.amazonaws.sfn#ActivityScheduleFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an activity schedule event that failed during an execution.

" + } + }, + "activityScheduledEventDetails": { + "target": "com.amazonaws.sfn#ActivityScheduledEventDetails" + }, + "activityStartedEventDetails": { + "target": "com.amazonaws.sfn#ActivityStartedEventDetails" + }, + "activitySucceededEventDetails": { + "target": "com.amazonaws.sfn#ActivitySucceededEventDetails" + }, + "activityTimedOutEventDetails": { + "target": "com.amazonaws.sfn#ActivityTimedOutEventDetails" + }, + "taskFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the failure of a task.

" + } + }, + "taskScheduledEventDetails": { + "target": "com.amazonaws.sfn#TaskScheduledEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that was scheduled.

" + } + }, + "taskStartFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskStartFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to start.

" + } + }, + "taskStartedEventDetails": { + "target": "com.amazonaws.sfn#TaskStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that was started.

" + } + }, + "taskSubmitFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskSubmitFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that where the submit failed.

" + } + }, + "taskSubmittedEventDetails": { + "target": "com.amazonaws.sfn#TaskSubmittedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a submitted task.

" + } + }, + "taskSucceededEventDetails": { + "target": "com.amazonaws.sfn#TaskSucceededEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that succeeded.

" + } + }, + "taskTimedOutEventDetails": { + "target": "com.amazonaws.sfn#TaskTimedOutEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that timed out.

" + } + }, + "executionFailedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionFailedEventDetails" + }, + "executionStartedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionStartedEventDetails" + }, + "executionSucceededEventDetails": { + "target": "com.amazonaws.sfn#ExecutionSucceededEventDetails" + }, + "executionAbortedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionAbortedEventDetails" + }, + "executionTimedOutEventDetails": { + "target": "com.amazonaws.sfn#ExecutionTimedOutEventDetails" + }, + "executionRedrivenEventDetails": { + "target": "com.amazonaws.sfn#ExecutionRedrivenEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the redrive attempt of an execution.

" + } + }, + "mapStateStartedEventDetails": { + "target": "com.amazonaws.sfn#MapStateStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about Map state that was started.

" + } + }, + "mapIterationStartedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that was started.

" + } + }, + "mapIterationSucceededEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that succeeded.

" + } + }, + "mapIterationFailedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that failed.

" + } + }, + "mapIterationAbortedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that was aborted.

" + } + }, + "lambdaFunctionFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionFailedEventDetails" + }, + "lambdaFunctionScheduleFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionScheduleFailedEventDetails" + }, + "lambdaFunctionScheduledEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionScheduledEventDetails" + }, + "lambdaFunctionStartFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionStartFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a lambda function that failed to start during an execution.

" + } + }, + "lambdaFunctionSucceededEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionSucceededEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that terminated successfully during an\n execution.

" + } + }, + "lambdaFunctionTimedOutEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionTimedOutEventDetails" + }, + "stateEnteredEventDetails": { + "target": "com.amazonaws.sfn#StateEnteredEventDetails" + }, + "stateExitedEventDetails": { + "target": "com.amazonaws.sfn#StateExitedEventDetails" + }, + "mapRunStartedEventDetails": { + "target": "com.amazonaws.sfn#MapRunStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details, such as mapRunArn, and the start date and time of a Map Run. mapRunArn is the Amazon Resource Name (ARN) of the Map Run that was started.

" + } + }, + "mapRunFailedEventDetails": { + "target": "com.amazonaws.sfn#MapRunFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains error and cause details about a Map Run that failed.

" + } + }, + "mapRunRedrivenEventDetails": { + "target": "com.amazonaws.sfn#MapRunRedrivenEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the redrive attempt of a Map Run.

" + } + }, + "evaluationFailedEventDetails": { + "target": "com.amazonaws.sfn#EvaluationFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an evaluation failure that occurred while processing a state.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the events of an execution.

" + } + }, + "com.amazonaws.sfn#HistoryEventExecutionDataDetails": { + "type": "structure", + "members": { + "truncated": { + "target": "com.amazonaws.sfn#truncated", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether input or output was truncated in the response. Always false\n for API calls. In CloudWatch logs, the value will be true if the data is truncated due to size limits.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about input or output in an execution history event.

" + } + }, + "com.amazonaws.sfn#HistoryEventList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#HistoryEvent" + }, + "traits": { + "smithy.api#documentation": "

Contains details about the events that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#HistoryEventType": { + "type": "enum", + "members": { + "ActivityFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityFailed" + } + }, + "ActivityScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityScheduled" + } + }, + "ActivityScheduleFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityScheduleFailed" + } + }, + "ActivityStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityStarted" + } + }, + "ActivitySucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivitySucceeded" + } + }, + "ActivityTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityTimedOut" + } + }, + "ChoiceStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ChoiceStateEntered" + } + }, + "ChoiceStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ChoiceStateExited" + } + }, + "ExecutionAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionAborted" + } + }, + "ExecutionFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionFailed" + } + }, + "ExecutionStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionStarted" + } + }, + "ExecutionSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionSucceeded" + } + }, + "ExecutionTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionTimedOut" + } + }, + "FailStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FailStateEntered" + } + }, + "LambdaFunctionFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionFailed" + } + }, + "LambdaFunctionScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionScheduled" + } + }, + "LambdaFunctionScheduleFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionScheduleFailed" + } + }, + "LambdaFunctionStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionStarted" + } + }, + "LambdaFunctionStartFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionStartFailed" + } + }, + "LambdaFunctionSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionSucceeded" + } + }, + "LambdaFunctionTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionTimedOut" + } + }, + "MapIterationAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationAborted" + } + }, + "MapIterationFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationFailed" + } + }, + "MapIterationStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationStarted" + } + }, + "MapIterationSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationSucceeded" + } + }, + "MapStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateAborted" + } + }, + "MapStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateEntered" + } + }, + "MapStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateExited" + } + }, + "MapStateFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateFailed" + } + }, + "MapStateStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateStarted" + } + }, + "MapStateSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateSucceeded" + } + }, + "ParallelStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateAborted" + } + }, + "ParallelStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateEntered" + } + }, + "ParallelStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateExited" + } + }, + "ParallelStateFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateFailed" + } + }, + "ParallelStateStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateStarted" + } + }, + "ParallelStateSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateSucceeded" + } + }, + "PassStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PassStateEntered" + } + }, + "PassStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PassStateExited" + } + }, + "SucceedStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SucceedStateEntered" + } + }, + "SucceedStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SucceedStateExited" + } + }, + "TaskFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskFailed" + } + }, + "TaskScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskScheduled" + } + }, + "TaskStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStarted" + } + }, + "TaskStartFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStartFailed" + } + }, + "TaskStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateAborted" + } + }, + "TaskStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateEntered" + } + }, + "TaskStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateExited" + } + }, + "TaskSubmitFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSubmitFailed" + } + }, + "TaskSubmitted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSubmitted" + } + }, + "TaskSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSucceeded" + } + }, + "TaskTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskTimedOut" + } + }, + "WaitStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateAborted" + } + }, + "WaitStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateEntered" + } + }, + "WaitStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateExited" + } + }, + "MapRunAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunAborted" + } + }, + "MapRunFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunFailed" + } + }, + "MapRunStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunStarted" + } + }, + "MapRunSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunSucceeded" + } + }, + "ExecutionRedriven": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionRedriven" + } + }, + "MapRunRedriven": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunRedriven" + } + }, + "EvaluationFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "EvaluationFailed" + } + } + } + }, + "com.amazonaws.sfn#Identity": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + } + } + }, + "com.amazonaws.sfn#IncludeExecutionData": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#IncludeExecutionDataGetExecutionHistory": { + "type": "boolean" + }, + "com.amazonaws.sfn#IncludedData": { + "type": "enum", + "members": { + "ALL_DATA": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ALL_DATA" + } + }, + "METADATA_ONLY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "METADATA_ONLY" + } + } + } + }, + "com.amazonaws.sfn#InspectionData": { + "type": "structure", + "members": { + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The raw state input.

" + } + }, + "afterArguments": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The input after Step Functions applies an Arguments filter. This event will only be present when QueryLanguage for the state machine or individual states is set to JSONata. For more info, see Transforming data with Step Functions.

" + } + }, + "afterInputPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The input after Step Functions applies the InputPath filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "afterParameters": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after Step Functions applies the Parameters filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "result": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The state's raw result.

" + } + }, + "afterResultSelector": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective result after Step Functions applies the ResultSelector filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "afterResultPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective result combined with the raw state input after Step Functions applies the ResultPath filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "request": { + "target": "com.amazonaws.sfn#InspectionDataRequest", + "traits": { + "smithy.api#documentation": "

The raw HTTP request that is sent when you test an HTTP Task.

" + } + }, + "response": { + "target": "com.amazonaws.sfn#InspectionDataResponse", + "traits": { + "smithy.api#documentation": "

The raw HTTP response that is returned when you test an HTTP Task.

" + } + }, + "variables": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

JSON string that contains the set of workflow variables after execution of the state. The set will include variables assigned in the state and variables set up as test state input.

" + } + }, + "errorDetails": { + "target": "com.amazonaws.sfn#InspectionErrorDetails", + "traits": { + "smithy.api#documentation": "

An object containing data about a handled exception in the tested state.

" + } + }, + "afterItemsPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemsPath filter is applied. Not populated when the QueryLanguage is JSONata.

" + } + }, + "afterItemSelector": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

An array containing the inputs for each Map iteration, transformed by the ItemSelector specified in a Map state.

" + } + }, + "afterItemBatcher": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemBatcher filter is applied in a Map state.

" + } + }, + "afterItemsPointer": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemsPointer filter is applied in a Map state.

" + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#InspectionToleratedFailureCount", + "traits": { + "smithy.api#documentation": "

The tolerated failure threshold for a Map state as defined in number of Map state iterations.

" + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#InspectionToleratedFailurePercentage", + "traits": { + "smithy.api#documentation": "

The tolerated failure threshold for a Map state as defined in percentage of Map state iterations.

" + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#InspectionMaxConcurrency", + "traits": { + "smithy.api#documentation": "

The max concurrency of the Map state.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP request and response information.

", + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionDataRequest": { + "type": "structure", + "members": { + "protocol": { + "target": "com.amazonaws.sfn#HTTPProtocol", + "traits": { + "smithy.api#documentation": "

The protocol used to make the HTTP request.

" + } + }, + "method": { + "target": "com.amazonaws.sfn#HTTPMethod", + "traits": { + "smithy.api#documentation": "

The HTTP method used for the HTTP request.

" + } + }, + "url": { + "target": "com.amazonaws.sfn#URL", + "traits": { + "smithy.api#documentation": "

The API endpoint used for the HTTP request.

" + } + }, + "headers": { + "target": "com.amazonaws.sfn#HTTPHeaders", + "traits": { + "smithy.api#documentation": "

The request headers associated with the HTTP request.

" + } + }, + "body": { + "target": "com.amazonaws.sfn#HTTPBody", + "traits": { + "smithy.api#documentation": "

The request body for the HTTP request.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP request information.

" + } + }, + "com.amazonaws.sfn#InspectionDataResponse": { + "type": "structure", + "members": { + "protocol": { + "target": "com.amazonaws.sfn#HTTPProtocol", + "traits": { + "smithy.api#documentation": "

The protocol used to return the HTTP response.

" + } + }, + "statusCode": { + "target": "com.amazonaws.sfn#HTTPStatusCode", + "traits": { + "smithy.api#documentation": "

The HTTP response status code for the HTTP response.

" + } + }, + "statusMessage": { + "target": "com.amazonaws.sfn#HTTPStatusMessage", + "traits": { + "smithy.api#documentation": "

The message associated with the HTTP status code.

" + } + }, + "headers": { + "target": "com.amazonaws.sfn#HTTPHeaders", + "traits": { + "smithy.api#documentation": "

The response headers associated with the HTTP response.

" + } + }, + "body": { + "target": "com.amazonaws.sfn#HTTPBody", + "traits": { + "smithy.api#documentation": "

The HTTP response returned.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP response information. The inspectionLevel request parameter specifies which details are returned.

" + } + }, + "com.amazonaws.sfn#InspectionErrorDetails": { + "type": "structure", + "members": { + "catchIndex": { + "target": "com.amazonaws.sfn#ExceptionHandlerIndex", + "traits": { + "smithy.api#documentation": "

The array index of the Catch which handled the exception.

" + } + }, + "retryIndex": { + "target": "com.amazonaws.sfn#ExceptionHandlerIndex", + "traits": { + "smithy.api#documentation": "

The array index of the Retry which handled the exception.

" + } + }, + "retryBackoffIntervalSeconds": { + "target": "com.amazonaws.sfn#RetryBackoffIntervalSeconds", + "traits": { + "smithy.api#documentation": "

The duration in seconds of the backoff for a retry on a failed state invocation.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

An object containing data about a handled exception in the tested state.

" + } + }, + "com.amazonaws.sfn#InspectionLevel": { + "type": "enum", + "members": { + "INFO": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "INFO" + } + }, + "DEBUG": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DEBUG" + } + }, + "TRACE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TRACE" + } + } + } + }, + "com.amazonaws.sfn#InspectionMaxConcurrency": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionToleratedFailureCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionToleratedFailurePercentage": { + "type": "float", + "traits": { + "smithy.api#range": { + "min": 0, + "max": 100 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InvalidArn": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided Amazon Resource Name (ARN) is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidDefinition": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided Amazon States Language definition is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidEncryptionConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Received when encryptionConfiguration is specified but various conditions exist which make the configuration invalid. For example, if type is set to CUSTOMER_MANAGED_KMS_KEY, but kmsKeyId is null, or kmsDataKeyReusePeriodSeconds is not between 60 and 900, or the KMS key is not symmetric or inactive.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidExecutionInput": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided JSON input data is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidLoggingConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Configuration is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidName": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided name is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidOutput": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided JSON output data is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidToken": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided token is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidTracingConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Your tracingConfiguration key does not match, or enabled has not\n been set to true or false.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsAccessDeniedException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Either your KMS key policy or API caller does not have the required permissions.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsDataKeyReusePeriodSeconds": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 60, + "max": 900 + } + } + }, + "com.amazonaws.sfn#KmsInvalidStateException": { + "type": "structure", + "members": { + "kmsKeyState": { + "target": "com.amazonaws.sfn#KmsKeyState", + "traits": { + "smithy.api#documentation": "

Current status of the KMS; key. For example: DISABLED, PENDING_DELETION, PENDING_IMPORT, UNAVAILABLE, CREATING.

" + } + }, + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The KMS key is not in valid state, for example: Disabled or Deleted.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsKeyId": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2048 + } + } + }, + "com.amazonaws.sfn#KmsKeyState": { + "type": "enum", + "members": { + "DISABLED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DISABLED" + } + }, + "PENDING_DELETION": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_DELETION" + } + }, + "PENDING_IMPORT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_IMPORT" + } + }, + "UNAVAILABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "UNAVAILABLE" + } + }, + "CREATING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CREATING" + } + } + } + }, + "com.amazonaws.sfn#KmsThrottlingException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Received when KMS returns ThrottlingException for a KMS call that Step Functions makes on behalf of the caller.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#LambdaFunctionFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that failed during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionScheduleFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a failed Lambda function schedule event that occurred during an\n execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionScheduledEventDetails": { + "type": "structure", + "members": { + "resource": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the scheduled Lambda function.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the Lambda function. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about input for an execution history event.

" + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the Lambda function.

" + } + }, + "taskCredentials": { + "target": "com.amazonaws.sfn#TaskCredentials", + "traits": { + "smithy.api#documentation": "

The credentials that Step Functions uses for the task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionStartFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a lambda function that failed to start during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionSucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the Lambda function. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that successfully terminated during an\n execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#ListActivities": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListActivitiesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListActivitiesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists the existing activities.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "activities", + "pageSize": "maxResults" + }, + "smithy.test#smokeTests": [ + { + "id": "ListActivitiesSuccess", + "params": {}, + "vendorParams": { + "region": "us-west-2" + }, + "vendorParamsShape": "aws.test#AwsVendorParams", + "expect": { + "success": {} + } + } + ] + } + }, + "com.amazonaws.sfn#ListActivitiesInput": { + "type": "structure", + "members": { + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListActivitiesOutput": { + "type": "structure", + "members": { + "activities": { + "target": "com.amazonaws.sfn#ActivityList", + "traits": { + "smithy.api#documentation": "

The list of activities.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListExecutions": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListExecutionsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListExecutionsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Lists all executions of a state machine or a Map Run. You can list all executions related to a state machine by specifying a state machine Amazon Resource Name (ARN), or those related to a Map Run by specifying a Map Run ARN. Using this API action, you can also list all redriven executions.

\n

You can also provide a state machine alias ARN or version ARN to list the executions associated with a specific alias or version.

\n

Results are sorted by time, with the most recent execution first. Running executions are sorted by their startDate or redriveDate, and other executions are sorted by their stopDate.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
\n

This API action is not supported by EXPRESS state machines. However, you may list EXPRESS children started by a map run using the mapRunArn parameter.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "executions", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListExecutionsInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine whose executions is listed.

\n

You can specify either a mapRunArn or a stateMachineArn, but not both.

\n

You can also return a list of executions associated with a specific alias or version, by specifying an alias ARN or a version ARN in the stateMachineArn parameter.

" + } + }, + "statusFilter": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

If specified, only list the executions whose current execution status matches the given\n filter.

\n

If you provide a PENDING_REDRIVE statusFilter, you must specify mapRunArn. \n For more information, see Child workflow execution redrive behaviour\n in the Step Functions Developer Guide.\n

\n

If you provide a stateMachineArn and a PENDING_REDRIVE statusFilter, the API returns a validation exception.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#ListExecutionsPageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run that started the child workflow executions. If the mapRunArn field is specified, a list of all of the child workflow executions started by a Map Run is returned. For more information, see Examining Map Run in the Step Functions Developer Guide.

\n

You can specify either a mapRunArn or a stateMachineArn, but not both.

" + } + }, + "redriveFilter": { + "target": "com.amazonaws.sfn#ExecutionRedriveFilter", + "traits": { + "smithy.api#documentation": "

Sets a filter to list executions based on whether or not they have been redriven.

\n

For a Distributed Map, redriveFilter sets a filter to list child workflow executions based on whether or not they have been redriven.

\n

If you do not provide a redriveFilter, Step Functions returns a list of both redriven and non-redriven executions.

\n

If you provide a state machine ARN in redriveFilter, the API returns a validation exception.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListExecutionsOutput": { + "type": "structure", + "members": { + "executions": { + "target": "com.amazonaws.sfn#ExecutionList", + "traits": { + "smithy.api#documentation": "

The list of matching executions.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#ListExecutionsPageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListExecutionsPageToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 3096 + } + } + }, + "com.amazonaws.sfn#ListMapRuns": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListMapRunsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListMapRunsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists all Map Runs that were started by a given state machine execution. Use this API action to obtain Map Run ARNs, and then call DescribeMapRun to obtain more information, if needed.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "mapRuns", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListMapRunsInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution for which the Map Runs must be listed.

", + "smithy.api#required": {} + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListMapRunsOutput": { + "type": "structure", + "members": { + "mapRuns": { + "target": "com.amazonaws.sfn#MapRunList", + "traits": { + "smithy.api#documentation": "

An array that lists information related to a Map Run, such as the Amazon Resource Name (ARN) of the Map Run and the ARN of the state machine that started the Map Run.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachineAliases": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachineAliasesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachineAliasesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + } + ], + "traits": { + "smithy.api#documentation": "

Lists aliases for a specified state machine ARN. Results are sorted by time, with the most recently created aliases listed first.

\n

To list aliases that reference a state machine version, you can specify the version ARN in the stateMachineArn parameter.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#ListStateMachineAliasesInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine for which you want to list aliases.

\n

If you specify a state machine version ARN, this API returns a list of aliases for that version.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachineAliasesOutput": { + "type": "structure", + "members": { + "stateMachineAliases": { + "target": "com.amazonaws.sfn#StateMachineAliasList", + "traits": { + "smithy.api#documentation": "

Aliases for the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachineVersions": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachineVersionsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachineVersionsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Lists versions for the specified state machine Amazon Resource Name (ARN).

\n

The results are sorted in descending order of the version creation time.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#ListStateMachineVersionsInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachineVersionsOutput": { + "type": "structure", + "members": { + "stateMachineVersions": { + "target": "com.amazonaws.sfn#StateMachineVersionList", + "traits": { + "smithy.api#documentation": "

Versions for the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachines": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachinesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachinesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists the existing state machines.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n
", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "stateMachines", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListStateMachinesInput": { + "type": "structure", + "members": { + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachinesOutput": { + "type": "structure", + "members": { + "stateMachines": { + "target": "com.amazonaws.sfn#StateMachineList", + "traits": { + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListTagsForResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListTagsForResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListTagsForResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

List tags for a given resource.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#ListTagsForResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListTagsForResourceOutput": { + "type": "structure", + "members": { + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

An array of tags associated with the resource.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#LogDestination": { + "type": "structure", + "members": { + "cloudWatchLogsLogGroup": { + "target": "com.amazonaws.sfn#CloudWatchLogsLogGroup", + "traits": { + "smithy.api#documentation": "

An object describing a CloudWatch log group. For more information, see AWS::Logs::LogGroup in the CloudFormation User Guide.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

" + } + }, + "com.amazonaws.sfn#LogDestinationList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#LogDestination" + } + }, + "com.amazonaws.sfn#LogLevel": { + "type": "enum", + "members": { + "ALL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ALL" + } + }, + "ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ERROR" + } + }, + "FATAL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FATAL" + } + }, + "OFF": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "OFF" + } + } + } + }, + "com.amazonaws.sfn#LoggingConfiguration": { + "type": "structure", + "members": { + "level": { + "target": "com.amazonaws.sfn#LogLevel", + "traits": { + "smithy.api#documentation": "

Defines which category of execution history events are logged.

" + } + }, + "includeExecutionData": { + "target": "com.amazonaws.sfn#IncludeExecutionData", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Determines whether execution data is included in your log. When set to false,\n data is excluded.

" + } + }, + "destinations": { + "target": "com.amazonaws.sfn#LogDestinationList", + "traits": { + "smithy.api#documentation": "

An array of objects that describes where your execution history events will be logged.\n Limited to size 1. Required, if your log level is not set to OFF.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

The LoggingConfiguration data type is used to set CloudWatch Logs\n options.

" + } + }, + "com.amazonaws.sfn#LongArn": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2000 + } + } + }, + "com.amazonaws.sfn#LongObject": { + "type": "long" + }, + "com.amazonaws.sfn#MapIterationEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the iteration\u2019s parent Map state.

" + } + }, + "index": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The index of the array belonging to the Map state iteration.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state.

" + } + }, + "com.amazonaws.sfn#MapIterationFailureCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#MapRunExecutionCounts": { + "type": "structure", + "members": { + "pending": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run, but haven't started executing yet.

", + "smithy.api#required": {} + } + }, + "running": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and are currently in-progress.

", + "smithy.api#required": {} + } + }, + "succeeded": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and have completed successfully.

", + "smithy.api#required": {} + } + }, + "failed": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run, but have failed.

", + "smithy.api#required": {} + } + }, + "timedOut": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and have timed out.

", + "smithy.api#required": {} + } + }, + "aborted": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and were running, but were either stopped by the user or by Step Functions because the Map Run failed.

", + "smithy.api#required": {} + } + }, + "total": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run.

", + "smithy.api#required": {} + } + }, + "resultsWritten": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Returns the count of child workflow executions whose results were written by ResultWriter. For more information, see ResultWriter in the Step Functions Developer Guide.

", + "smithy.api#required": {} + } + }, + "failuresNotRedrivable": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of FAILED, ABORTED, or TIMED_OUT child workflow executions that cannot be redriven because their execution status is terminal. For example, child workflows with an execution status of FAILED, ABORTED, or TIMED_OUT and a redriveStatus of NOT_REDRIVABLE.

" + } + }, + "pendingRedrive": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of unsuccessful child workflow executions currently waiting to be redriven. The status of these child workflow executions could be FAILED, ABORTED, or TIMED_OUT in the original execution attempt or a previous redrive attempt.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about all of the child workflow executions started by a Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the Map Run failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run failure event that occurred during a state machine execution.

" + } + }, + "com.amazonaws.sfn#MapRunItemCounts": { + "type": "structure", + "members": { + "pending": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items to process in child workflow executions that haven't started running yet.

", + "smithy.api#required": {} + } + }, + "running": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items being processed in child workflow executions that are currently in-progress.

", + "smithy.api#required": {} + } + }, + "succeeded": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have completed successfully.

", + "smithy.api#required": {} + } + }, + "failed": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have failed.

", + "smithy.api#required": {} + } + }, + "timedOut": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have timed out.

", + "smithy.api#required": {} + } + }, + "aborted": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that were either stopped by the user or by Step Functions, because the Map Run failed.

", + "smithy.api#required": {} + } + }, + "total": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in all the child workflow executions started by a Map Run.

", + "smithy.api#required": {} + } + }, + "resultsWritten": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Returns the count of items whose results were written by ResultWriter. For more information, see ResultWriter in the Step Functions Developer Guide.

", + "smithy.api#required": {} + } + }, + "failuresNotRedrivable": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of FAILED, ABORTED, or TIMED_OUT items in child workflow executions that cannot be redriven because the execution status of those child workflows is terminal. For example, child workflows with an execution status of FAILED, ABORTED, or TIMED_OUT and a redriveStatus of NOT_REDRIVABLE.

" + } + }, + "pendingRedrive": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of unsuccessful items in child workflow executions currently waiting to be redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about items that were processed in all of the child workflow executions that were started by a Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunLabel": { + "type": "string" + }, + "com.amazonaws.sfn#MapRunList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#MapRunListItem" + } + }, + "com.amazonaws.sfn#MapRunListItem": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The executionArn of the execution from which the Map Run was started.

", + "smithy.api#required": {} + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the executed state machine.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date on which the Map Run started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date on which the Map Run stopped.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunRedrivenEventDetails": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run that was redriven.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times the Map Run has been redriven at this point in the execution's history including this event. The redrive count for a redriven Map Run is always greater than 0.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run that was redriven.

" + } + }, + "com.amazonaws.sfn#MapRunStartedEventDetails": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run that was started.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run that was started during a state machine execution.

" + } + }, + "com.amazonaws.sfn#MapRunStatus": { + "type": "enum", + "members": { + "RUNNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RUNNING" + } + }, + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "ABORTED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ABORTED" + } + } + } + }, + "com.amazonaws.sfn#MapStateStartedEventDetails": { + "type": "structure", + "members": { + "length": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The size of the array for Map state iterations.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Details about a Map state that was started.

" + } + }, + "com.amazonaws.sfn#MaxConcurrency": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#MissingRequiredParameter": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Request is missing a required parameter. This error occurs if both definition\n and roleArn are not specified.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#MockErrorOutput": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

A string denoting the error code of the exception thrown when invoking the tested state. This field is required if mock.errorOutput is specified.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A string containing the cause of the exception thrown when executing the state's logic.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

A JSON object that contains a mocked error.

" + } + }, + "com.amazonaws.sfn#MockInput": { + "type": "structure", + "members": { + "result": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A JSON string containing the mocked result of the state invocation.

" + } + }, + "errorOutput": { + "target": "com.amazonaws.sfn#MockErrorOutput", + "traits": { + "smithy.api#documentation": "

The mocked error output when calling TestState. When specified, the mocked response is returned as a JSON object that contains an error and cause field.

" + } + }, + "fieldValidationMode": { + "target": "com.amazonaws.sfn#MockResponseValidationMode", + "traits": { + "smithy.api#documentation": "

Determines the level of strictness when validating mocked results against their respective API models. Values include:

\n
    \n
  • \n

    \n STRICT: All required fields must be present, and all present fields must conform to the API's schema.

    \n
  • \n
  • \n

    \n PRESENT: All present fields must conform to the API's schema.

    \n
  • \n
  • \n

    \n NONE: No validation is performed.

    \n
  • \n
\n

If no value is specified, the default value is STRICT.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

A JSON object that contains a mocked result or errorOutput.

" + } + }, + "com.amazonaws.sfn#MockResponseValidationMode": { + "type": "enum", + "members": { + "STRICT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "STRICT" + } + }, + "PRESENT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PRESENT" + } + }, + "NONE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NONE" + } + } + } + }, + "com.amazonaws.sfn#Name": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + } + } + }, + "com.amazonaws.sfn#PageSize": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 1000 + } + } + }, + "com.amazonaws.sfn#PageToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 1024 + } + } + }, + "com.amazonaws.sfn#Publish": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#PublishStateMachineVersion": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#PublishStateMachineVersionInput" + }, + "output": { + "target": "com.amazonaws.sfn#PublishStateMachineVersionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates a version from the\n current revision of a state machine. Use versions to create immutable snapshots of your state\n machine. You can start executions from versions either directly or with an alias. To create an\n alias, use CreateStateMachineAlias.

\n

You can publish up to 1000 versions for each state machine. You must manually delete unused versions using the DeleteStateMachineVersion API action.

\n

\n PublishStateMachineVersion is an idempotent API. It doesn't create a\n duplicate state machine version if it already exists for the current revision. Step Functions bases PublishStateMachineVersion's idempotency check on the\n stateMachineArn, name, and revisionId parameters.\n Requests with the same parameters return a successful idempotent response. If you don't\n specify a revisionId, Step Functions checks for a previously published\n version of the state machine's current revision.

\n

\n Related operations:\n

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#PublishStateMachineVersionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

Only publish the state machine version if the current state machine's revision ID matches the specified ID.

\n

Use this option to avoid publishing a version if the state machine changed since you last\n updated it. If the specified revision ID doesn't match the state machine's current revision\n ID, the API returns ConflictException.

\n \n

To specify an initial revision ID for a state machine with no revision ID assigned,\n specify the string INITIAL for the revisionId parameter. For\n example, you can specify a revisionID of INITIAL when you create a\n state machine using the CreateStateMachine API action.

\n
" + } + }, + "description": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

An optional description of the state machine version.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#PublishStateMachineVersionOutput": { + "type": "structure", + "members": { + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the version was created.

", + "smithy.api#required": {} + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) (ARN) that identifies the state machine version.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#RedriveCount": { + "type": "integer" + }, + "com.amazonaws.sfn#RedriveExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#RedriveExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#RedriveExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ExecutionLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#ExecutionNotRedrivable" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Restarts unsuccessful executions of Standard workflows that didn't complete successfully in the last 14 days. These include failed, aborted, or timed out executions. When you redrive an execution, it continues the failed execution from the unsuccessful step and uses the same input. Step Functions preserves the results and execution history of the successful steps, and doesn't rerun these steps when you redrive an execution. Redriven executions use the same state machine definition and execution ARN as the original execution attempt.

\n

For workflows that include an Inline Map or Parallel state, RedriveExecution API action reschedules and redrives only the iterations and branches that failed or aborted.

\n

To redrive a workflow that includes a Distributed Map state whose Map Run failed, you must redrive the parent workflow. The parent workflow redrives all the unsuccessful states, including a failed Map Run. If a Map Run was not started in the original execution attempt, the redriven parent workflow starts the Map Run.

\n \n

This API action is not supported by EXPRESS state machines.

\n

However, you can restart the unsuccessful executions of Express child workflows in a Distributed Map by redriving its Map Run. When you redrive a Map Run, the Express child workflows are rerun using the StartExecution API action. For more information, see Redriving Map Runs.

\n
\n

You can redrive executions if your original execution meets the following conditions:

\n
    \n
  • \n

    The execution status isn't SUCCEEDED.

    \n
  • \n
  • \n

    Your workflow execution has not exceeded the redrivable period of 14 days. Redrivable period refers to the time during which you can redrive a given execution. This period starts from the day a state machine completes its execution.

    \n
  • \n
  • \n

    The workflow execution has not exceeded the maximum open time of one year. For more information about state machine quotas, see Quotas related to state machine executions.

    \n
  • \n
  • \n

    The execution event history count is less than 24,999. Redriven executions append their event history to the existing event history. Make sure your workflow execution contains less than 24,999 events to accommodate the ExecutionRedriven history event and at least one other history event.

    \n
  • \n
", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#RedriveExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to be redriven.

", + "smithy.api#required": {} + } + }, + "clientToken": { + "target": "com.amazonaws.sfn#ClientToken", + "traits": { + "smithy.api#documentation": "

A unique, case-sensitive identifier that you provide to ensure the idempotency of the request. If you don\u2019t specify a client token, the Amazon Web Services SDK automatically generates a client token and uses it for the request to ensure idempotency. The API will return idempotent responses for the last 10 client tokens used to successfully redrive the execution. These client tokens are valid for up to 15 minutes after they are first used.

", + "smithy.api#idempotencyToken": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#RedriveExecutionOutput": { + "type": "structure", + "members": { + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ResourceNotFound": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "resourceName": { + "target": "com.amazonaws.sfn#Arn" + } + }, + "traits": { + "smithy.api#documentation": "

Could not find the referenced resource.

", + "smithy.api#error": "client", + "smithy.api#httpError": 404 + } + }, + "com.amazonaws.sfn#RetrierRetryCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#RetryBackoffIntervalSeconds": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#RevealSecrets": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#ReverseOrder": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#RevisionId": { + "type": "string" + }, + "com.amazonaws.sfn#RoutingConfigurationList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#RoutingConfigurationListItem" + }, + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2 + } + } + }, + "com.amazonaws.sfn#RoutingConfigurationListItem": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies one or two state machine versions defined in the routing configuration.

\n

If you specify the ARN of a second version, it must belong to the same state machine as the first version.

", + "smithy.api#required": {} + } + }, + "weight": { + "target": "com.amazonaws.sfn#VersionWeight", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The percentage of traffic you want to route to a state machine version. The sum of the\n weights in the routing configuration must be equal to 100.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the routing configuration of a state machine alias. In a routing\n configuration, you define an array of objects that specify up to two state machine versions.\n You also specify the percentage of traffic to be routed to each version.

" + } + }, + "com.amazonaws.sfn#SendTaskFailure": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskFailureInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskFailureOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers, Task states using the callback\n pattern, and optionally Task states using the job run pattern to report that the task identified by the taskToken failed.

\n

For an execution with encryption enabled, Step Functions will encrypt the error and cause fields using the KMS key for the execution role.

\n

A caller can mark a task as fail without using any KMS permissions in the execution role if the caller provides a null value for both error and cause fields because no data needs to be encrypted.

" + } + }, + "com.amazonaws.sfn#SendTaskFailureInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskFailureOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SendTaskHeartbeat": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskHeartbeatInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskHeartbeatOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers and Task states using the callback\n pattern, and optionally Task states using the job run pattern to report to Step Functions that the task represented by the specified\n taskToken is still making progress. This action resets the\n Heartbeat clock. The Heartbeat threshold is specified in the state\n machine's Amazon States Language definition (HeartbeatSeconds). This action does not in itself\n create an event in the execution history. However, if the task times out, the execution\n history contains an ActivityTimedOut entry for activities, or a\n TaskTimedOut entry for tasks using the job run or\n callback\n pattern.

\n \n

The Timeout of a task, defined in the state machine's Amazon States Language definition, is\n its maximum allowed duration, regardless of the number of SendTaskHeartbeat requests received. Use HeartbeatSeconds to configure the timeout interval\n for heartbeats.

\n
" + } + }, + "com.amazonaws.sfn#SendTaskHeartbeatInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskHeartbeatOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SendTaskSuccess": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskSuccessInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskSuccessOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidOutput" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers, Task states using the callback\n pattern, and optionally Task states using the job run pattern to report that the task identified by the taskToken completed\n successfully.

" + } + }, + "com.amazonaws.sfn#SendTaskSuccessInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output of the task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskSuccessOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SensitiveCause": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 32768 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveData": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 262144 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveDataJobInput": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 1048576 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveError": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ServiceQuotaExceededException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The request would cause a service quota to be exceeded.

\n

HTTP Status Code: 402

", + "smithy.api#error": "client", + "smithy.api#httpError": 402 + } + }, + "com.amazonaws.sfn#StartExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StartExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StartExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#ExecutionLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Starts a state machine execution.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n
    \n
  • \n

    The following qualified state machine ARN refers to a Distributed Map state with a label mapStateLabel in a state machine named myStateMachine.

    \n

    \n arn:partition:states:region:account-id:stateMachine:myStateMachine/mapStateLabel\n

    \n \n

    If you provide a qualified state machine ARN that refers to a Distributed Map state, the request fails with ValidationException.

    \n
    \n
  • \n
  • \n

    The following qualified state machine ARN refers to an alias named PROD.

    \n

    \n arn::states:::stateMachine:\n

    \n \n

    If you provide a qualified state machine ARN that refers to a version ARN or an alias ARN, the request starts execution for that version or alias.

    \n
    \n
  • \n
  • \n

    The following unqualified state machine ARN refers to a state machine named myStateMachine.

    \n

    \n arn::states:::stateMachine:\n

    \n
  • \n
\n

If you start an execution with an unqualified state machine ARN, Step Functions uses the latest revision of the state machine for the execution.

\n

To start executions of a state machine version, call\n StartExecution and provide the version ARN or the ARN of an alias that points to the version.

\n \n

\n StartExecution is idempotent for STANDARD workflows. For a\n STANDARD workflow, if you call StartExecution with the same name\n and input as a running execution, the call succeeds and return the same response as the\n original request. If the execution is closed or if the input is different, it returns a\n 400 ExecutionAlreadyExists error. You can reuse the name 90 days after it closes.

\n

\n StartExecution isn't idempotent for EXPRESS workflows.

\n
", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#StartExecutionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to execute.

\n

The stateMachineArn parameter accepts one of the following inputs:

\n
    \n
  • \n

    \n An unqualified state machine ARN \u2013 Refers to a state machine ARN that isn't qualified with a version or alias ARN. The following is an example of an unqualified state machine ARN.

    \n

    \n arn::states:::stateMachine:\n

    \n

    Step Functions doesn't associate state machine executions that you start with an unqualified ARN with a version. This is true even if that version uses the same revision that the execution used.

    \n
  • \n
  • \n

    \n A state machine version ARN \u2013 Refers to a version ARN, which is a combination of state machine ARN and the version number separated by a colon (:). The following is an example of the ARN for version 10.

    \n

    \n arn::states:::stateMachine::10\n

    \n

    Step Functions doesn't associate executions that you start with a version ARN with any aliases that point to that version.

    \n
  • \n
  • \n

    \n A state machine alias ARN \u2013 Refers to an alias ARN, which is a combination of state machine ARN and the alias name separated by a colon (:). The following is an example of the ARN for an alias named PROD.

    \n

    \n arn::states:::stateMachine:\n

    \n

    Step Functions associates executions\n that you start with an alias ARN with that alias and the state machine version used for\n that execution.

    \n
  • \n
", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

Optional name of the execution. For STANDARD workflows, this name must be unique for your Amazon Web Services account, region, and state machine.\n If a previous execution with the same name exists, you can reuse the name 90 days after it closes. For EXPRESS workflows, execution names\n can be reused immediately.\n For more information, see \n Limits Related to State Machine Executions in the Step Functions Developer Guide.

\n

If you don't provide a name for the execution, Step Functions automatically generates a universally unique identifier (UUID) as the execution name.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the execution, for example:

\n

\n \"{\\\"first_name\\\" : \\\"Alejandro\\\"}\"\n

\n \n

If you don't include any JSON input data, you still must include the two braces, for\n example: \"{}\"\n

\n
\n

Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

Passes the X-Ray trace header. The trace header can also be passed in the request\n payload.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n
" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StartExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#StartSyncExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StartSyncExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StartSyncExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + } + ], + "traits": { + "smithy.api#documentation": "

Starts a Synchronous Express state machine execution. StartSyncExecution \n\t\t\t is not available for STANDARD workflows.

\n \n

\n StartSyncExecution will return a 200 OK response, even if your\n execution fails, because the status code in the API response doesn't reflect function\n errors. Error codes are reserved for errors that prevent your execution from running, such\n as permissions errors, limit errors, or issues with your state machine code and\n configuration.

\n
", + "smithy.api#endpoint": { + "hostPrefix": "sync-" + } + } + }, + "com.amazonaws.sfn#StartSyncExecutionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to execute.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the execution, for example:

\n

\n \"{\\\"first_name\\\" : \\\"Alejandro\\\"}\"\n

\n \n

If you don't include any JSON input data, you still must include the two braces, for\n example: \"{}\"\n

\n
\n

Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

Passes the X-Ray trace header. The trace header can also be passed in the request\n payload.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n
" + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StartSyncExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

" + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution has already ended, the date the execution stopped.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#SyncExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

\n \n

This field is set only if the execution succeeds. If the execution fails, this field is\n null.

\n
" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

The X-Ray trace header that was passed to the execution.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n
" + } + }, + "billingDetails": { + "target": "com.amazonaws.sfn#BillingDetails", + "traits": { + "smithy.api#documentation": "

An object that describes workflow billing details, including billed duration and memory\n use.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#StateEnteredEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a state entered during an execution.

" + } + }, + "com.amazonaws.sfn#StateExitedEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + }, + "assignedVariables": { + "target": "com.amazonaws.sfn#AssignedVariables", + "traits": { + "smithy.api#documentation": "

Map of variable name and value as a serialized JSON representation.

" + } + }, + "assignedVariablesDetails": { + "target": "com.amazonaws.sfn#AssignedVariablesDetails", + "traits": { + "smithy.api#documentation": "

Provides details about input or output in an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an exit from a state during an execution.

" + } + }, + "com.amazonaws.sfn#StateMachineAliasList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineAliasListItem" + } + }, + "com.amazonaws.sfn#StateMachineAliasListItem": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine alias. The alias ARN is a combination of state machine ARN and the alias name separated by a colon (:). For example, stateMachineARN:PROD.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The creation date of a state machine alias.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific state machine alias.

" + } + }, + "com.amazonaws.sfn#StateMachineAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

A state machine with the same name but a different definition or role ARN already\n exists.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineDeleting": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified state machine is being deleted.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified state machine does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of state machines has been reached. Existing state machines must be\n deleted before a new state machine can be created.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineListItem" + } + }, + "com.amazonaws.sfn#StateMachineListItem": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n
    \n
  • \n

    white space

    \n
  • \n
  • \n

    brackets < > { } [ ]\n

    \n
  • \n
  • \n

    wildcard characters ? *\n

    \n
  • \n
  • \n

    special characters \" # % \\ ^ | ~ ` $ & , ; : /\n

    \n
  • \n
  • \n

    control characters (U+0000-001F, U+007F-009F, U+FFFE-FFFF)

    \n
  • \n
  • \n

    surrogates (U+D800-DFFF)

    \n
  • \n
  • \n

    invalid characters ( U+10FFFF)

    \n
  • \n
\n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the state machine.

" + } + }, + "com.amazonaws.sfn#StateMachineStatus": { + "type": "enum", + "members": { + "ACTIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ACTIVE" + } + }, + "DELETING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DELETING" + } + } + } + }, + "com.amazonaws.sfn#StateMachineType": { + "type": "enum", + "members": { + "STANDARD": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "STANDARD" + } + }, + "EXPRESS": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "EXPRESS" + } + } + } + }, + "com.amazonaws.sfn#StateMachineTypeNotSupported": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

State machine type is not supported.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineVersionList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineVersionListItem" + } + }, + "com.amazonaws.sfn#StateMachineVersionListItem": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine version. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The creation date of a state machine version.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific state machine version.

" + } + }, + "com.amazonaws.sfn#StateName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + } + } + }, + "com.amazonaws.sfn#StopExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StopExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StopExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Stops an execution.

\n

This API action is not supported by EXPRESS state machines.

\n

For an execution with encryption enabled, Step Functions will encrypt the error and cause fields using the KMS key for the execution role.

\n

A caller can stop an execution without using any KMS permissions in the execution role if the caller provides a null value for both error and cause fields because no data needs to be encrypted.

" + } + }, + "com.amazonaws.sfn#StopExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to stop.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StopExecutionOutput": { + "type": "structure", + "members": { + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is stopped.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SyncExecutionStatus": { + "type": "enum", + "members": { + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "TIMED_OUT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TIMED_OUT" + } + } + } + }, + "com.amazonaws.sfn#Tag": { + "type": "structure", + "members": { + "key": { + "target": "com.amazonaws.sfn#TagKey", + "traits": { + "smithy.api#documentation": "

The key of a tag.

" + } + }, + "value": { + "target": "com.amazonaws.sfn#TagValue", + "traits": { + "smithy.api#documentation": "

The value of a tag.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Tags are key-value pairs that can be associated with Step Functions state machines and\n activities.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#TagKey": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 128 + } + } + }, + "com.amazonaws.sfn#TagKeyList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#TagKey" + } + }, + "com.amazonaws.sfn#TagList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#Tag" + } + }, + "com.amazonaws.sfn#TagResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#TagResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#TagResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + } + ], + "traits": { + "smithy.api#documentation": "

Add a tag to a Step Functions resource.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#TagResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

The list of tags to add to a resource.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#TagResourceOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#TagValue": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + } + } + }, + "com.amazonaws.sfn#TaskCredentials": { + "type": "structure", + "members": { + "roleArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The ARN of an IAM role that Step Functions assumes for the task. The role can allow cross-account access to resources.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the credentials that Step Functions uses for a task.

" + } + }, + "com.amazonaws.sfn#TaskDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The activity does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#TaskFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task failure event.

" + } + }, + "com.amazonaws.sfn#TaskScheduledEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "region": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The region of the scheduled task

", + "smithy.api#required": {} + } + }, + "parameters": { + "target": "com.amazonaws.sfn#ConnectorParameters", + "traits": { + "smithy.api#documentation": "

The JSON data passed to the resource referenced in a task state.\n Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

", + "smithy.api#required": {} + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the task.

" + } + }, + "heartbeatInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration between two heartbeats for the task.

" + } + }, + "taskCredentials": { + "target": "com.amazonaws.sfn#TaskCredentials", + "traits": { + "smithy.api#documentation": "

The credentials that Step Functions uses for the task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#TaskStartFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to start during an execution.

" + } + }, + "com.amazonaws.sfn#TaskStartedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of a task during an execution.

" + } + }, + "com.amazonaws.sfn#TaskSubmitFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to submit during an execution.

" + } + }, + "com.amazonaws.sfn#TaskSubmittedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The response from a resource when a task has started. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task submitted to a resource .

" + } + }, + "com.amazonaws.sfn#TaskSucceededEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The full JSON response from a resource when a task has succeeded. This response becomes\n the output of the related task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the successful completion of a task state.

" + } + }, + "com.amazonaws.sfn#TaskTimedOut": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The task token has either expired or the task associated with the token has already been closed.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#TaskTimedOutEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a resource timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#TaskToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2048 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#TestExecutionStatus": { + "type": "enum", + "members": { + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "RETRIABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RETRIABLE" + } + }, + "CAUGHT_ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CAUGHT_ERROR" + } + } + } + }, + "com.amazonaws.sfn#TestState": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#TestStateInput" + }, + "output": { + "target": "com.amazonaws.sfn#TestStateOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Accepts the definition of a single state and executes it. You can test a state without creating a state machine or updating an existing state machine. Using this API, you can test the following:

\n \n

You can call this API on only one state at a time. The states that you can test include the following:

\n \n

The TestState API assumes an IAM role which must contain the required IAM permissions for the resources your state is accessing. For information about the permissions a state might need, see IAM permissions to test a state.

\n

The TestState API can run for up to five minutes. If the execution of a state exceeds this duration, it fails with the States.Timeout error.

\n

\n TestState only supports the following when a mock is specified: Activity tasks, .sync or .waitForTaskToken\n service integration patterns, Parallel, or Map states.

", + "smithy.api#endpoint": { + "hostPrefix": "sync-" + } + } + }, + "com.amazonaws.sfn#TestStateConfiguration": { + "type": "structure", + "members": { + "retrierRetryCount": { + "target": "com.amazonaws.sfn#RetrierRetryCount", + "traits": { + "smithy.api#documentation": "

The number of retry attempts that have occurred for the state's Retry that applies to the mocked error.

" + } + }, + "errorCausedByState": { + "target": "com.amazonaws.sfn#TestStateStateName", + "traits": { + "smithy.api#documentation": "

The name of the state from which an error originates when an error is mocked for a Map or Parallel state.

" + } + }, + "mapIterationFailureCount": { + "target": "com.amazonaws.sfn#MapIterationFailureCount", + "traits": { + "smithy.api#documentation": "

The number of Map state iterations that failed during the Map state invocation.

" + } + }, + "mapItemReaderData": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The data read by ItemReader in Distributed Map states as found in its original source.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains configurations for the tested state.

" + } + }, + "com.amazonaws.sfn#TestStateInput": { + "type": "structure", + "members": { + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language (ASL) definition of the state or state machine.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution role with the required IAM permissions for the state.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A string that contains the JSON input data for the state.

" + } + }, + "inspectionLevel": { + "target": "com.amazonaws.sfn#InspectionLevel", + "traits": { + "smithy.api#documentation": "

Determines the values to return when a state is tested. You can specify one of the following types:

\n
    \n
  • \n

    \n INFO: Shows the final state output. By default, Step Functions sets inspectionLevel to INFO if you don't specify a level.

    \n
  • \n
  • \n

    \n DEBUG: Shows the final state output along with the input and output data processing result.

    \n
  • \n
  • \n

    \n TRACE: Shows the HTTP request and response for an HTTP Task. This level also shows the final state output along with the input and output data processing result.

    \n
  • \n
\n

Each of these levels also provide information about the status of the state execution and the next state to transition to.

" + } + }, + "revealSecrets": { + "target": "com.amazonaws.sfn#RevealSecrets", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Specifies whether or not to include secret information in the test result. For HTTP Tasks, a secret includes the data that an EventBridge connection adds to modify the HTTP request headers, query parameters, and body. Step Functions doesn't omit any information included in the state definition or the HTTP response.

\n

If you set revealSecrets to true, you must make sure that the IAM user that calls the TestState API has permission for the states:RevealSecrets action. For an example of IAM policy that sets the states:RevealSecrets permission, see IAM permissions to test a state. Without this permission, Step Functions throws an access denied error.

\n

By default, revealSecrets is set to false.

" + } + }, + "variables": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

JSON object literal that sets variables used in the state under test. Object keys are the variable names and values are the variable values.

" + } + }, + "stateName": { + "target": "com.amazonaws.sfn#TestStateStateName", + "traits": { + "smithy.api#documentation": "

Denotes the particular state within a state machine definition to be tested. If this field is specified, the definition must contain a fully-formed state machine definition.

" + } + }, + "mock": { + "target": "com.amazonaws.sfn#MockInput", + "traits": { + "smithy.api#documentation": "

Defines a mocked result or error for the state under test.

\n

A mock can only be specified for Task, Map, or Parallel states. If it is specified for another state type, an exception will be thrown.

" + } + }, + "context": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A JSON string representing a valid Context object for the state under test. This field may only be specified if a mock is specified in the same request.

" + } + }, + "stateConfiguration": { + "target": "com.amazonaws.sfn#TestStateConfiguration", + "traits": { + "smithy.api#documentation": "

Contains configurations for the state under test.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#TestStateOutput": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error returned when the execution of a state fails.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A detailed explanation of the cause for the error when the execution of a state fails.

" + } + }, + "inspectionData": { + "target": "com.amazonaws.sfn#InspectionData", + "traits": { + "smithy.api#documentation": "

Returns additional details about the state's execution, including its input and output data processing flow, and HTTP request and response information. The inspectionLevel request parameter specifies which details are returned.

" + } + }, + "nextState": { + "target": "com.amazonaws.sfn#StateName", + "traits": { + "smithy.api#documentation": "

The name of the next state to transition to. If you haven't defined a next state in your definition or if the execution of the state fails, this field doesn't contain a value.

" + } + }, + "status": { + "target": "com.amazonaws.sfn#TestExecutionStatus", + "traits": { + "smithy.api#documentation": "

The execution status of the state.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#TestStateStateName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#TimeoutInSeconds": { + "type": "long", + "traits": { + "smithy.api#default": 0 + } + }, + "com.amazonaws.sfn#Timestamp": { + "type": "timestamp" + }, + "com.amazonaws.sfn#ToleratedFailureCount": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#ToleratedFailurePercentage": { + "type": "float", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#TooManyTags": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "resourceName": { + "target": "com.amazonaws.sfn#Arn" + } + }, + "traits": { + "smithy.api#documentation": "

You've exceeded the number of tags allowed for a resource. See the Limits Topic in the\n Step Functions Developer Guide.

", + "smithy.api#error": "client", + "smithy.api#httpError": 400 + } + }, + "com.amazonaws.sfn#TraceHeader": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#pattern": "^\\p{ASCII}*$" + } + }, + "com.amazonaws.sfn#TracingConfiguration": { + "type": "structure", + "members": { + "enabled": { + "target": "com.amazonaws.sfn#Enabled", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

When set to true, X-Ray tracing is enabled.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Selects whether or not the state machine's X-Ray tracing is enabled. Default is\n false\n

" + } + }, + "com.amazonaws.sfn#URL": { + "type": "string" + }, + "com.amazonaws.sfn#UnsignedInteger": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#UnsignedLong": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#UntagResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UntagResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#UntagResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

Remove a tag from a Step Functions resource

" + } + }, + "com.amazonaws.sfn#UntagResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + }, + "tagKeys": { + "target": "com.amazonaws.sfn#TagKeyList", + "traits": { + "smithy.api#documentation": "

The list of tags to remove from the resource.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UntagResourceOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateMapRun": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateMapRunInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateMapRunOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates an in-progress Map Run's configuration to include changes to the settings that control maximum concurrency and Map Run failure.

" + } + }, + "com.amazonaws.sfn#UpdateMapRunInput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run.

", + "smithy.api#required": {} + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#MaxConcurrency", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum number of child workflow executions that can be specified to run in parallel for the Map Run at the same time.

" + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#ToleratedFailurePercentage", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum percentage of failed items before the Map Run fails.

" + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#ToleratedFailureCount", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum number of failed items before the Map Run fails.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateMapRunOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidLoggingConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidTracingConfiguration" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#MissingRequiredParameter" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates an existing state machine by modifying its definition,\n roleArn, loggingConfiguration, or EncryptionConfiguration. Running executions will continue\n to use the previous definition and roleArn. You must include at\n least one of definition or roleArn or you will receive a\n MissingRequiredParameter error.

\n

A qualified state machine ARN refers to a Distributed Map state defined within a state machine. For example, the qualified state machine ARN arn:partition:states:region:account-id:stateMachine:stateMachineName/mapStateLabel refers to a Distributed Map state with a label mapStateLabel in the state machine named stateMachineName.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n
    \n
  • \n

    The following qualified state machine ARN refers to a Distributed Map state with a label mapStateLabel in a state machine named myStateMachine.

    \n

    \n arn:partition:states:region:account-id:stateMachine:myStateMachine/mapStateLabel\n

    \n \n

    If you provide a qualified state machine ARN that refers to a Distributed Map state, the request fails with ValidationException.

    \n
    \n
  • \n
  • \n

    The following qualified state machine ARN refers to an alias named PROD.

    \n

    \n arn::states:::stateMachine:\n

    \n \n

    If you provide a qualified state machine ARN that refers to a version ARN or an alias ARN, the request starts execution for that version or alias.

    \n
    \n
  • \n
  • \n

    The following unqualified state machine ARN refers to a state machine named myStateMachine.

    \n

    \n arn::states:::stateMachine:\n

    \n
  • \n
\n

After you update your state machine, you can set the publish parameter to\n true in the same action to publish a new version. This\n way, you can opt-in to strict versioning of your state machine.

\n \n

Step Functions assigns monotonically increasing integers for state machine versions, starting at version number 1.

\n
\n \n

All StartExecution calls within a few seconds use the updated\n definition and roleArn. Executions started immediately after you\n call UpdateStateMachine may use the previous state machine\n definition and roleArn.

\n
", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates the configuration of an existing state machine alias by modifying its description or routingConfiguration.

\n

You must specify at least one of the description or routingConfiguration parameters to update a state machine alias.

\n \n

\n UpdateStateMachineAlias is an idempotent API. Step Functions bases the\n idempotency check on the stateMachineAliasArn, description, and\n routingConfiguration parameters. Requests with the same parameters return an\n idempotent response.

\n
\n \n

This operation is eventually consistent. All StartExecution requests\n made within a few seconds use the latest alias configuration. Executions started immediately\n after calling UpdateStateMachineAlias may use the previous routing\n configuration.

\n
\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#UpdateStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

", + "smithy.api#required": {} + } + }, + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description of the state machine alias.

" + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of the state machine alias.

\n

An array of RoutingConfig objects that specifies up to two state machine versions that the alias starts executions for.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineAliasOutput": { + "type": "structure", + "members": { + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine alias was updated.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

" + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role of the state machine.

" + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration", + "traits": { + "smithy.api#documentation": "

Use the LoggingConfiguration data type to set CloudWatch Logs\n options.

" + } + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "publish": { + "target": "com.amazonaws.sfn#Publish", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Specifies whether the state machine version is published. The default is\n false. To publish a version after updating the state machine, set\n publish to true.

" + } + }, + "versionDescription": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

An optional description of the state machine version to publish.

\n

You can only specify the versionDescription parameter if you've set publish to true.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineOutput": { + "type": "structure", + "members": { + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine was updated.

", + "smithy.api#required": {} + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the updated state machine.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the published state machine version.

\n

If the publish parameter isn't set to true, this field returns null.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinition": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionInput" + }, + "output": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Validates the syntax of a state machine definition specified in Amazon States Language (ASL), a\n JSON-based, structured language.

\n

You can validate that a state machine definition is correct without creating a state\n machine resource.

\n

Suggested uses for ValidateStateMachineDefinition:

\n
    \n
  • \n

    Integrate automated checks into your code review or Continuous Integration\n (CI) process to check state machine definitions before starting\n deployments.

    \n
  • \n
  • \n

    Run validation from a Git pre-commit hook to verify the definition before\n committing to your source repository.

    \n
  • \n
\n

Validation will look for problems in your state machine definition and return a\n result and a list of diagnostic\n elements.

\n

The result value will be OK when your\n workflow definition can be successfully created or updated. Note the result can be\n OK even when diagnostic warnings are present in the response. The\n result value will be FAIL when the\n workflow definition contains errors that would prevent you from creating or updating\n your state machine.

\n

The list of ValidateStateMachineDefinitionDiagnostic data elements can contain zero or more WARNING and/or ERROR elements.

\n \n

The ValidateStateMachineDefinition API might add\n new diagnostics in the future, adjust diagnostic codes, or change the message\n wording. Your automated processes should only rely on the value of the result field value (OK, FAIL). Do not rely on the exact order, count, or\n wording of diagnostic messages.

\n
" + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionCode": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnostic": { + "type": "structure", + "members": { + "severity": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity", + "traits": { + "smithy.api#documentation": "

A value of ERROR means that you cannot create or update a state machine with this definition.

\n

\n WARNING level diagnostics alert you to potential issues, but they will not prevent you from creating or updating your state machine.

", + "smithy.api#required": {} + } + }, + "code": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionCode", + "traits": { + "smithy.api#documentation": "

Identifying code for the diagnostic.

", + "smithy.api#required": {} + } + }, + "message": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionMessage", + "traits": { + "smithy.api#documentation": "

Message describing the diagnostic condition.

", + "smithy.api#required": {} + } + }, + "location": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionLocation", + "traits": { + "smithy.api#documentation": "

Location of the issue in the state machine, if available.

\n

For errors specific to a field, the location could be in the format: /States//, for example: /States/FailState/ErrorPath.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Describes potential issues found during state machine validation. Rather than raise an\n exception, validation will return a list of diagnostic\n elements containing diagnostic information.

\n \n

The ValidateStateMachineDefinitionlAPI might add\n new diagnostics in the future, adjust diagnostic codes, or change the message\n wording. Your automated processes should only rely on the value of the result field value (OK, FAIL). Do not rely on the exact order, count, or\n wording of diagnostic messages.

\n
\n

\n List of warning codes\n

\n
\n
NO_DOLLAR
\n
\n

No .$ on a field that appears to be a JSONPath or Intrinsic Function.

\n
\n
NO_PATH
\n
\n

Field value looks like a path, but field name does not end with 'Path'.

\n
\n
PASS_RESULT_IS_STATIC
\n
\n

Attempt to use a path in the result of a pass state.

\n
\n
\n

\n List of error codes\n

\n
\n
INVALID_JSON_DESCRIPTION
\n
\n

JSON syntax problem found.

\n
\n
MISSING_DESCRIPTION
\n
\n

Received a null or empty workflow input.

\n
\n
SCHEMA_VALIDATION_FAILED
\n
\n

Schema validation reported errors.

\n
\n
INVALID_RESOURCE
\n
\n

The value of a Task-state resource field is invalid.

\n
\n
MISSING_END_STATE
\n
\n

The workflow does not have a terminal state.

\n
\n
DUPLICATE_STATE_NAME
\n
\n

The same state name appears more than once.

\n
\n
INVALID_STATE_NAME
\n
\n

The state name does not follow the naming convention.

\n
\n
STATE_MACHINE_NAME_EMPTY
\n
\n

The state machine name has not been specified.

\n
\n
STATE_MACHINE_NAME_INVALID
\n
\n

The state machine name does not follow the naming convention.

\n
\n
STATE_MACHINE_NAME_TOO_LONG
\n
\n

The state name exceeds the allowed length.

\n
\n
STATE_MACHINE_NAME_ALREADY_EXISTS
\n
\n

The state name already exists.

\n
\n
DUPLICATE_LABEL_NAME
\n
\n

A label name appears more than once.

\n
\n
INVALID_LABEL_NAME
\n
\n

You have provided an invalid label name.

\n
\n
MISSING_TRANSITION_TARGET
\n
\n

The value of \"Next\" field doesn't match a known state name.

\n
\n
TOO_DEEPLY_NESTED
\n
\n

The states are too deeply nested.

\n
\n
" + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnosticList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnostic" + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionInput": { + "type": "structure", + "members": { + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. For more information, see\n Amazon States Language (ASL).

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

The target type of state machine for this definition. The default is STANDARD.

" + } + }, + "severity": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity", + "traits": { + "smithy.api#documentation": "

Minimum level of diagnostics to return. ERROR returns only ERROR diagnostics, whereas WARNING returns both WARNING and ERROR diagnostics. The default is ERROR.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionMaxResult", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of diagnostics that are returned per call. The default and maximum value is 100. Setting the value to 0 will also use the default of 100.

\n

If the number of diagnostics returned in the response exceeds maxResults, the value of the truncated field in the response will be set to true.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionLocation": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionMaxResult": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionMessage": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionOutput": { + "type": "structure", + "members": { + "result": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionResultCode", + "traits": { + "smithy.api#documentation": "

The result value will be OK when no syntax errors are found, or\n FAIL if the workflow definition does not pass verification.

", + "smithy.api#required": {} + } + }, + "diagnostics": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnosticList", + "traits": { + "smithy.api#documentation": "

An array of diagnostic errors and warnings found during validation of the state machine definition. Since warnings do not prevent deploying your workflow definition, the result value could be OK even when warning diagnostics are present in the response.

", + "smithy.api#required": {} + } + }, + "truncated": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionTruncated", + "traits": { + "smithy.api#documentation": "

The result value will be true if the number of diagnostics found in the workflow definition exceeds maxResults. When all diagnostics results are returned, the value will be false.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionResultCode": { + "type": "enum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "OK" + } + }, + "FAIL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAIL" + } + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity": { + "type": "enum", + "members": { + "ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ERROR" + } + }, + "WARNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WARNING" + } + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionTruncated": { + "type": "boolean" + }, + "com.amazonaws.sfn#ValidationException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "reason": { + "target": "com.amazonaws.sfn#ValidationExceptionReason", + "traits": { + "smithy.api#documentation": "

The input does not satisfy the constraints specified by an Amazon Web Services service.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

The input does not satisfy the constraints specified by an Amazon Web Services service.

", + "smithy.api#error": "client", + "smithy.api#httpError": 400 + } + }, + "com.amazonaws.sfn#ValidationExceptionReason": { + "type": "enum", + "members": { + "API_DOES_NOT_SUPPORT_LABELED_ARNS": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "API_DOES_NOT_SUPPORT_LABELED_ARNS" + } + }, + "MISSING_REQUIRED_PARAMETER": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MISSING_REQUIRED_PARAMETER" + } + }, + "CANNOT_UPDATE_COMPLETED_MAP_RUN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CANNOT_UPDATE_COMPLETED_MAP_RUN" + } + }, + "INVALID_ROUTING_CONFIGURATION": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "INVALID_ROUTING_CONFIGURATION" + } + } + } + }, + "com.amazonaws.sfn#VariableName": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VariableNameList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#VariableName" + } + }, + "com.amazonaws.sfn#VariableReferences": { + "type": "map", + "key": { + "target": "com.amazonaws.sfn#StateName" + }, + "value": { + "target": "com.amazonaws.sfn#VariableNameList" + }, + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VariableValue": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VersionDescription": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VersionWeight": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#includedDetails": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#truncated": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + } + } +} \ No newline at end of file diff --git a/codegen/smithy-model/stepfunctions.provenance.json b/codegen/smithy-model/stepfunctions.provenance.json new file mode 100644 index 00000000..e677fca5 --- /dev/null +++ b/codegen/smithy-model/stepfunctions.provenance.json @@ -0,0 +1,10 @@ +{ + "repository": "https://github.com/aws/api-models-aws", + "commit": "18e485ce1c8dbf29d47e5ce00d9b8513da95912c", + "path": "models/sfn/service/2016-11-23/sfn-2016-11-23.json", + "sha256": "38dd64c074f5c05320978970dafa6e0f199ab2e33b6065232002e8a603c5f6ed", + "service": "com.amazonaws.sfn#AWSStepFunctions", + "serviceVersion": "2016-11-23", + "retrievedAt": "2026-10-25", + "bytes": 370185 +} diff --git a/codegen/src/codegen.rs b/codegen/src/codegen.rs index 0331563c..824df942 100644 --- a/codegen/src/codegen.rs +++ b/codegen/src/codegen.rs @@ -3,15 +3,15 @@ //! This module takes a `ResolvedModel` and produces Rust source code strings //! for types, input/output structs, operations, errors, and request types. -use std::collections::BTreeMap; -use std::fmt::Write; - -use anyhow::Result; +use std::{collections::BTreeMap, fmt::Write}; +use anyhow::{Result, bail}; use heck::{ToLowerCamelCase, ToPascalCase}; -use crate::config::ServiceConfig; -use crate::shapes::{EnumVariantInfo, FieldInfo, HttpBinding, OperationCategories, ResolvedModel}; +use crate::{ + config::{Protocol, ServiceConfig}, + shapes::{EnumVariantInfo, FieldInfo, HttpBinding, OperationCategories, ResolvedModel}, +}; /// Header comment placed at the top of every generated file. fn file_header(config: &ServiceConfig) -> String { @@ -21,12 +21,39 @@ fn file_header(config: &ServiceConfig) -> String { ) } +fn json_epoch_protocol(config: &ServiceConfig) -> bool { + matches!(config.protocol, Protocol::AwsJson1_0 | Protocol::AwsJson1_1) +} +fn epoch_field(field: &FieldInfo, config: &ServiceConfig) -> bool { + json_epoch_protocol(config) + && field + .timestamp_format + .as_deref() + .is_some_and(|format| matches!(format, "" | "epoch-seconds")) +} +fn needs_epoch_module(resolved: &ResolvedModel, config: &ServiceConfig) -> bool { + config.uses_serde() + && resolved + .input_structs + .values() + .chain(resolved.output_structs.values()) + .chain(resolved.shared_structs.values()) + .flatten() + .any(|field| epoch_field(field, config)) +} + /// Generate all source files and return them as a map of path -> content. pub fn generate_all( resolved: &ResolvedModel, config: &ServiceConfig, ) -> Result> { let mut files = BTreeMap::new(); + if needs_epoch_module(resolved, config) { + files.insert( + "epoch_seconds.rs".to_owned(), + include_str!("../templates/epoch_seconds.rs").to_owned(), + ); + } files.insert("types.rs".to_owned(), generate_types(resolved, config)?); files.insert("operations.rs".to_owned(), generate_operations(config)?); @@ -113,7 +140,7 @@ fn generate_types(resolved: &ResolvedModel, config: &ServiceConfig) -> Result Result<()> { - writeln!(out, "/// {display_name} {name} enum.")?; + writeln!(out, "/// {display_name} `{name}` enum.")?; writeln!( out, "#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]" @@ -139,8 +167,12 @@ fn write_enum( writeln!(out, "pub enum {name} {{")?; for (i, variant) in variants.iter().enumerate() { - if i == 0 { + if config.strict_model { + writeln!(out, " /// The `{}` wire variant.", variant.string_value)?; + } else if i == 0 { writeln!(out, " /// Default variant.")?; + } + if i == 0 { writeln!(out, " #[default]")?; } if variant.rust_name != variant.string_value { @@ -229,23 +261,41 @@ fn write_struct( display_name: &str, config: &ServiceConfig, ) -> Result<()> { - writeln!(out, "/// {display_name} {name}.")?; + writeln!(out, "/// {display_name} `{name}`.")?; if config.uses_serde() { - writeln!( - out, - "#[derive(Debug, Clone, Default, Serialize, Deserialize)]" - )?; + if config.strict_model { + writeln!(out, "#[derive(Clone, Default, Serialize, Deserialize)]")?; + } else { + writeln!( + out, + "#[derive(Debug, Clone, Default, Serialize, Deserialize)]" + )?; + } if let Some(ref rename) = config.serde_rename { writeln!(out, "#[serde(rename_all = \"{rename}\")]")?; } + } else if config.strict_model { + writeln!(out, "#[derive(Clone, Default)]")?; } else { writeln!(out, "#[derive(Debug, Clone, Default)]")?; } + if config.strict_model { + writeln!(out, "#[non_exhaustive]")?; + if config.uses_serde() { + writeln!(out, "#[serde(deny_unknown_fields)]")?; + } + if fields.len() > 5 { + writeln!(out, "#[derive(typed_builder::TypedBuilder)]")?; + } + } writeln!(out, "pub struct {name} {{")?; for field in fields { + if config.strict_model { + writeln!(out, " /// The `{}` wire member.", field.smithy_name)?; + } // Write HTTP binding comment if requested if include_http_comments && let Some(ref binding) = field.http_binding { let comment = match binding { @@ -260,8 +310,35 @@ fn write_struct( writeln!(out, "{comment}")?; } + if config.strict_model && fields.len() > 5 && !field.required { + writeln!(out, " #[builder(default)]")?; + } // Write serde field attributes when serde is enabled if config.uses_serde() { + if json_epoch_protocol(config) { + match field.timestamp_format.as_deref() { + Some("" | "epoch-seconds") => { + if field.rust_type.starts_with("Option<") { + writeln!( + out, + " #[serde(default, with = \"crate::epoch_seconds::option\")]" + )?; + } else { + writeln!(out, " #[serde(with = \"crate::epoch_seconds\")]")?; + } + } + Some("date-time") => {} + Some(_) => bail!( + "unsupported JSON timestamp format for {}", + field.smithy_name + ), + None if field.rust_type.contains("chrono::DateTime") => bail!( + "timestamp collection requires an explicit adapter: {}", + field.smithy_name + ), + None => {} + } + } // Explicit rename if rename_all would not produce the correct Smithy name if let Some(ref strategy) = config.serde_rename && needs_explicit_rename(&field.rust_name, &field.smithy_name, strategy) @@ -274,6 +351,11 @@ fn write_struct( out, " #[serde(skip_serializing_if = \"Option::is_none\")]" )?; + } else if config.strict_model + && field.required + && (field.rust_type.starts_with("Vec<") || field.rust_type.starts_with("HashMap<")) + { + // Required collections must be present, including when empty. } else if field.rust_type.starts_with("Vec<") { if config.always_serialize_arrays { writeln!(out, " #[serde(default)]")?; @@ -296,6 +378,18 @@ fn write_struct( writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl std::fmt::Debug for {name} {{")?; + writeln!( + out, + " fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{" + )?; + writeln!( + out, + " f.debug_struct(\"{name}\").finish_non_exhaustive()" + )?; + writeln!(out, " }}\n}}")?; + } Ok(()) } @@ -549,12 +643,12 @@ fn generate_operations(config: &ServiceConfig) -> Result { writeln!(out, "{header}")?; writeln!(out)?; - writeln!(out, "/// All supported {prefix} operations.")?; + writeln!(out, "/// All supported `{prefix}Operation` values.")?; writeln!(out, "#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]")?; writeln!(out, "pub enum {prefix}Operation {{")?; for op in all_operations { - writeln!(out, " /// The {op} operation.")?; + writeln!(out, " /// The `{op}` operation.")?; writeln!(out, " {op},")?; } @@ -577,7 +671,7 @@ fn generate_operations(config: &ServiceConfig) -> Result { // Generate from_str writeln!( out, - " /// Parse an operation name string into an {prefix}Operation." + " /// Parse an operation name string into a `{prefix}Operation`." )?; writeln!(out, " #[must_use]")?; writeln!(out, " pub fn from_name(name: &str) -> Option {{")?; @@ -668,7 +762,7 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu } for err in errors { - writeln!(out, " /// {} error.", err.name)?; + writeln!(out, " /// `{}` error.", err.name)?; if err.name == default_name { writeln!(out, " #[default]")?; } @@ -746,9 +840,35 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl {prefix}ErrorCode {{")?; + writeln!(out, " /// Default bounded protocol message.")?; + writeln!(out, " #[must_use]")?; + writeln!(out, " pub fn default_message(&self) -> &'static str {{")?; + writeln!(out, " match self {{")?; + for error in errors { + let message = config + .custom_errors + .get(&error.name) + .map_or(error.wire_name.as_str(), |custom| custom.message.as_str()); + writeln!(out, " Self::{} => {message:?},", error.name)?; + } + if errors.is_empty() { + writeln!( + out, + " Self::InternalException => \"InternalException\"," + )?; + } + writeln!(out, " }}\n }}\n}}")?; + } + // Error struct (no resource/request_id for JSON services) writeln!(out, "/// An {display_name} error response.")?; - writeln!(out, "#[derive(Debug)]")?; + if config.strict_model { + writeln!(out, "#[non_exhaustive]")?; + } else { + writeln!(out, "#[derive(Debug)]")?; + } writeln!(out, "pub struct {prefix}Error {{")?; writeln!(out, " /// The error code.")?; writeln!(out, " pub code: {prefix}ErrorCode,")?; @@ -764,16 +884,29 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl fmt::Debug for {prefix}Error {{")?; + writeln!( + out, + " fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {{" + )?; + writeln!(out, " f.write_str(\"{prefix}Error()\")")?; + writeln!(out, " }}\n}}")?; + } // Display impl writeln!(out, "impl fmt::Display for {prefix}Error {{")?; writeln!( out, " fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {{" )?; - writeln!( - out, - " write!(f, \"{prefix}Error({{}}): {{}}\", self.code, self.message)" - )?; + if config.strict_model { + writeln!(out, " f.write_str(\"{prefix}Error()\")")?; + } else { + writeln!( + out, + " write!(f, \"{prefix}Error({{}}): {{}}\", self.code, self.message)" + )?; + } writeln!(out, " }}")?; writeln!(out, "}}")?; writeln!(out)?; @@ -802,7 +935,14 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, " pub fn new(code: {prefix}ErrorCode) -> Self {{")?; writeln!(out, " Self {{")?; writeln!(out, " status_code: code.default_status_code(),")?; - writeln!(out, " message: code.as_str().to_owned(),")?; + if config.strict_model { + writeln!( + out, + " message: code.default_message().to_owned()," + )?; + } else { + writeln!(out, " message: code.as_str().to_owned(),")?; + } writeln!(out, " code,")?; writeln!(out, " source: None,")?; writeln!(out, " }}")?; @@ -838,8 +978,8 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out)?; // Convenience constructors for common errors - // Generate validation(), internal_error(), missing_action(), unknown_operation(), not_implemented() - // based on which errors are present + // Generate validation(), internal_error(), missing_action(), unknown_operation(), + // not_implemented() based on which errors are present let has_error = |name: &str| errors.iter().any(|e| e.name == name); @@ -936,7 +1076,15 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "///")?; writeln!(out, "/// # Examples")?; writeln!(out, "///")?; - writeln!(out, "/// ```ignore")?; + if config.strict_model { + writeln!(out, "/// ```")?; + writeln!( + out, + "/// use rustack_{name}_model::{{{name}_error, {prefix}ErrorCode}};" + )?; + } else { + writeln!(out, "/// ```ignore")?; + } writeln!(out, "/// let err = {name}_error!({default_name});")?; writeln!( out, @@ -966,6 +1114,8 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu fn status_code_const(status: u16) -> &'static str { match status { 400 => "http::StatusCode::BAD_REQUEST", + 402 => "http::StatusCode::PAYMENT_REQUIRED", + 408 => "http::StatusCode::REQUEST_TIMEOUT", 401 => "http::StatusCode::UNAUTHORIZED", 403 => "http::StatusCode::FORBIDDEN", 404 => "http::StatusCode::NOT_FOUND", @@ -1385,7 +1535,8 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { writeln!(out, " #[must_use]")?; writeln!( out, - " pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {{" + " pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) \ + -> Self {{" )?; writeln!(out, " self.source = Some(Box::new(source));")?; writeln!(out, " self")?; @@ -1601,7 +1752,8 @@ fn generate_request(config: &ServiceConfig) -> Result { )?; writeln!( out, - " .field(\"session_token\", &self.session_token.as_ref().map(|_| \"[REDACTED]\"))" + " .field(\"session_token\", &self.session_token.as_ref().map(|_| \ + \"[REDACTED]\"))" )?; writeln!(out, " .finish()")?; writeln!(out, " }}")?; @@ -1684,9 +1836,16 @@ fn generate_lib(resolved: &ResolvedModel, config: &ServiceConfig) -> Result Result Result, + /// Whether AWS Query-compatible errors are configured. + pub aws_query_compatible: bool, /// Serde rename strategy (None means no rename). pub serde_rename: Option, /// Whether to emit HTTP binding comments. @@ -174,8 +181,6 @@ pub struct ServiceConfig { pub emit_request_wrapper: bool, /// Ordered list of all operations to generate. pub all_operations: Vec, - /// Map from operation name to phase number. - pub operation_phases: BTreeMap, /// Operation categories for file organization (if categorized layout). pub categories: Option>>, /// File layout strategy. @@ -188,6 +193,8 @@ pub struct ServiceConfig { pub overlay_extra_modules: Vec, /// Whether to always serialize arrays. pub always_serialize_arrays: bool, + /// Opt-in strict public model generation. + pub strict_model: bool, } /// Supported protocol variants. @@ -217,6 +224,9 @@ impl ServiceConfig { other => bail!("Unknown protocol: {other}"), }; + if config.protocol.error_type_format.is_some() || config.output.dir.is_some() { + bail!("custom error formats and output.dir are unsupported; use --output"); + } let serde_rename = match config.protocol.serde_rename.as_str() { "none" | "" => None, s => Some(s.to_owned()), @@ -225,22 +235,20 @@ impl ServiceConfig { // Build all_operations list. // Priority: explicit `all` list > phases > categories let mut all_operations = Vec::new(); - let mut operation_phases = BTreeMap::new(); if !config.operations.all.is_empty() { // Use the explicit ordered list all_operations = config.operations.all.clone(); } else { // Build from phases - for (phase_num, phase_ops) in [ - (0, &config.operations.phase0), - (1, &config.operations.phase1), - (2, &config.operations.phase2), - (3, &config.operations.phase3), + for phase_ops in [ + &config.operations.phase0, + &config.operations.phase1, + &config.operations.phase2, + &config.operations.phase3, ] { for op in phase_ops { all_operations.push(op.clone()); - operation_phases.insert(op.clone(), phase_num); } } @@ -269,18 +277,20 @@ impl ServiceConfig { rust_prefix: config.service.rust_prefix, namespace: config.service.namespace, protocol, + target_prefix: config.protocol.target_prefix, + aws_query_compatible: config.protocol.aws_query_compatible, serde_rename, emit_http_bindings: config.protocol.emit_http_bindings, emit_serde_derives: config.protocol.emit_serde_derives, emit_request_wrapper: config.protocol.emit_request_wrapper, all_operations, - operation_phases, categories, file_layout: config.output.file_layout, custom_errors: config.errors.custom, overlay_preserve: config.overlay.preserve, overlay_extra_modules: config.overlay.extra_modules, always_serialize_arrays: config.output.always_serialize_arrays, + strict_model: config.output.strict_model, }) } diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 61bbf7b2..6d28bece 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -3,20 +3,43 @@ //! Reads an AWS Smithy JSON AST model and generates Rust source files //! based on a TOML service configuration. +#![forbid(unsafe_code)] + mod codegen; mod config; mod model; mod shapes; +#[cfg(test)] +mod tests; -use std::fs; -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, ensure}; +use tokio::{fs, io::AsyncReadExt, time::timeout}; use crate::config::{ServiceConfig, ServiceConfigFile}; -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + timeout(Duration::from_secs(60), generate()) + .await + .context("Code generation deadline exceeded")? +} + +async fn generate() -> Result<()> { + let args: Vec = std::env::args_os() + .map(|arg| { + arg.into_string() + .map_err(|_| anyhow::anyhow!("Arguments must be UTF-8")) + }) + .collect::>()?; + ensure!( + args.len() <= 16 && args.iter().all(|arg| arg.len() <= 4096), + "Argument budget exceeded" + ); // Parse named arguments: --config, --model, --output let config_path = find_arg(&args, "--config"); @@ -34,7 +57,8 @@ fn main() -> Result<()> { ); // Read and parse the TOML config - let config_toml = fs::read_to_string(&config_path) + let config_toml = read_utf8(&config_path, 1024 * 1024) + .await .with_context(|| format!("Failed to read config file: {}", config_path.display()))?; let config_file: ServiceConfigFile = toml::from_str(&config_toml).context("Failed to parse TOML config")?; @@ -65,7 +89,8 @@ fn main() -> Result<()> { eprintln!("Writing output to: {}", output_dir.display()); // Read and parse the Smithy JSON model. - let model_json = fs::read_to_string(&model_path) + let model_json = read_utf8(&model_path, 16 * 1024 * 1024) + .await .with_context(|| format!("Failed to read model file: {}", model_path.display()))?; let smithy_model: model::SmithyModel = @@ -73,6 +98,8 @@ fn main() -> Result<()> { eprintln!("Parsed model: {} shapes", smithy_model.shapes.len()); + ensure!(smithy_model.smithy == "2.0", "Unsupported Smithy version"); + // Resolve shapes and types. let resolved = shapes::resolve_model(&smithy_model, &service_config) .context("Failed to resolve model shapes")?; @@ -93,8 +120,12 @@ fn main() -> Result<()> { // Write output files. for (rel_path, content) in &files { let full_path = output_dir.join(rel_path); - ensure_parent_dir(&full_path)?; + if service_config.overlay_preserve.contains(rel_path) && fs::try_exists(&full_path).await? { + continue; + } + ensure_parent_dir(&full_path).await?; fs::write(&full_path, content) + .await .with_context(|| format!("Failed to write {}", full_path.display()))?; eprintln!(" Wrote: {}", full_path.display()); } @@ -113,10 +144,27 @@ fn find_arg(args: &[String], flag: &str) -> Option { } /// Ensure the parent directory of a path exists. -fn ensure_parent_dir(path: &Path) -> Result<()> { +async fn ensure_parent_dir(path: &Path) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) + .await .with_context(|| format!("Failed to create directory: {}", parent.display()))?; } Ok(()) } + +/// Read ordinary developer-owned input files under an explicit byte ceiling. +async fn read_utf8(path: &Path, maximum: u64) -> Result { + ensure!( + fs::metadata(path).await?.is_file(), + "Codegen input must be an ordinary file" + ); + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(maximum + 1).read_to_end(&mut bytes).await?; + ensure!( + u64::try_from(bytes.len())? <= maximum, + "Codegen input byte limit exceeded" + ); + String::from_utf8(bytes).context("Invalid UTF-8 codegen input") +} diff --git a/codegen/src/model.rs b/codegen/src/model.rs index a43aab65..f0267256 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; -use serde::Deserialize; +use serde::{Deserialize, de::IgnoredAny}; /// Top-level Smithy model document. #[derive(Debug, Deserialize)] @@ -31,13 +31,13 @@ pub enum Shape { String(SimpleShape), /// A boolean shape. #[serde(rename = "boolean")] - Boolean(SimpleShape), + Boolean(IgnoredAny), /// A 32-bit integer shape. #[serde(rename = "integer")] - Integer(SimpleShape), + Integer(IgnoredAny), /// A 64-bit integer shape. #[serde(rename = "long")] - Long(SimpleShape), + Long(IgnoredAny), /// A timestamp shape. #[serde(rename = "timestamp")] Timestamp(SimpleShape), @@ -58,19 +58,19 @@ pub enum Shape { Union(UnionShape), /// A service shape (ignored). #[serde(rename = "service")] - Service(SimpleShape), + Service(IgnoredAny), /// A resource shape (ignored). #[serde(rename = "resource")] - Resource(SimpleShape), + Resource(IgnoredAny), /// An integer enum shape (ignored). #[serde(rename = "intEnum")] - IntEnum(SimpleShape), + IntEnum(IgnoredAny), /// A double shape. #[serde(rename = "double")] - Double(SimpleShape), + Double(IgnoredAny), /// A float shape. #[serde(rename = "float")] - Float(SimpleShape), + Float(IgnoredAny), } /// A simple shape with optional traits. @@ -112,9 +112,6 @@ pub struct OperationShape { /// Error shape references. #[serde(default)] pub errors: Vec, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A reference to another shape. @@ -130,9 +127,6 @@ pub struct EnumShape { /// Enum variants (member name -> member shape). #[serde(default)] pub members: HashMap, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A list shape. @@ -140,9 +134,6 @@ pub struct EnumShape { pub struct ListShape { /// The shape of the list's elements. pub member: ShapeRef, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A map shape. @@ -152,9 +143,6 @@ pub struct MapShape { pub key: ShapeRef, /// The shape of the map's values. pub value: ShapeRef, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A union shape. @@ -163,9 +151,6 @@ pub struct UnionShape { /// Union variants. #[serde(default)] pub members: HashMap, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } impl SmithyModel { diff --git a/codegen/src/shapes.rs b/codegen/src/shapes.rs index 478895dd..dce3f9c7 100644 --- a/codegen/src/shapes.rs +++ b/codegen/src/shapes.rs @@ -7,8 +7,10 @@ use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result}; -use crate::config::ServiceConfig; -use crate::model::{MemberShape, Shape, SmithyModel, StructureShape}; +use crate::{ + config::ServiceConfig, + model::{MemberShape, Shape, SmithyModel, StructureShape}, +}; /// Maximum recursion depth when resolving shape references. const MAX_RESOLVE_DEPTH: usize = 15; @@ -38,6 +40,8 @@ pub struct FieldInfo { pub rust_type: String, /// Whether this field is required. pub required: bool, + /// Timestamp format; empty means protocol default, None means not a timestamp. + pub timestamp_format: Option, /// HTTP binding annotation if any. pub http_binding: Option, } @@ -75,8 +79,6 @@ pub struct ErrorInfo { pub wire_name: String, /// Default HTTP status code (from `smithy.api#httpError`, or 400/500 based on fault). pub status_code: u16, - /// Default human-readable message. - pub message: String, /// Fault type: `"client"` or `"server"`. pub fault: String, } @@ -331,12 +333,33 @@ fn resolve_struct_fields( rust_name: to_snake_case(name), rust_type, required, + timestamp_format: timestamp_format(model, member), http_binding, } }) .collect() } +/// Resolve a timestamp member override before its target-shape override. +fn timestamp_format(model: &SmithyModel, member: &MemberShape) -> Option { + let shape = model.shapes.get(&member.target); + let target_traits = match shape { + Some(Shape::Timestamp(shape)) => Some(&shape.traits), + _ if member.target == "smithy.api#Timestamp" => None, + _ => return None, + }; + Some( + member + .traits + .get("smithy.api#timestampFormat") + .or_else(|| target_traits.and_then(|traits| traits.get("smithy.api#timestampFormat"))) + .map_or("", |value| { + value.as_str().unwrap_or("") + }) + .to_owned(), + ) +} + /// Check if a type is a collection type that should not be wrapped in Option. fn is_collection_type(ty: &str) -> bool { ty.starts_with("Vec<") || ty.starts_with("HashMap<") @@ -506,6 +529,7 @@ pub fn resolve_model(model: &SmithyModel, config: &ServiceConfig) -> Result"), required: false, + timestamp_format: timestamp_format(model, member), http_binding: None, } }) @@ -565,14 +589,11 @@ fn collect_errors( // for a cleaner Rust API, but keep the wire name for serialization) let name = wire_name.clone(); - let message = name.clone(); - seen_names.insert(name.clone()); errors.push(ErrorInfo { name, wire_name, status_code, - message, fault, }); } @@ -585,7 +606,6 @@ fn collect_errors( name: name.clone(), wire_name: name.clone(), status_code: custom.status, - message: custom.message.clone(), fault: if custom.status >= 500 { "server".to_owned() } else { diff --git a/codegen/src/tests.rs b/codegen/src/tests.rs new file mode 100644 index 00000000..14e90432 --- /dev/null +++ b/codegen/src/tests.rs @@ -0,0 +1,171 @@ +//! Generator protocol-policy tests independent of generated runtime fixtures. + +use serde_json::json; + +use crate::{ + codegen::generate_all, + config::{CustomError, ServiceConfig, ServiceConfigFile}, + model::SmithyModel, + shapes::resolve_model, +}; + +fn config(protocol: &str, strict: bool) -> ServiceConfig { + let text = format!( + r#" +[service] +name="fixture" +display_name="Fixture" +rust_prefix="Fixture" +namespace="fixture" +protocol="{protocol}" +[protocol] +serde_rename="camelCase" +emit_serde_derives=true +[operations] +all=["Run"] +[output] +file_layout="flat" +strict_model={strict} +"# + ); + let file: ServiceConfigFile = toml::from_str(&text).unwrap(); + ServiceConfig::from_file(file).unwrap() +} +fn model(target: Option<&str>, member: Option<&str>, collection: bool) -> SmithyModel { + let mut value = json!({"smithy":"2.0","shapes":{ + "fixture#Run":{"type":"operation","input":{"target":"fixture#RunRequest"}}, + "fixture#RunRequest":{"type":"structure","members":{ + "required":{"target":"fixture#Time","traits":{"smithy.api#required":{}}}, + "optional":{"target":"fixture#Time"}, + "plain":{"target":"smithy.api#String"}}}, + "fixture#Time":{"type":"timestamp","traits":{}} + }}); + if let Some(format) = target { + value["shapes"]["fixture#Time"]["traits"]["smithy.api#timestampFormat"] = json!(format); + } + if let Some(format) = member { + value["shapes"]["fixture#RunRequest"]["members"]["required"]["traits"] + ["smithy.api#timestampFormat"] = json!(format); + } + if collection { + value["shapes"]["fixture#Times"] = + json!({"type":"list","member":{"target":"fixture#Time"}}); + value["shapes"]["fixture#RunRequest"]["members"]["times"] = + json!({"target":"fixture#Times"}); + } + serde_json::from_value(value).unwrap() +} +#[test] +fn test_should_emit_numeric_codecs_only_for_json_protocol_defaults() { + for protocol in [ + "awsJson1_0", + "awsJson1_1", + "restJson1", + "awsQuery", + "restXml", + ] { + let config = config(protocol, true); + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + let numeric = protocol.starts_with("awsJson"); + assert_eq!(files.contains_key("epoch_seconds.rs"), numeric); + assert_eq!( + files["input.rs"].contains("with = \"crate::epoch_seconds"), + numeric + ); + if numeric { + assert!(files["input.rs"].contains("default, with = \"crate::epoch_seconds::option\"")); + } + } +} +#[test] +fn test_should_resolve_member_override_before_shape_before_protocol_default() { + let config = config("awsJson1_0", true); + for (target, member, expected) in [ + (Some("date-time"), Some("epoch-seconds"), "epoch-seconds"), + (Some("epoch-seconds"), Some("date-time"), "date-time"), + (Some("date-time"), None, "date-time"), + (None, None, ""), + ] { + let resolved = resolve_model(&model(target, member, false), &config).unwrap(); + let field = resolved.input_structs["RunInput"] + .iter() + .find(|field| field.smithy_name == "required") + .unwrap(); + assert_eq!(field.timestamp_format.as_deref(), Some(expected)); + let files = generate_all(&resolved, &config).unwrap(); + let required = files["input.rs"].split("pub required:").next().unwrap(); + assert_eq!( + required + .lines() + .rev() + .find(|line| !line.trim().is_empty()) + .unwrap() + .contains("with = \"crate::epoch_seconds\""), + expected != "date-time" + ); + } +} +#[test] +fn test_should_reject_unsupported_explicit_formats_and_unadapted_collections() { + let config = config("awsJson1_0", true); + for model in [ + model(None, Some("http-date"), false), + model(None, None, true), + ] { + let resolved = resolve_model(&model, &config).unwrap(); + assert!(generate_all(&resolved, &config).is_err()); + } +} +#[test] +fn test_should_keep_strict_models_redacted_documented_and_without_blanket_ignores() { + let config = config("awsJson1_0", true); + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + assert!(files["input.rs"].contains("#[serde(deny_unknown_fields)]")); + assert!(files["input.rs"].contains("#[non_exhaustive]")); + assert!(files["input.rs"].contains("impl std::fmt::Debug for RunInput")); + assert!(files["input.rs"].contains("/// The `required` wire member.")); + assert!(files.values().all(|file| !file.contains("#[allow(") + && !file.contains("#![allow(") + && !file.contains("```ignore"))); + assert_eq!(files, generate_all(&resolved, &config).unwrap()); +} + +#[test] +fn test_should_emit_one_redacted_debug_without_serde_derives() { + for protocol in ["restXml", "awsJson1_0", "awsJson1_1"] { + let mut config = config(protocol, true); + config.emit_serde_derives = false; + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + assert!(files["input.rs"].contains("#[derive(Clone, Default)]")); + assert!(!files["input.rs"].contains("derive(Debug")); + assert_eq!( + files["input.rs"] + .matches("impl std::fmt::Debug for RunInput") + .count(), + 1 + ); + assert!(!files.contains_key("epoch_seconds.rs")); + assert!(!files["lib.rs"].contains("mod epoch_seconds")); + } +} + +#[test] +fn test_should_preserve_payment_required_and_request_timeout_error_statuses() { + let mut config = config("awsJson1_0", true); + for (name, status) in [("Quota", 402), ("Timeout", 408)] { + config.custom_errors.insert( + name.into(), + CustomError { + status, + message: "static".into(), + }, + ); + } + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + assert!(files["error.rs"].contains("Self::Quota => http::StatusCode::PAYMENT_REQUIRED")); + assert!(files["error.rs"].contains("Self::Timeout => http::StatusCode::REQUEST_TIMEOUT")); +} diff --git a/codegen/templates/epoch_seconds.rs b/codegen/templates/epoch_seconds.rs new file mode 100644 index 00000000..36ab79c9 --- /dev/null +++ b/codegen/templates/epoch_seconds.rs @@ -0,0 +1,90 @@ +//! Generated JSON epoch-second adapters with exact decimal millisecond precision. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as DeError, ser::Error as SerError}; +use serde_json::value::RawValue; + +const INVALID: &str = "invalid epoch-second timestamp"; + +/// Serialize a required timestamp as a numeric JSON epoch value. +/// +/// # Errors +/// Rejects leap seconds, invalid numeric representation or serializer failures. +pub fn serialize(value: &DateTime, serializer: S) -> Result { + if value.timestamp_subsec_nanos() >= 1_000_000_000 { return Err(S::Error::custom(INVALID)); } + let nanos = i128::from(value.timestamp()).checked_mul(1_000_000_000) + .and_then(|seconds| seconds.checked_add(i128::from(value.timestamp_subsec_nanos()))) + .ok_or_else(|| S::Error::custom(INVALID))?; + let millis = nanos / 1_000_000; + let sign = if millis < 0 { "-" } else { "" }; + let magnitude = millis.unsigned_abs(); + let seconds = magnitude / 1000; + let fraction = magnitude % 1000; + let number = if fraction == 0 { format!("{sign}{seconds}") } else { format!("{sign}{seconds}.{fraction:03}") }; + RawValue::from_string(number).map_err(|_| S::Error::custom(INVALID))?.serialize(serializer) +} + +/// Deserialize a required JSON number, truncating submillisecond precision. +/// +/// # Errors +/// Rejects nonnumeric, oversized, out-of-range or invalid epoch values. +pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let value = Box::::deserialize(deserializer)?; + parse(value.get()).ok_or_else(|| D::Error::custom(INVALID)) +} + +fn parse(raw: &str) -> Option> { + if raw.len() > 128 { return None; } + let (negative, raw) = raw.strip_prefix('-').map_or((false, raw), |rest| (true, rest)); + let (mantissa, exponent) = match raw.split_once(['e', 'E']) { + Some((value, exponent)) => { + let (negative, digits) = exponent.strip_prefix('-').map_or_else( + || (false, exponent.strip_prefix('+').unwrap_or(exponent)), |digits| (true, digits)); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { return None; } + let magnitude = digits.bytes().fold(0_i32, |value, digit| + value.saturating_mul(10).saturating_add(i32::from(digit - b'0')).min(1024)); + (value, if negative { -magnitude } else { magnitude }) + }, + None => (raw, 0), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if whole.is_empty() || !whole.bytes().chain(fraction.bytes()).all(|byte| byte.is_ascii_digit()) { return None; } + let digits: Vec<_> = whole.bytes().chain(fraction.bytes()).collect(); + if digits.iter().all(|digit| *digit == b'0') { return DateTime::from_timestamp_millis(0); } + let keep = i32::try_from(whole.len()).ok()?.checked_add(exponent)?.checked_add(3)?; + if keep <= 0 { return DateTime::from_timestamp_millis(0); } + // Leading zeroes must not make a small representable exponent look huge. + let leading = digits.iter().take_while(|digit| **digit == b'0').count(); + let significant = keep.checked_sub(i32::try_from(leading).ok()?)?; + if significant <= 0 { return DateTime::from_timestamp_millis(0); } + if significant > 19 { return None; } + let mut millis = 0_i128; + let count = usize::try_from(significant).ok()?; + for offset in 0..count { + let digit = digits.get(leading.checked_add(offset)?).copied().unwrap_or(b'0'); + millis = millis.checked_mul(10)?.checked_add(i128::from(digit.checked_sub(b'0')?))?; + } + let signed = if negative { millis.checked_neg()? } else { millis }; + DateTime::from_timestamp_millis(i64::try_from(signed).ok()?) +} + +/// Optional JSON timestamp adapter; missing members also need serde default. +pub mod option { + use std::borrow::Borrow; + use super::{DateTime, Deserialize, Deserializer, DeError, RawValue, Serializer, Utc, INVALID}; + + /// Serialize a timestamp or JSON null. + /// + /// # Errors + /// Returns the required adapter's validation/serialization errors. + pub fn serialize>>>(value: V, serializer: S) -> Result { + match value.borrow() { Some(value) => super::serialize(value, serializer), None => serializer.serialize_none() } + } + /// Deserialize a timestamp or JSON null. + /// + /// # Errors + /// Rejects invalid nonnull values with the required adapter's rules. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result>, D::Error> { + Option::>::deserialize(deserializer)?.map(|value| super::parse(value.get()).ok_or_else(|| D::Error::custom(INVALID))).transpose() + } +} diff --git a/crates/rustack-auth/src/canonical.rs b/crates/rustack-auth/src/canonical.rs index 7c72b632..a819e04f 100644 --- a/crates/rustack-auth/src/canonical.rs +++ b/crates/rustack-auth/src/canonical.rs @@ -22,7 +22,7 @@ use crate::error::AuthError; /// The set of characters that must be percent-encoded in URI path segments. /// -/// Per AWS SigV4 spec, all characters except unreserved characters +/// Per AWS `SigV4` spec, all characters except unreserved characters /// (A-Z, a-z, 0-9, `-`, `_`, `.`, `~`) must be encoded. /// Forward slashes in the path are preserved (not encoded). const URI_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC @@ -116,7 +116,7 @@ pub fn build_canonical_uri(path: &str) -> String { /// Parameters are sorted by key name first, then by value for duplicate keys. /// The raw query string values are preserved as-is (no decode/re-encode) because /// different clients use different encoding rules when signing. For example, -/// AWS SDKs percent-encode `:` and `*` but minio-java (via OkHttp) leaves them +/// AWS SDKs percent-encode `:` and `*` but minio-java (via `OkHttp`) leaves them /// raw. The server must use the exact same encoding the client used for signing, /// which is whatever appears in the HTTP request. /// @@ -187,10 +187,10 @@ pub(crate) fn validate_unique_query_names(query: &str) -> Result<(), AuthError> Ok(()) } -/// Build the canonical query string following the AWS SigV4 specification. +/// Build the canonical query string following the AWS `SigV4` specification. /// /// Each parameter name and value is first percent-decoded (so already-encoded -/// input is not double-encoded) and then re-encoded using the SigV4 encoding +/// input is not double-encoded) and then re-encoded using the `SigV4` encoding /// rules: every character except the unreserved set (`A-Z`, `a-z`, `0-9`, `-`, /// `_`, `.`, `~`) is percent-encoded with uppercase hex. Parameters are then /// sorted by encoded name. Duplicate decoded names are rejected because S3 @@ -209,7 +209,7 @@ pub(crate) fn validate_unique_query_names(query: &str) -> Result<(), AuthError> /// signature collisions where tampered parameters still verify. /// /// A literal `+` is treated as a literal plus (re-encoded as `%2B`), per the -/// SigV4 encoding rules. Note that downstream decoders disagree on the +/// `SigV4` encoding rules. Note that downstream decoders disagree on the /// meaning of a raw `+` (S3 decodes a literal plus, API Gateway and Lambda /// decode a space), so verification callers must NOT attempt the normalized /// fallback when the wire query contains a raw `+`: no single normalized @@ -275,7 +275,7 @@ pub fn build_canonical_query_string_normalized(query: &str) -> Result String { sorted.join(";") } -/// Percent-encode a string using the AWS SigV4 encoding rules. +/// Percent-encode a string using the AWS `SigV4` encoding rules. /// /// Every character except the unreserved set (`A-Z`, `a-z`, `0-9`, `-`, `_`, /// `.`, `~`) is percent-encoded with uppercase hexadecimal digits. @@ -374,7 +374,7 @@ fn sigv4_encode(input: &str) -> String { utf8_percent_encode(input, URI_ENCODE_SET).to_string() } -/// URI-encode a single path segment using the AWS SigV4 encoding rules. +/// URI-encode a single path segment using the AWS `SigV4` encoding rules. fn uri_encode(input: &str) -> String { sigv4_encode(input) } diff --git a/crates/rustack-auth/src/credentials.rs b/crates/rustack-auth/src/credentials.rs index b7095120..1a0736ef 100644 --- a/crates/rustack-auth/src/credentials.rs +++ b/crates/rustack-auth/src/credentials.rs @@ -44,7 +44,7 @@ pub struct StaticCredentialProvider { } impl StaticCredentialProvider { - /// Create a new `StaticCredentialProvider` from an iterable of (access_key_id, secret_key) + /// Create a new `StaticCredentialProvider` from an iterable of (`access_key_id`, `secret_key`) /// pairs. pub fn new(credentials: impl IntoIterator) -> Self { Self { diff --git a/crates/rustack-auth/src/error.rs b/crates/rustack-auth/src/error.rs index 31218dae..e4449807 100644 --- a/crates/rustack-auth/src/error.rs +++ b/crates/rustack-auth/src/error.rs @@ -1,11 +1,21 @@ -//! Error types for SigV4 authentication. +//! Error types for `SigV4` authentication. //! //! All authentication failures are represented by [`AuthError`], which provides //! specific variants for each failure mode encountered during signature verification. +use digest::InvalidLength; + /// Errors that can occur during AWS Signature Version 4 authentication. #[derive(Debug, thiserror::Error)] pub enum AuthError { + /// The cryptographic provider rejected MAC initialization. + #[error("Cryptographic initialization failed")] + CryptoInitialization { + /// Original provider error, without key or request contents. + #[source] + source: InvalidLength, + }, + /// Strict authentication requires a credential provider. #[error("Strict authentication requires a credential provider")] MissingCredentialProvider, diff --git a/crates/rustack-auth/src/lib.rs b/crates/rustack-auth/src/lib.rs index 435bd06b..89c8c968 100644 --- a/crates/rustack-auth/src/lib.rs +++ b/crates/rustack-auth/src/lib.rs @@ -1,9 +1,9 @@ //! AWS Signature Version 2 and 4 request authentication for Rustack. //! -//! This crate provides SigV4 and SigV2 signature verification for incoming HTTP +//! This crate provides `SigV4` and `SigV2` signature verification for incoming HTTP //! requests to AWS-compatible services. It supports header-based authentication //! (via the `Authorization` header), presigned URL authentication (via query -//! parameters), and legacy SigV2 authentication (HMAC-SHA1). +//! parameters), and legacy `SigV2` authentication (HMAC-SHA1). //! //! # Overview //! @@ -29,12 +29,12 @@ //! //! # Modules //! -//! - [`canonical`] - Canonical request construction per the SigV4 specification +//! - [`canonical`] - Canonical request construction per the `SigV4` specification //! - [`credentials`] - Credential provider trait and in-memory implementation //! - [`error`] - Authentication error types //! - [`presigned`] - Presigned URL verification -//! - [`sigv2`] - Legacy SigV2 signature verification (HMAC-SHA1) -//! - [`sigv4`] - Main SigV4 signature verification logic +//! - [`sigv2`] - Legacy `SigV2` signature verification (HMAC-SHA1) +//! - [`sigv4`] - Main `SigV4` signature verification logic pub mod canonical; pub mod credentials; diff --git a/crates/rustack-auth/src/presigned.rs b/crates/rustack-auth/src/presigned.rs index b28175bb..053403c1 100644 --- a/crates/rustack-auth/src/presigned.rs +++ b/crates/rustack-auth/src/presigned.rs @@ -116,9 +116,9 @@ pub fn parse_presigned_params(query: &str) -> Result = credential.splitn(5, '/').collect(); - if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { + let [access_key_id, date, region, service, "aws4_request"] = cred_parts.as_slice() else { return Err(AuthError::InvalidCredential); - } + }; let expires: u64 = expires_str .parse() @@ -131,10 +131,10 @@ pub fn parse_presigned_params(query: &str) -> Result Result Result bool { + let signature_matches = |canonical_query: &str| -> Result { let canonical_request = format!( "{method}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n\n{signed_headers_str}\n{UNSIGNED_PAYLOAD}" ); let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); let string_to_sign = build_string_to_sign(&parsed.timestamp, &credential_scope, &canonical_hash); - let expected_signature = compute_signature(&signing_key, &string_to_sign); - bool::from(provided_bytes.ct_eq(expected_signature.as_bytes())) + let expected_signature = compute_signature(&signing_key, &string_to_sign)?; + Ok(bool::from(provided_bytes.ct_eq(expected_signature.as_bytes()))) }; debug!("Built presigned canonical request"); @@ -235,7 +236,7 @@ pub fn verify_presigned( // `X-Amz-Signature`). This matches clients that sign whatever encoding // appears in the URL. let canonical_query = build_canonical_query_string_without_signature(query); - if signature_matches(&canonical_query) { + if signature_matches(&canonical_query)? { debug!(access_key_id = %parsed.access_key_id, "Presigned URL verification succeeded"); return Ok(AuthResult { access_key_id: parsed.access_key_id, @@ -262,9 +263,10 @@ pub fn verify_presigned( // is impossible; the raw attempt above already failed, so fall through to // the signature mismatch below. let normalized_matches = !query.contains('+') - && build_canonical_query_string_without_signature_normalized(query).is_ok_and( - |normalized| normalized != canonical_query && signature_matches(&normalized), - ); + && match build_canonical_query_string_without_signature_normalized(query) { + Ok(normalized) if normalized != canonical_query => signature_matches(&normalized)?, + _ => false, + }; if normalized_matches { debug!(access_key_id = %parsed.access_key_id, "Presigned URL verification succeeded (normalized query string)"); return Ok(AuthResult { @@ -287,9 +289,9 @@ fn build_canonical_query_string_without_signature(query: &str) -> String { build_canonical_query_string(&filter_signature_param(query)) } -/// Build the SigV4-normalized canonical query string excluding `X-Amz-Signature`. +/// Build the `SigV4`-normalized canonical query string excluding `X-Amz-Signature`. /// -/// The remaining parameters are percent-decoded, re-encoded per the SigV4 spec, +/// The remaining parameters are percent-decoded, re-encoded per the `SigV4` spec, /// and sorted. See [`build_canonical_query_string_normalized`]. /// /// # Errors @@ -484,8 +486,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Attacker's rewrite: the same values attached to the opposite raw // names. The normalized canonical query (and signature) is unchanged. @@ -571,9 +573,10 @@ mod tests { &hex::encode(Sha256::digest(canonical.as_bytes())), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(), &to_sign, - ); + ) + .unwrap(); for wire_names in [ "list-type=1&list-type=2", "%6cist-type=1&list-type=2", @@ -641,7 +644,8 @@ mod tests { // Verify the signature computation is correct by manually // replicating the presigned URL flow. - let signing_key = derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(); // Build canonical request for the presigned URL test vector. let canonical_request = "GET\n/test.txt\nX-Amz-Algorithm=AWS4-HMAC-SHA256&\ @@ -662,7 +666,7 @@ mod tests { &canonical_hash, ); - let signature = compute_signature(&signing_key, &string_to_sign); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); assert_eq!( signature, "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404" @@ -698,8 +702,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Build the full query with signature. let full_query = format!("{query_without_sig}&X-Amz-Signature={signature}"); @@ -755,8 +759,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); let full_query = format!("{query_without_sig}&X-Amz-Signature={signature}"); let uri = format!("http://examplebucket.s3.amazonaws.com/test.txt?{full_query}"); @@ -805,8 +809,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Tamper: `%2B` becomes a raw `+` on the wire. let tampered_query = query_without_sig.replace("x=%2B", "x=+"); @@ -859,8 +863,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Tamper: `%20` becomes a raw `+` on the wire. let tampered_query = query_without_sig.replace("x=%20", "x=+"); diff --git a/crates/rustack-auth/src/sigv2.rs b/crates/rustack-auth/src/sigv2.rs index 7ca9eb5f..26b44988 100644 --- a/crates/rustack-auth/src/sigv2.rs +++ b/crates/rustack-auth/src/sigv2.rs @@ -1,6 +1,6 @@ //! AWS Signature Version 2 verification. //! -//! SigV2 is an older signing mechanism that uses HMAC-SHA1. The `Authorization` +//! `SigV2` is an older signing mechanism that uses HMAC-SHA1. The `Authorization` //! header has the format: //! //! ```text @@ -30,18 +30,18 @@ use crate::{credentials::CredentialProvider, error::AuthError, sigv4::AuthResult type HmacSha1 = Hmac; -/// Check whether the `Authorization` header uses SigV2 format (`AWS AKID:sig`). +/// Check whether the `Authorization` header uses `SigV2` format (`AWS AKID:sig`). #[must_use] pub fn is_sigv2(auth_header: &str) -> bool { auth_header.starts_with("AWS ") && !auth_header.starts_with("AWS4-") } -/// Verify an AWS SigV2-signed HTTP request. +/// Verify an AWS `SigV2`-signed HTTP request. /// /// # Errors /// /// Returns an [`AuthError`] if the header is malformed, the access key is not -/// found, or the signature does not match. +/// found, the MAC provider rejects initialization, or the signature does not match. pub fn verify_sigv2( parts: &http::request::Parts, credential_provider: &dyn CredentialProvider, @@ -63,7 +63,7 @@ pub fn verify_sigv2( debug!("Built SigV2 string to sign"); - let expected_signature = compute_sigv2_signature(&secret_key, &string_to_sign); + let expected_signature = compute_sigv2_signature(&secret_key, &string_to_sign)?; if provided_signature .as_bytes() @@ -83,7 +83,7 @@ pub fn verify_sigv2( } } -/// Parse a SigV2 `Authorization` header: `AWS AKID:Signature`. +/// Parse a `SigV2` `Authorization` header: `AWS AKID:Signature`. fn parse_sigv2_header(header: &str) -> Result<(String, String), AuthError> { let rest = header .strip_prefix("AWS ") @@ -98,7 +98,7 @@ fn parse_sigv2_header(header: &str) -> Result<(String, String), AuthError> { Ok((access_key_id.to_owned(), signature.to_owned())) } -/// Build the SigV2 string to sign from the request parts. +/// Build the `SigV2` string to sign from the request parts. /// /// ```text /// HTTP-Verb + "\n" + @@ -126,7 +126,7 @@ fn build_string_to_sign(parts: &http::request::Parts) -> String { format!("{method}\n{content_md5}\n{content_type}\n{date}\n{amz_headers}{resource}") } -/// Build the CanonicalizedAmzHeaders string. +/// Build the `CanonicalizedAmzHeaders` string. /// /// All x-amz-* headers are lowercased, sorted, and joined with newlines. /// Each header is formatted as `name:value\n`. @@ -155,7 +155,7 @@ fn build_canonicalized_amz_headers(parts: &http::request::Parts) -> String { result } -/// Build the CanonicalizedResource string. +/// Build the `CanonicalizedResource` string. /// /// This is the URI path plus any sub-resource query parameters (sorted). fn build_canonicalized_resource(parts: &http::request::Parts) -> String { @@ -228,13 +228,13 @@ fn build_canonicalized_resource(parts: &http::request::Parts) -> String { } } -/// Compute the SigV2 signature: Base64(HMAC-SHA1(secret, string_to_sign)). -fn compute_sigv2_signature(secret_key: &str, string_to_sign: &str) -> String { - let mut mac = - HmacSha1::new_from_slice(secret_key.as_bytes()).expect("HMAC can accept any key length"); +/// Compute the `SigV2` signature: Base64(HMAC-SHA1(secret, `string_to_sign`)). +fn compute_sigv2_signature(secret_key: &str, string_to_sign: &str) -> Result { + let mut mac = HmacSha1::new_from_slice(secret_key.as_bytes()) + .map_err(|source| AuthError::CryptoInitialization { source })?; mac.update(string_to_sign.as_bytes()); let result = mac.finalize().into_bytes(); - BASE64.encode(result) + Ok(BASE64.encode(result)) } /// Extract a header value as a string, returning empty string if missing. @@ -284,12 +284,27 @@ mod tests { assert!(parse_sigv2_header("NOTAWS key:sig").is_err()); } + #[test] + fn test_should_match_rfc2202_sha1_vectors() { + // https://www.rfc-editor.org/rfc/rfc2202.html section 3, cases 1 and 2. + let first = compute_sigv2_signature(&"\u{b}".repeat(20), "Hi There").unwrap(); + assert_eq!( + hex::encode(BASE64.decode(first).unwrap()), + "b617318655057264e28bc0b6fb378c8ef146be00" + ); + let second = compute_sigv2_signature("Jefe", "what do ya want for nothing?").unwrap(); + assert_eq!( + hex::encode(BASE64.decode(second).unwrap()), + "effcdf6ae5eb2fa2d27416d5f184df9c259a7c79" + ); + } + #[test] fn test_should_compute_sigv2_signature() { - let sig = compute_sigv2_signature("secret", "data"); + let sig = compute_sigv2_signature("secret", "data").unwrap(); assert_ne!(sig, ""); // Base64(HMAC-SHA1) produces a deterministic output. - let sig2 = compute_sigv2_signature("secret", "data"); + let sig2 = compute_sigv2_signature("secret", "data").unwrap(); assert_eq!(sig, sig2); } @@ -300,7 +315,7 @@ mod tests { // Build a simple GET request. let date = "Sat, 28 Feb 2026 12:00:00 GMT"; let string_to_sign = format!("GET\n\n\n{date}\n/test-bucket/"); - let signature = compute_sigv2_signature(TEST_SECRET_KEY, &string_to_sign); + let signature = compute_sigv2_signature(TEST_SECRET_KEY, &string_to_sign).unwrap(); let auth_header = format!("AWS {TEST_ACCESS_KEY}:{signature}"); diff --git a/crates/rustack-auth/src/sigv4.rs b/crates/rustack-auth/src/sigv4.rs index 1adb156c..ced54d58 100644 --- a/crates/rustack-auth/src/sigv4.rs +++ b/crates/rustack-auth/src/sigv4.rs @@ -1,6 +1,6 @@ //! AWS Signature Version 4 verification. //! -//! This module implements the core SigV4 signature verification flow: +//! This module implements the core `SigV4` signature verification flow: //! //! 1. Parse the `Authorization` header to extract the algorithm, credential scope, signed headers, //! and provided signature. @@ -32,7 +32,7 @@ const SUPPORTED_ALGORITHM: &str = "AWS4-HMAC-SHA256"; type HmacSha256 = Hmac; -/// The result of a successful SigV4 verification. +/// The result of a successful `SigV4` verification. #[derive(Debug, Clone)] pub struct AuthResult { /// The access key ID that signed the request. @@ -45,7 +45,7 @@ pub struct AuthResult { pub signed_headers: Vec, } -/// Parsed components of an AWS SigV4 `Authorization` header. +/// Parsed components of an AWS `SigV4` `Authorization` header. /// /// Format: /// ```text @@ -71,7 +71,7 @@ pub struct ParsedAuth { pub signature: String, } -/// Parse an AWS SigV4 `Authorization` header value into its components. +/// Parse an AWS `SigV4` `Authorization` header value into its components. /// /// # Errors /// @@ -108,25 +108,25 @@ pub fn parse_authorization_header(header: &str) -> Result // Parse credential: AKID/date/region/service/aws4_request let cred_parts: Vec<&str> = credential.splitn(5, '/').collect(); - if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { + let [access_key_id, date, region, service, "aws4_request"] = cred_parts.as_slice() else { return Err(AuthError::InvalidCredential); - } + }; let parsed_signed_headers: Vec = signed_headers.split(';').map(ToOwned::to_owned).collect(); Ok(ParsedAuth { algorithm: algorithm.to_owned(), - access_key_id: cred_parts[0].to_owned(), - date: cred_parts[1].to_owned(), - region: cred_parts[2].to_owned(), - service: cred_parts[3].to_owned(), + access_key_id: (*access_key_id).to_owned(), + date: (*date).to_owned(), + region: (*region).to_owned(), + service: (*service).to_owned(), signed_headers: parsed_signed_headers, signature: signature.to_owned(), }) } -/// Build the SigV4 string to sign. +/// Build the `SigV4` string to sign. /// /// Format: /// ```text @@ -157,7 +157,7 @@ pub fn build_string_to_sign( format!("{SUPPORTED_ALGORITHM}\n{timestamp}\n{credential_scope}\n{canonical_request_hash}") } -/// Derive the SigV4 signing key using HMAC-SHA256 chain. +/// Derive the `SigV4` signing key using HMAC-SHA256 chain. /// /// ```text /// DateKey = HMAC-SHA256("AWS4" + secret_key, date) @@ -170,33 +170,51 @@ pub fn build_string_to_sign( /// /// ``` /// use rustack_auth::sigv4::derive_signing_key; +/// # fn example() -> Result<(), rustack_auth::AuthError> { /// /// let key = derive_signing_key( /// "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", /// "20130524", /// "us-east-1", /// "s3", -/// ); +/// )?; /// assert!(!key.is_empty()); +/// # Ok(()) } /// ``` -#[must_use] -pub fn derive_signing_key(secret_key: &str, date: &str, region: &str, service: &str) -> Vec { - let date_key = hmac_sha256(format!("AWS4{secret_key}").as_bytes(), date.as_bytes()); - let date_region_key = hmac_sha256(&date_key, region.as_bytes()); - let date_region_service_key = hmac_sha256(&date_region_key, service.as_bytes()); +/// +/// # Errors +/// Returns [`AuthError::CryptoInitialization`] if the MAC provider rejects a key. +pub fn derive_signing_key( + secret_key: &str, + date: &str, + region: &str, + service: &str, +) -> Result, AuthError> { + let date_key = hmac_sha256(format!("AWS4{secret_key}").as_bytes(), date.as_bytes())?; + let date_region_key = hmac_sha256(&date_key, region.as_bytes())?; + let date_region_service_key = hmac_sha256(&date_region_key, service.as_bytes())?; hmac_sha256(&date_region_service_key, b"aws4_request") } /// Compute the HMAC-SHA256 signature of `data` using the given `signing_key`. /// /// Returns the hex-encoded signature. -#[must_use] -pub fn compute_signature(signing_key: &[u8], data: &str) -> String { - let sig = hmac_sha256(signing_key, data.as_bytes()); - hex::encode(sig) +/// +/// # Errors +/// Returns [`AuthError::CryptoInitialization`] if the MAC provider rejects the key. +/// +/// ``` +/// # fn example() -> Result<(), rustack_auth::AuthError> { +/// let signature = rustack_auth::sigv4::compute_signature(b"secret", "message")?; +/// assert_eq!(signature.len(), 64); +/// # Ok(()) } +/// ``` +pub fn compute_signature(signing_key: &[u8], data: &str) -> Result { + let sig = hmac_sha256(signing_key, data.as_bytes())?; + Ok(hex::encode(sig)) } -/// Verify an AWS SigV4-signed HTTP request. +/// Verify an AWS `SigV4`-signed HTTP request. /// /// This function: /// 1. Parses the `Authorization` header @@ -207,7 +225,7 @@ pub fn compute_signature(signing_key: &[u8], data: &str) -> String { /// /// The query string is verified exactly as it appears on the wire /// (raw-preserving). Unlike [`verify_s3_sigv4`], this does NOT attempt the -/// SigV4-normalized fallback: the normalized form asserts that alternate +/// `SigV4`-normalized fallback: the normalized form asserts that alternate /// wire encodings are equivalent, which is only true when the serving /// endpoint percent-decodes query parameters before acting on them. Whether /// the fallback is safe is a property of the serving endpoint, never of the @@ -222,6 +240,7 @@ pub fn compute_signature(signing_key: &[u8], data: &str) -> String { /// - The access key is not found /// - Required signed headers are missing /// - The signature does not match +/// - The MAC provider rejects initialization pub fn verify_sigv4( parts: &http::request::Parts, body_hash: &str, @@ -230,13 +249,13 @@ pub fn verify_sigv4( verify_sigv4_with_policy(parts, body_hash, credential_provider, false, false) } -/// Verify S3 SigV4, allowing the explicit unsigned-payload protocol exception. +/// Verify S3 `SigV4`, allowing the explicit unsigned-payload protocol exception. /// /// This verifies the seed signature only for streaming markers. Callers MUST additionally /// verify every chunk and signed trailer with [`StreamingVerifier`] before publishing data. /// /// In addition to the raw-preserving attempt, this accepts the -/// SigV4-normalized canonical query string as a fallback (see +/// `SigV4`-normalized canonical query string as a fallback (see /// [`build_canonical_query_string_normalized`]): spec-compliant S3 clients /// such as Transmit 5 sign the encoded form even when the wire carries the /// raw value. This endpoint entry point is the opt-in — S3 percent-decodes @@ -244,12 +263,13 @@ pub fn verify_sigv4( /// (`rustack-s3-http/src/router.rs::parse_query_params`), so the normalized /// equivalence holds. Do NOT use this entry point for an endpoint that reads /// query parameters from the raw query string without decoding (e.g. -/// CloudFront's `DeleteRealtimeLogConfig` reading `Name`): a verified +/// `CloudFront`'s `DeleteRealtimeLogConfig` reading `Name`): a verified /// signature could then authorize a different effective value than the one /// acted upon. Such endpoints must use [`verify_sigv4`]. /// /// # Errors -/// Returns an authentication error for invalid signatures or payload declarations. +/// Returns an authentication error for invalid signatures, payload declarations, +/// credential/header failures or MAC provider initialization failures. pub fn verify_s3_sigv4( parts: &http::request::Parts, body_hash: &str, @@ -324,26 +344,26 @@ fn verify_sigv4_with_policy( parsed.date, parsed.region, parsed.service ); let signing_key = - derive_signing_key(&secret_key, &parsed.date, &parsed.region, &parsed.service); + derive_signing_key(&secret_key, &parsed.date, &parsed.region, &parsed.service)?; let provided_bytes = parsed.signature.as_bytes(); // Check the provided signature against the canonical request built from // `query_string`. Returns true on a constant-time match. #[rustfmt::skip] - let signature_matches = |query_string: &str| -> bool { + let signature_matches = |query_string: &str| -> Result { let canonical_request = format!( "{method}\n{canonical_uri}\n{query_string}\n{canonical_headers}\n\n{signed_headers_str}\n{payload_hash}" ); let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let expected_signature = compute_signature(&signing_key, &string_to_sign); - bool::from(provided_bytes.ct_eq(expected_signature.as_bytes())) + let expected_signature = compute_signature(&signing_key, &string_to_sign)?; + Ok(bool::from(provided_bytes.ct_eq(expected_signature.as_bytes()))) }; // First attempt: raw query string values preserved as-is. This matches // clients that sign whatever encoding appears on the wire (e.g. AWS SDKs // with pre-encoded values, minio-java via OkHttp with raw values). - if signature_matches(&canonical_query) { + if signature_matches(&canonical_query)? { debug!(access_key_id = %parsed.access_key_id, "Signature verification succeeded"); return Ok(AuthResult { access_key_id: parsed.access_key_id, @@ -377,9 +397,10 @@ fn verify_sigv4_with_policy( // the signature mismatch below. let normalized_matches = allow_normalized_fallback && !query.contains('+') - && build_canonical_query_string_normalized(query).is_ok_and(|normalized| { - normalized != canonical_query && signature_matches(&normalized) - }); + && match build_canonical_query_string_normalized(query) { + Ok(normalized) if normalized != canonical_query => signature_matches(&normalized)?, + _ => false, + }; if normalized_matches { debug!(access_key_id = %parsed.access_key_id, "Signature verification succeeded (normalized query string)"); return Ok(AuthResult { @@ -394,7 +415,7 @@ fn verify_sigv4_with_policy( Err(AuthError::SignatureDoesNotMatch) } -/// SigV4 streaming HMAC chain, including terminal chunks and signed trailers. +/// `SigV4` streaming HMAC chain, including terminal chunks and signed trailers. pub struct StreamingVerifier { key: Vec, timestamp: String, @@ -411,7 +432,7 @@ impl std::fmt::Debug for StreamingVerifier { impl StreamingVerifier { /// Authenticate the seed and initialize the chunk chain. /// # Errors - /// Returns seed signature or header validation failures. + /// Returns seed signature/header validation or MAC provider initialization failures. pub fn new( parts: &http::request::Parts, provider: &dyn CredentialProvider, @@ -421,7 +442,7 @@ impl StreamingVerifier { let parsed = parse_authorization_header(&header)?; let secret = provider.get_secret_key(&parsed.access_key_id)?; Ok(Self { - key: derive_signing_key(&secret, &parsed.date, &parsed.region, &parsed.service), + key: derive_signing_key(&secret, &parsed.date, &parsed.region, &parsed.service)?, timestamp: extract_header_value(parts, "x-amz-date")?, scope: format!( "{}/{}/{}/aws4_request", @@ -433,7 +454,8 @@ impl StreamingVerifier { /// Verify a data chunk, including the final zero-size chunk. /// # Errors - /// Rejects a missing, malformed or mismatching chunk signature. + /// Rejects a missing, malformed or mismatching chunk signature, or MAC provider + /// initialization failure. Failure leaves the previous signature unchanged. pub fn verify_chunk(&mut self, actual_sha256: &str, signature: &str) -> Result<(), AuthError> { let text = format!( "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", @@ -448,7 +470,8 @@ impl StreamingVerifier { /// Verify the canonical declared trailer block after the terminal chunk. /// # Errors - /// Returns an error if the trailer signature does not match. + /// Returns an error if the trailer signature does not match or MAC provider + /// initialization fails. Failure leaves the previous signature unchanged. pub fn verify_trailer(&mut self, canonical: &str, signature: &str) -> Result<(), AuthError> { let text = format!( "AWS4-HMAC-SHA256-TRAILER\n{}\n{}\n{}\n{}", @@ -461,7 +484,7 @@ impl StreamingVerifier { } fn verify_next(&mut self, text: &str, signature: &str) -> Result<(), AuthError> { - let expected = compute_signature(&self.key, text); + let expected = compute_signature(&self.key, text)?; if !bool::from(expected.as_bytes().ct_eq(signature.as_bytes())) { return Err(AuthError::SignatureDoesNotMatch); } @@ -537,10 +560,11 @@ fn collect_signed_headers<'a>( } /// Compute HMAC-SHA256 and return the raw bytes. -fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { - let mut mac = HmacSha256::new_from_slice(key).expect("HMAC can accept keys of any length"); +fn hmac_sha256(key: &[u8], data: &[u8]) -> Result, AuthError> { + let mut mac = HmacSha256::new_from_slice(key) + .map_err(|source| AuthError::CryptoInitialization { source })?; mac.update(data); - mac.finalize().into_bytes().to_vec() + Ok(mac.finalize().into_bytes().to_vec()) } /// Compute the SHA-256 hash of the given payload and return it as a hex string. @@ -585,10 +609,26 @@ mod tests { } #[test] - fn test_should_derive_signing_key_matching_aws_test_vector() { - let key = derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE); - // The signing key itself is not published as hex in the AWS docs, - // but we can verify it produces the correct signature when used. + fn test_should_match_rfc4231_sha256_vectors_with_short_and_long_keys() { + // https://www.rfc-editor.org/rfc/rfc4231.html sections 4.2 and 4.7. + assert_eq!( + compute_signature(&[0x0b; 20], "Hi There").unwrap(), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7" + ); + assert_eq!( + compute_signature( + &[0xaa; 131], + "Test Using Larger Than Block-Size Key - Hash Key First" + ) + .unwrap(), + "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54" + ); + } + + #[test] + fn test_should_derive_a_32_byte_signing_key() { + let key = + derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE).unwrap(); assert_eq!(key.len(), 32); // SHA-256 produces 32 bytes } @@ -651,7 +691,8 @@ mod tests { #[test] fn test_should_compute_correct_signature_for_aws_get_object_example() { // Full end-to-end test using the AWS GET Object example. - let signing_key = derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE).unwrap(); #[rustfmt::skip] let string_to_sign = "AWS4-HMAC-SHA256\n\ @@ -659,7 +700,7 @@ mod tests { 20130524/us-east-1/s3/aws4_request\n\ 7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"; - let signature = compute_signature(&signing_key, string_to_sign); + let signature = compute_signature(&signing_key, string_to_sign).unwrap(); assert_eq!( signature, "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" @@ -684,9 +725,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &text, - ); + ) + .unwrap(); parts.headers.insert( "authorization", format!( @@ -766,7 +808,7 @@ mod tests { hash_payload(b""), digest ); - let signature = compute_signature(&verifier.key, &text); + let signature = compute_signature(&verifier.key, &text).unwrap(); assert!( verifier .verify_chunk(&hash_payload(b"wrong"), &signature) @@ -877,9 +919,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ @@ -929,9 +972,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", service), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", service).unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ Credential={TEST_ACCESS_KEY}/20130524/us-east-1/{service}/aws4_request,\ @@ -1067,9 +1111,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ diff --git a/crates/rustack-core/Cargo.toml b/crates/rustack-core/Cargo.toml index 9474e0cb..32890ce2 100644 --- a/crates/rustack-core/Cargo.toml +++ b/crates/rustack-core/Cargo.toml @@ -29,4 +29,6 @@ tracing = { workspace = true } uuid = { workspace = true } [dev-dependencies] -tokio = { workspace = true } +proptest = { workspace = true } +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-core/src/config.rs b/crates/rustack-core/src/config.rs index 30d7acd8..964446a2 100644 --- a/crates/rustack-core/src/config.rs +++ b/crates/rustack-core/src/config.rs @@ -1,6 +1,6 @@ //! Configuration management for Rustack services. //! -//! All configuration is driven by environment variables, matching LocalStack conventions. +//! All configuration is driven by environment variables, matching `LocalStack` conventions. use crate::types::AwsRegion; diff --git a/crates/rustack-core/src/http.rs b/crates/rustack-core/src/http.rs index 6debd1f0..4223bd84 100644 --- a/crates/rustack-core/src/http.rs +++ b/crates/rustack-core/src/http.rs @@ -320,7 +320,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_should_expire_pending_body_without_eof() { let budget = BodyBudget::new(3, Duration::from_secs(1), Duration::from_millis(5)).unwrap(); assert!(matches!( @@ -372,7 +372,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_should_not_reset_total_deadline_on_progress() { let budget = BodyBudget::new(1000, Duration::from_millis(20), Duration::from_millis(10)).unwrap(); diff --git a/crates/rustack-core/src/json.rs b/crates/rustack-core/src/json.rs new file mode 100644 index 00000000..5791623b --- /dev/null +++ b/crates/rustack-core/src/json.rs @@ -0,0 +1,307 @@ +//! Bounded JSON preflight before typed deserialization. +//! +//! Encoded strings, decoded keys/values, aggregate members and container depth +//! are checked without building a dynamic Value tree. Errors redact input. + +use std::{collections::BTreeSet, fmt, num::NonZeroUsize}; + +use serde::{ + Deserializer, + de::{DeserializeOwned, DeserializeSeed, Error, MapAccess, SeqAccess, Visitor}, +}; + +/// Redacted JSON validation and target-schema failure classes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum JsonError { + /// Invalid parser limits. + #[error("invalid JSON limits")] + InvalidLimits, + /// Input exceeds its byte allowance. + #[error("JSON byte limit exceeded")] + Bytes, + /// Nested containers exceed their allowance. + #[error("JSON depth limit exceeded")] + Depth, + /// Aggregate object pairs/array elements exceed their allowance. + #[error("JSON member limit exceeded")] + Members, + /// An encoded/decoded scalar string exceeds its allowance. + #[error("JSON string limit exceeded")] + String, + /// A decoded object key exceeds its separate allowance. + #[error("JSON key limit exceeded")] + Key, + /// Two decoded keys in one object are equal. + #[error("duplicate JSON key")] + Duplicate, + /// Invalid JSON, including non-finite numbers. + #[error("invalid JSON syntax")] + Syntax, + /// Bounded JSON does not match the requested domain type. + #[error("invalid JSON schema")] + Schema, +} + +/// Validated byte, container-depth, member and string/key limits. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct JsonLimits { + bytes: NonZeroUsize, + depth: NonZeroUsize, + members: NonZeroUsize, + string_bytes: NonZeroUsize, + key_bytes: NonZeroUsize, +} + +impl JsonLimits { + /// Validate limits before accepting untrusted bytes. + /// + /// # Errors + /// Rejects zero limits, bytes over 16 MiB, depth over 64, members over 65,536, + /// strings over 1 MiB. Strings cannot exceed the body limit; keys cannot + /// exceed strings. Consumers choose their own tighter key allowance. + /// + /// ``` + /// let limits = rustack_core::json::JsonLimits::new(1024, 8, 64, 256, 64)?; + /// # Ok::<(), rustack_core::json::JsonError>(()) + /// ``` + pub fn new( + bytes: usize, + depth: usize, + members: usize, + string_bytes: usize, + key_bytes: usize, + ) -> Result { + if bytes > 16 * 1024 * 1024 + || depth > 64 + || members > 65_536 + || string_bytes > 1024 * 1024 + || string_bytes > bytes + || key_bytes > string_bytes + { + return Err(JsonError::InvalidLimits); + } + let nonzero = |value| NonZeroUsize::new(value).ok_or(JsonError::InvalidLimits); + Ok(Self { + bytes: nonzero(bytes)?, + depth: nonzero(depth)?, + members: nonzero(members)?, + string_bytes: nonzero(string_bytes)?, + key_bytes: nonzero(key_bytes)?, + }) + } +} + +/// Preflight bounded structure and decoded-key uniqueness without a value tree. +/// +/// Depth counts containers: an object/array of scalars has depth one. Members +/// count all object pairs and array elements across the entire input. +/// +/// # Errors +/// Returns a redacted syntax, duplicate or resource-limit class. +/// +/// ``` +/// use rustack_core::json::{JsonLimits, validate}; +/// validate(br#"{"items":[1,true,null]}"#, JsonLimits::new(1024, 8, 64, 256, 64)?)?; +/// # Ok::<(), rustack_core::json::JsonError>(()) +/// ``` +pub fn validate(input: &[u8], limits: JsonLimits) -> Result<(), JsonError> { + if input.len() > limits.bytes.get() { + return Err(JsonError::Bytes); + } + encoded_strings(input, limits.string_bytes.get())?; + let mut context = Scan { + limits, + remaining: limits.members.get(), + failure: None, + }; + let mut decoder = serde_json::Deserializer::from_slice(input); + let result = Node { + context: &mut context, + depth: 0, + } + .deserialize(&mut decoder); + if result.is_err() { + return Err(context.failure.unwrap_or(JsonError::Syntax)); + } + decoder.end().map_err(|_| JsonError::Syntax) +} + +/// Preflight JSON before materializing a typed model; domain validation follows. +/// +/// # Errors +/// Returns preflight classes or Schema without a deserializer's input text. +/// +/// ``` +/// use std::collections::BTreeMap; +/// use rustack_core::json::{JsonLimits, from_slice}; +/// let model: BTreeMap = from_slice(br#"{"count":7}"#, +/// JsonLimits::new(1024, 8, 64, 256, 64)?)?; +/// assert_eq!(model.get("count"), Some(&7)); +/// # Ok::<(), rustack_core::json::JsonError>(()) +/// ``` +pub fn from_slice(input: &[u8], limits: JsonLimits) -> Result { + validate(input, limits)?; + serde_json::from_slice(input).map_err(|_| JsonError::Schema) +} + +fn encoded_strings(input: &[u8], decoded_limit: usize) -> Result<(), JsonError> { + // A JSON escape uses at most six encoded bytes per decoded UTF-8 byte. + // Cap the parser's scratch space before processing escaped strings. + let maximum = decoded_limit + .checked_mul(6) + .ok_or(JsonError::InvalidLimits)?; + let mut inside = false; + let mut escaped = false; + let mut length = 0_usize; + for byte in input { + if !inside { + if *byte == b'"' { + inside = true; + length = 0; + } + } else if !escaped && *byte == b'"' { + inside = false; + } else { + length = length.checked_add(1).ok_or(JsonError::String)?; + if length > maximum { + return Err(JsonError::String); + } + escaped = !escaped && *byte == b'\\'; + } + } + Ok(()) +} + +#[derive(Debug)] +struct Scan { + limits: JsonLimits, + remaining: usize, + failure: Option, +} + +impl Scan { + fn reject(&mut self, error: JsonError) -> E { + self.failure.get_or_insert(error); + E::custom("invalid bounded JSON") + } + fn member(&mut self) -> Result<(), E> { + match self.remaining.checked_sub(1) { + Some(remaining) => { + self.remaining = remaining; + Ok(()) + } + None => Err(self.reject(JsonError::Members)), + } + } +} + +#[derive(Debug)] +struct Node<'a> { + context: &'a mut Scan, + depth: usize, +} + +impl<'de> DeserializeSeed<'de> for Node<'_> { + type Value = (); + fn deserialize>(self, decoder: D) -> Result<(), D::Error> { + decoder.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Node<'_> { + type Value = (); + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("bounded JSON") + } + fn visit_bool(self, _: bool) -> Result<(), E> { + Ok(()) + } + fn visit_i64(self, _: i64) -> Result<(), E> { + Ok(()) + } + fn visit_u64(self, _: u64) -> Result<(), E> { + Ok(()) + } + fn visit_f64(self, value: f64) -> Result<(), E> { + if value.is_finite() { + Ok(()) + } else { + Err(self.context.reject(JsonError::Syntax)) + } + } + fn visit_unit(self) -> Result<(), E> { + Ok(()) + } + fn visit_str(self, value: &str) -> Result<(), E> { + if value.len() > self.context.limits.string_bytes.get() { + return Err(self.context.reject(JsonError::String)); + } + Ok(()) + } + fn visit_seq>(self, mut array: A) -> Result<(), A::Error> { + if self.depth >= self.context.limits.depth.get() { + return Err(self.context.reject(JsonError::Depth)); + } + let depth = self + .depth + .checked_add(1) + .ok_or_else(|| self.context.reject(JsonError::Depth))?; + while array + .next_element_seed(Element { + context: self.context, + depth, + })? + .is_some() + {} + Ok(()) + } + fn visit_map>(self, mut object: A) -> Result<(), A::Error> { + if self.depth >= self.context.limits.depth.get() { + return Err(self.context.reject(JsonError::Depth)); + } + let depth = self + .depth + .checked_add(1) + .ok_or_else(|| self.context.reject(JsonError::Depth))?; + let mut keys = BTreeSet::new(); + while let Some(key) = object.next_key::()? { + self.context.member()?; + if key.len() > self.context.limits.key_bytes.get() { + return Err(self.context.reject(JsonError::Key)); + } + if !keys.insert(key) { + return Err(self.context.reject(JsonError::Duplicate)); + } + object.next_value_seed(Node { + context: self.context, + depth, + })?; + } + Ok(()) + } +} + +// Charge only actual elements, so an empty array remains valid after an +// enclosing object consumes the last available member. +#[derive(Debug)] +struct Element<'a> { + context: &'a mut Scan, + depth: usize, +} + +impl<'de> DeserializeSeed<'de> for Element<'_> { + type Value = (); + fn deserialize>(self, decoder: D) -> Result<(), D::Error> { + self.context.member()?; + Node { + context: self.context, + depth: self.depth, + } + .deserialize(decoder) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-core/src/json/tests.rs b/crates/rustack-core/src/json/tests.rs new file mode 100644 index 00000000..ccaacfb7 --- /dev/null +++ b/crates/rustack-core/src/json/tests.rs @@ -0,0 +1,195 @@ +use std::collections::BTreeMap; + +use proptest::{ + arbitrary::any, + collection::{btree_map, vec}, + prelude::{ProptestConfig, Strategy}, + prop_assert, prop_assert_eq, prop_oneof, proptest, +}; +use rstest::rstest; +use serde::Deserialize; +use serde_json::{Value, json}; + +use super::*; + +fn limits() -> JsonLimits { + JsonLimits::new(16 * 1024, 16, 1024, 256, 32).unwrap() +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct Model { + count: u32, +} + +#[test] +fn test_should_validate_before_typed_schema_and_redact_schema_errors() { + assert_eq!( + from_slice::(br#"{"count":7}"#, limits()).unwrap(), + Model { count: 7 } + ); + let error = from_slice::(br#"{"count":"secret-schema-value"}"#, limits()).unwrap_err(); + assert_eq!(error, JsonError::Schema); + assert!(!format!("{error:?} {error}").contains("secret-schema-value")); + assert_eq!( + from_slice::(br#"{"count":7,"unknown":1,"unknown":2}"#, limits()), + Err(JsonError::Duplicate) + ); +} + +#[rstest] +#[case(br#"{"a":1,"a":2}"#, JsonError::Duplicate)] +#[case(br#"{"a":1,"\u0061":2}"#, JsonError::Duplicate)] +#[case(br#"{"x":{"a":1,"a":2}}"#, JsonError::Duplicate)] +#[case(b"{", JsonError::Syntax)] +#[case(b"[1,]", JsonError::Syntax)] +#[case(b"null true", JsonError::Syntax)] +#[case(b"", JsonError::Syntax)] +#[case(b"01", JsonError::Syntax)] +#[case(b"1e99999", JsonError::Syntax)] +#[case(br#""\ud800""#, JsonError::Syntax)] +#[case(br#""\x00""#, JsonError::Syntax)] +#[case(&[b'"', 0xff, b'"'], JsonError::Syntax)] +fn test_should_reject_duplicates_and_malformed_syntax( + #[case] input: &[u8], + #[case] expected: JsonError, +) { + assert_eq!(validate(input, limits()), Err(expected)); +} + +#[test] +fn test_should_scope_duplicate_keys_to_their_own_object() { + validate(br#"{"a":1,"child":{"a":2}}"#, limits()).unwrap(); + validate(br#"[{"a":1},{"a":2}]"#, limits()).unwrap(); + validate(br#"{"\u00e9":1,"e\u0301":2}"#, limits()).unwrap(); +} + +#[test] +fn test_should_bound_container_depth_without_rejecting_scalar_children() { + let shallow = JsonLimits::new(1024, 1, 64, 256, 32).unwrap(); + for input in [ + b"{}".as_slice(), + b"[]", + b"[1,true,null]", + br#"{"a":1}"#, + b"false", + ] { + validate(input, shallow).unwrap(); + } + for input in [br#"{"a":{}}"#.as_slice(), b"[[]]", br#"[{"a":1}]"#] { + assert_eq!(validate(input, shallow), Err(JsonError::Depth)); + } + let deeply_nested = format!("{}0{}", "[".repeat(4000), "]".repeat(4000)); + let bounded = JsonLimits::new(16 * 1024, 64, 8192, 256, 32).unwrap(); + assert_eq!( + validate(deeply_nested.as_bytes(), bounded), + Err(JsonError::Depth) + ); +} + +#[test] +fn test_should_count_all_object_pairs_and_array_elements_in_one_budget() { + let three = JsonLimits::new(1024, 8, 3, 256, 32).unwrap(); + validate(br#"{"a":[1,2]}"#, three).unwrap(); + assert_eq!( + validate(br#"{"a":[1,2,3]}"#, three), + Err(JsonError::Members) + ); + validate(br#"{"a":{"b":1},"c":2}"#, three).unwrap(); + assert_eq!( + validate(br#"{"a":{"b":1},"c":2,"d":3}"#, three), + Err(JsonError::Members) + ); + let one = JsonLimits::new(1024, 8, 1, 256, 32).unwrap(); + validate(br#"{"a":[]}"#, one).unwrap(); + validate(br#"{"a":{}}"#, one).unwrap(); + validate(b"[]", one).unwrap(); + assert_eq!(validate(br#"{"a":[1]}"#, one), Err(JsonError::Members)); +} + +#[test] +fn test_should_bound_strings_and_keys_by_decoded_utf8_bytes_and_encoded_work() { + let small = JsonLimits::new(1024, 8, 64, 2, 1).unwrap(); + validate("\"é\"".as_bytes(), small).unwrap(); + validate(br#""\u00e9""#, small).unwrap(); + assert_eq!(validate("\"界\"".as_bytes(), small), Err(JsonError::String)); + assert_eq!(validate(br#""\u754c""#, small), Err(JsonError::String)); + assert_eq!(validate("{\"é\":1}".as_bytes(), small), Err(JsonError::Key)); + assert_eq!(validate(br#"{"\u00e9":1}"#, small), Err(JsonError::Key)); + let many_escapes = format!("\"{}\"", "\\u0061".repeat(100)); + assert_eq!( + validate(many_escapes.as_bytes(), small), + Err(JsonError::String) + ); + validate(br#""\\""#, small).unwrap(); + validate(br#""\"""#, small).unwrap(); + let raised_key_limit = JsonLimits::new(1024, 1, 1, 512, 512).unwrap(); + let long_key = format!("{{\"{}\":0}}", "k".repeat(300)); + validate(long_key.as_bytes(), raised_key_limit).unwrap(); + assert_eq!(validate(long_key.as_bytes(), limits()), Err(JsonError::Key)); +} + +#[test] +fn test_should_enforce_input_bytes_before_lexical_processing() { + let tiny = JsonLimits::new(4, 1, 1, 4, 4).unwrap(); + validate(b"null", tiny).unwrap(); + assert_eq!(validate(b"null ", tiny), Err(JsonError::Bytes)); + assert_eq!(validate(&[0xff; 5], tiny), Err(JsonError::Bytes)); +} + +#[test] +fn test_should_reject_invalid_limit_relationships_and_ranges() { + for args in [ + (0, 1, 1, 1, 1), + (1, 0, 1, 1, 1), + (1, 1, 0, 1, 1), + (1, 1, 1, 0, 1), + (1, 1, 1, 1, 0), + (16 * 1024 * 1024 + 1, 1, 1, 1, 1), + (1024, 65, 1, 1, 1), + (1024, 1, 65_537, 1, 1), + (2 * 1024 * 1024, 1, 1, 1024 * 1024 + 1, 1), + (1024, 1, 1, 256, 257), + (1024, 1, 1, 1, 2), + (1, 1, 1, 2, 1), + ] { + assert!(matches!( + JsonLimits::new(args.0, args.1, args.2, args.3, args.4), + Err(JsonError::InvalidLimits) + )); + } +} + +fn values() -> impl Strategy { + prop_oneof![ + any::().prop_map(Value::Bool), + any::().prop_map(|value| json!(value)), + "[a-zA-Z0-9é界\\\"\n]{0,16}".prop_map(Value::String), + proptest::strategy::Just(Value::Null), + ] + .prop_recursive(3, 64, 4, |inner| { + prop_oneof![ + vec(inner.clone(), 0..4).prop_map(Value::Array), + btree_map("[a-z]{1,8}", inner, 0..4) + .prop_map(|map: BTreeMap| Value::Object(map.into_iter().collect())), + ] + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn test_should_never_panic_on_bounded_malformed_json(input in vec(any::(), 0..4096)) { + let result = validate(&input, JsonLimits::new(4096, 8, 64, 64, 32).unwrap()); + if result.is_ok() { prop_assert!(serde_json::from_slice::(&input).is_ok()); } + } + + #[test] + fn test_should_preserve_bounded_recursive_values(value in values()) { + let bytes = serde_json::to_vec(&value).unwrap(); + prop_assert!(bytes.len() <= 16 * 1024); + let parsed: Value = from_slice(&bytes, limits()).unwrap(); + prop_assert_eq!(parsed, value); + } +} diff --git a/crates/rustack-core/src/lib.rs b/crates/rustack-core/src/lib.rs index 9a3f3640..6c1b8d54 100644 --- a/crates/rustack-core/src/lib.rs +++ b/crates/rustack-core/src/lib.rs @@ -4,9 +4,12 @@ //! Rustack service implementations, including multi-account/multi-region //! state management, configuration, and common AWS type definitions. +#![forbid(unsafe_code)] + mod config; mod error; pub mod http; +pub mod json; pub mod settings; mod state; mod types; diff --git a/crates/rustack-core/src/settings.rs b/crates/rustack-core/src/settings.rs index d30fb025..cad81cff 100644 --- a/crates/rustack-core/src/settings.rs +++ b/crates/rustack-core/src/settings.rs @@ -22,6 +22,7 @@ const SERVICE_NAMES: &[&str] = &[ "SSM", "SNS", "LAMBDA", + "STEPFUNCTIONS", "EVENTS", "LOGS", "KMS", @@ -56,6 +57,21 @@ const KEYS: &[&str] = &[ "S3_DOMAIN", "S3_MAX_MEMORY_OBJECT_SIZE", "LAMBDA_DOCKER_ENABLED", + "LAMBDA_DOCKER_SOCKET", + "LAMBDA_DOCKER_RELAY_IMAGE", + "LAMBDA_DOCKER_NETWORK", + "LAMBDA_DOCKER_PULL_POLICY", + "LAMBDA_DOCKER_REGISTRIES", + "LAMBDA_DOCKER_REGISTRY_AUTH_FILE", + "LAMBDA_DOCKER_GATEWAY_ENDPOINT", + "LAMBDA_DOCKER_SDK_ACCESS_KEY_ID", + "LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY", + "LAMBDA_DOCKER_UID", + "LAMBDA_DOCKER_GID", + "LAMBDA_DOCKER_CPU", + "LAMBDA_DOCKER_LOG_RING_BYTES", + "LAMBDA_DOCKER_CROSS_ARCHITECTURE", + "LAMBDA_DOCKER_ZIP_IMAGE", "LAMBDA_EXECUTOR", "LAMBDA_MAX_WARM_INSTANCES", "LAMBDA_IDLE_TIMEOUT_SECS", @@ -101,6 +117,85 @@ static SETTINGS: OnceLock = OnceLock::new(); #[non_exhaustive] #[serde(default, rename_all = "camelCase", deny_unknown_fields)] pub struct RuntimeBudgets { + /// Maximum busy, idle and quarantined Docker environments. + pub lambda_docker_environments: usize, + /// Aggregate Docker environment and relay memory capacity. + pub lambda_docker_memory_bytes: u64, + /// Absolute Docker pull queue and progress deadline. + pub lambda_docker_pull_seconds: u64, + /// Accepted queued and running Docker deployments. + pub lambda_docker_deployment_jobs: usize, + /// Absolute Docker deployment preparation deadline. + pub lambda_docker_deployment_seconds: u64, + /// Retained preparing, live and quarantined private image pins. + pub lambda_docker_image_pins: usize, + /// Independent Docker management and control IO deadline. + pub lambda_docker_io_seconds: u64, + /// Code archive upload IO budget, independently bounded by cold startup. + pub lambda_docker_code_upload_seconds: u64, + /// Docker startup and probe observation deadline. + pub lambda_docker_start_seconds: u64, + /// Independent Docker cleanup observation deadline. + pub lambda_docker_cleanup_seconds: u64, + + /// Accepted workflow API requests, including ready replies. + pub workflow_api_accepted: usize, + /// Retained workflow API preparation/response CPU workers. + pub workflow_api_workers: usize, + /// Incremental workflow pagination capture bytes. + pub workflow_pagination_bytes: usize, + /// Live immutable workflow continuation handles. + pub workflow_pagination_tokens: usize, + /// Current workflow state machines. + pub workflow_registry_machines: usize, + /// Retained workflow definition and tag versions, independently. + pub workflow_registry_versions: usize, + /// Retained workflow execution records. + pub workflow_registry_records: usize, + /// Accepted workflow registry commands. + pub workflow_registry_commands: usize, + /// Workflow execution name claims retained for ninety days. + pub workflow_name_tombstones: usize, + /// Terminal workflow record retention in seconds. + pub workflow_retention_seconds: u64, + /// Live and unsettled workflow executions. + pub workflow_open_executions: usize, + /// Retained workflow runtime CPU workers. + pub workflow_evaluations: usize, + /// Concurrent workflow Task dispatches. + pub workflow_dispatches: usize, + /// Global retained frame slots, including root and ancestor leases. + pub workflow_frames: usize, + /// Retained frame slots per execution. + pub workflow_frames_per_execution: usize, + /// Global intermediate indexed result storage bytes. + pub workflow_result_bytes: usize, + /// Intermediate indexed result storage bytes per execution. + pub workflow_result_bytes_per_execution: usize, + /// Independent workflow execution duration ceiling. + pub workflow_max_duration_seconds: u64, + /// Maximum transitions per workflow execution. + pub workflow_transitions: u32, + /// Retained history bytes per workflow execution. + pub workflow_history_bytes: usize, + /// Conservative workflow snapshot copy and per-part encoded byte ceiling. + pub workflow_snapshot_bytes: usize, + /// Global retained workflow history bytes. + pub workflow_total_history_bytes: usize, + /// Accepted and quarantined workflow Lambda bridge calls. + pub workflow_lambda_accepted_calls: usize, + /// Initial workflow Lambda cleanup observation budget. + pub workflow_lambda_cleanup_seconds: u64, + /// Grace interval before cancelling remaining workflows. + pub workflow_drain_seconds: u64, + /// Live callback token attempts, including provisional completion. + pub workflow_callback_tokens: usize, + /// Retained active and closed callback classification entries. + pub workflow_callback_tombstones: usize, + /// Accepted callback commands, including ready acknowledgments. + pub workflow_callback_commands: usize, + /// Retained callback mint entropy workers. + pub workflow_callback_workers: usize, /// Maximum live HTTP connections. pub connections: usize, /// Maximum concurrently active business requests, including response bodies. @@ -113,6 +208,8 @@ pub struct RuntimeBudgets { pub lambda_invoke_seconds: u64, /// Total graceful shutdown deadline in seconds. pub shutdown_seconds: u64, + /// Absolute process startup budget, including Docker qualification and snapshots. + pub startup_seconds: u64, /// Maximum aggregate control-plane body bytes. pub control_body_bytes: u64, /// Maximum Lambda ZIP JSON envelope bytes. @@ -132,12 +229,53 @@ pub struct RuntimeBudgets { impl Default for RuntimeBudgets { fn default() -> Self { Self { + lambda_docker_environments: 32, + lambda_docker_memory_bytes: 8_589_934_592, + lambda_docker_pull_seconds: 300, + lambda_docker_deployment_jobs: 8, + lambda_docker_deployment_seconds: 300, + lambda_docker_image_pins: 1024, + lambda_docker_io_seconds: 5, + lambda_docker_code_upload_seconds: 60, + lambda_docker_start_seconds: 15, + lambda_docker_cleanup_seconds: 10, + + workflow_api_accepted: 32, + workflow_api_workers: 8, + workflow_pagination_bytes: 67_108_864, + workflow_pagination_tokens: 10_000, + workflow_registry_machines: 256, + workflow_registry_versions: 1024, + workflow_registry_records: 10_000, + workflow_registry_commands: 128, + workflow_name_tombstones: 100_000, + workflow_retention_seconds: 86_400, + workflow_open_executions: 128, + workflow_evaluations: 8, + workflow_dispatches: 32, + workflow_frames: 4096, + workflow_frames_per_execution: 512, + workflow_result_bytes: 268_435_456, + workflow_result_bytes_per_execution: 33_554_432, + workflow_max_duration_seconds: 3600, + workflow_transitions: 100_000, + workflow_history_bytes: 33_554_432, + workflow_snapshot_bytes: 536_870_912, + workflow_total_history_bytes: 268_435_456, + workflow_lambda_accepted_calls: 128, + workflow_lambda_cleanup_seconds: 10, + workflow_drain_seconds: 5, + workflow_callback_tokens: 4096, + workflow_callback_tombstones: 100_000, + workflow_callback_commands: 1024, + workflow_callback_workers: 8, connections: 256, requests: 128, header_seconds: 5, request_seconds: 30, lambda_invoke_seconds: 930, shutdown_seconds: 30, + startup_seconds: 600, control_body_bytes: 16 * 1024 * 1024, lambda_code_body_bytes: 96 * 1024 * 1024, upstream_body_bytes: 64 * 1024 * 1024, @@ -155,6 +293,23 @@ impl RuntimeBudgets { /// # Errors /// Rejects zero, excessive, or inconsistent resource deadlines and sizes. pub fn validate(&self) -> Result<(), SettingsError> { + self.validate_docker()?; + self.validate_workflow_api()?; + self.validate_workflow_execution()?; + self.validate_workflow_frames()?; + self.validate_workflow_callbacks()?; + if self.workflow_api_workers > self.workflow_api_accepted { + return Err(invalid( + "budgets.workflowApiWorkers", + "must not exceed accepted requests", + )); + } + if self.workflow_history_bytes > self.workflow_total_history_bytes { + return Err(invalid( + "budgets.workflowHistoryBytes", + "must not exceed total history bytes", + )); + } for (key, value, max) in [ ("budgets.connections", self.connections as u64, 65_536), ("budgets.requests", self.requests as u64, 65_536), @@ -166,6 +321,7 @@ impl RuntimeBudgets { 3600, ), ("budgets.shutdownSeconds", self.shutdown_seconds, 3600), + ("budgets.startupSeconds", self.startup_seconds, 3600), ( "budgets.controlBodyBytes", self.control_body_bytes, @@ -208,6 +364,299 @@ impl RuntimeBudgets { } Ok(()) } + fn validate_docker(&self) -> Result<(), SettingsError> { + for (key, value, min, max) in [ + ( + "lambdaDockerEnvironments", + self.lambda_docker_environments as u64, + 1, + 32, + ), + ( + "lambdaDockerMemoryBytes", + self.lambda_docker_memory_bytes, + 268_435_456, + 1_099_511_627_776, + ), + ( + "lambdaDockerPullSeconds", + self.lambda_docker_pull_seconds, + 1, + 600, + ), + ( + "lambdaDockerDeploymentJobs", + self.lambda_docker_deployment_jobs as u64, + 1, + 32, + ), + ( + "lambdaDockerDeploymentSeconds", + self.lambda_docker_deployment_seconds, + 1, + 3600, + ), + ( + "lambdaDockerImagePins", + self.lambda_docker_image_pins as u64, + 1, + 8192, + ), + ( + "lambdaDockerIoSeconds", + self.lambda_docker_io_seconds, + 1, + 60, + ), + ( + "lambdaDockerCodeUploadSeconds", + self.lambda_docker_code_upload_seconds, + 1, + 600, + ), + ( + "lambdaDockerStartSeconds", + self.lambda_docker_start_seconds, + 1, + 60, + ), + ( + "lambdaDockerCleanupSeconds", + self.lambda_docker_cleanup_seconds, + 1, + 60, + ), + ] { + if !(min..=max).contains(&value) { + return Err(invalid( + &format!("budgets.{key}"), + "outside supported range", + )); + } + } + if self.lambda_docker_deployment_jobs > self.lambda_docker_image_pins { + return Err(invalid( + "budgets.lambdaDockerDeploymentJobs", + "must not exceed image pins", + )); + } + Ok(()) + } + fn validate_workflow_api(&self) -> Result<(), SettingsError> { + for (key, value, min, max) in [ + ( + "budgets.workflowApiAccepted", + self.workflow_api_accepted as u64, + 1, + 128, + ), + ( + "budgets.workflowApiWorkers", + self.workflow_api_workers as u64, + 1, + 32, + ), + ( + "budgets.workflowPaginationBytes", + self.workflow_pagination_bytes as u64, + 8_388_608, + 1_073_741_824, + ), + ( + "budgets.workflowPaginationTokens", + self.workflow_pagination_tokens as u64, + 1, + 100_000, + ), + ( + "budgets.workflowRegistryMachines", + self.workflow_registry_machines as u64, + 1, + 4096, + ), + ( + "budgets.workflowRegistryVersions", + self.workflow_registry_versions as u64, + 1, + 4096, + ), + ( + "budgets.workflowRegistryRecords", + self.workflow_registry_records as u64, + 1, + 100_000, + ), + ( + "budgets.workflowRegistryCommands", + self.workflow_registry_commands as u64, + 1, + 1024, + ), + ( + "budgets.workflowNameTombstones", + self.workflow_name_tombstones as u64, + 1, + 1_000_000, + ), + ( + "budgets.workflowRetentionSeconds", + self.workflow_retention_seconds, + 60, + 7_776_000, + ), + ] { + if !(min..=max).contains(&value) { + return Err(invalid(key, "outside supported workflow range")); + } + } + Ok(()) + } + fn validate_workflow_frames(&self) -> Result<(), SettingsError> { + for (key, value, min, max) in [ + ("budgets.workflowFrames", self.workflow_frames, 1, 4096), + ( + "budgets.workflowFramesPerExecution", + self.workflow_frames_per_execution, + 1, + 512, + ), + ( + "budgets.workflowResultBytes", + self.workflow_result_bytes, + 4096, + 1024 * 1024 * 1024, + ), + ( + "budgets.workflowResultBytesPerExecution", + self.workflow_result_bytes_per_execution, + 4096, + 128 * 1024 * 1024, + ), + ] { + if !(min..=max).contains(&value) { + return Err(invalid(key, "outside supported workflow frame range")); + } + } + if self.workflow_frames_per_execution > self.workflow_frames + || self.workflow_result_bytes_per_execution > self.workflow_result_bytes + { + return Err(invalid( + "budgets.workflowFrames", + "per-execution frame/result limits must not exceed global limits", + )); + } + Ok(()) + } + fn validate_workflow_callbacks(&self) -> Result<(), SettingsError> { + for (key, value, maximum) in [ + ( + "budgets.workflowCallbackTokens", + self.workflow_callback_tokens, + 65_536, + ), + ( + "budgets.workflowCallbackTombstones", + self.workflow_callback_tombstones, + 1_000_000, + ), + ( + "budgets.workflowCallbackCommands", + self.workflow_callback_commands, + 100_000, + ), + ( + "budgets.workflowCallbackWorkers", + self.workflow_callback_workers, + 128, + ), + ] { + if !(1..=maximum).contains(&value) { + return Err(invalid(key, "outside supported callback range")); + } + } + if self.workflow_callback_tombstones < self.workflow_callback_tokens { + return Err(invalid( + "budgets.workflowCallbackTombstones", + "must include live token reservations", + )); + } + Ok(()) + } + fn validate_workflow_execution(&self) -> Result<(), SettingsError> { + for (key, value, min, max) in [ + ( + "budgets.workflowOpenExecutions", + self.workflow_open_executions as u64, + 1, + 4096, + ), + ( + "budgets.workflowEvaluations", + self.workflow_evaluations as u64, + 1, + 128, + ), + ( + "budgets.workflowDispatches", + self.workflow_dispatches as u64, + 1, + 1024, + ), + ( + "budgets.workflowMaxDurationSeconds", + self.workflow_max_duration_seconds, + 1, + 31_536_000, + ), + ( + "budgets.workflowTransitions", + u64::from(self.workflow_transitions), + 1, + 1_000_000, + ), + ( + "budgets.workflowHistoryBytes", + self.workflow_history_bytes as u64, + 1_048_576, + 268_435_456, + ), + ( + "budgets.workflowSnapshotBytes", + self.workflow_snapshot_bytes as u64, + 1_048_576, + 1_073_741_824, + ), + ( + "budgets.workflowTotalHistoryBytes", + self.workflow_total_history_bytes as u64, + 8_388_608, + 4_294_967_296, + ), + ( + "budgets.workflowLambdaAcceptedCalls", + self.workflow_lambda_accepted_calls as u64, + 1, + 1024, + ), + ( + "budgets.workflowLambdaCleanupSeconds", + self.workflow_lambda_cleanup_seconds, + 1, + 60, + ), + ( + "budgets.workflowDrainSeconds", + self.workflow_drain_seconds, + 0, + 60, + ), + ] { + if !(min..=max).contains(&value) { + return Err(invalid(key, "outside supported workflow range")); + } + } + Ok(()) + } } #[derive(Deserialize, Default)] @@ -436,7 +885,12 @@ fn validate_value(key: &str, value: &mut String) -> Result<(), SettingsError> { if value.is_empty() && key != "SERVICES" { return Err(invalid(key, "must not be empty")); } - if value.len() > 4096 || value.chars().any(char::is_control) { + let maximum = if key == "LAMBDA_DOCKER_REGISTRIES" { + 8127 + } else { + 4096 + }; + if value.len() > maximum || value.chars().any(char::is_control) { return Err(invalid( key, "exceeds byte limit or contains control characters", @@ -448,6 +902,7 @@ fn validate_value(key: &str, value: &mut String) -> Result<(), SettingsError> { "PERSISTENCE" | "S3_VIRTUAL_HOSTING" | "LAMBDA_DOCKER_ENABLED" + | "LAMBDA_DOCKER_CROSS_ARCHITECTURE" | "CLOUDFRONT_DETERMINISTIC_IDS" | "CLOUDFRONT_FAIL_ON_FUNCTION" | "CLOUDFRONT_FORWARD_USER_METADATA" @@ -657,6 +1112,136 @@ pub fn advertised_endpoint() -> Option<&'static str> { mod tests { use super::*; + #[test] + fn test_should_validate_workflow_budgets_and_strict_credentials() -> Result<(), SettingsError> { + let config = ValidatedSettings::parse( + "environment:\n STEPFUNCTIONS_SKIP_SIGNATURE_VALIDATION: false\nbudgets:\n \ + workflowDrainSeconds: 0\n workflowApiAccepted: 2\n workflowApiWorkers: 1", + &BTreeMap::new(), + )?; + assert!( + config + .validate_credentials(&["stepfunctions".to_owned()]) + .is_err() + ); + assert_eq!(config.budgets().workflow_drain_seconds, 0); + for setting in [ + "workflowApiAccepted: 0", + "workflowApiWorkers: 33", + "workflowPaginationBytes: 8388607", + "workflowPaginationTokens: 100001", + "workflowRegistryMachines: 4097", + "workflowRegistryVersions: 0", + "workflowRegistryRecords: 100001", + "workflowRegistryCommands: 1025", + "workflowNameTombstones: 0", + "workflowRetentionSeconds: 59", + "workflowOpenExecutions: 0", + "workflowEvaluations: 129", + "workflowDispatches: 1025", + "workflowFrames: 0", + "workflowFrames: 4097", + "workflowFramesPerExecution: 513", + "workflowResultBytes: 4095", + "workflowResultBytes: 1073741825", + "workflowResultBytesPerExecution: 134217729", + "workflowFrames: 1\n workflowFramesPerExecution: 2", + "workflowResultBytes: 4096\n workflowResultBytesPerExecution: 4097", + "workflowMaxDurationSeconds: 0", + "workflowTransitions: 1000001", + "workflowHistoryBytes: 1048575", + "workflowSnapshotBytes: 1048575", + "workflowSnapshotBytes: 1073741825", + "workflowTotalHistoryBytes: 4294967297", + "workflowLambdaAcceptedCalls: 0", + "workflowLambdaCleanupSeconds: 61", + "workflowDrainSeconds: 61", + "workflowCallbackTokens: 0", + "workflowCallbackTokens: 65537", + "workflowCallbackTombstones: 1000001", + "workflowCallbackTombstones: 1\n workflowCallbackTokens: 2", + "workflowCallbackCommands: 0", + "workflowCallbackCommands: 100001", + "workflowCallbackWorkers: 0", + "workflowCallbackWorkers: 129", + "workflowApiAccepted: 1\n workflowApiWorkers: 2", + "workflowHistoryBytes: 33554432\n workflowTotalHistoryBytes: 8388608", + ] { + assert!( + ValidatedSettings::parse(&format!("budgets:\n {setting}"), &BTreeMap::new()) + .is_err(), + "{setting}" + ); + } + Ok(()) + } + + #[test] + fn test_should_enforce_exact_frame_and_result_configuration_bounds() -> Result<(), SettingsError> + { + for (frames, local_frames, results, local_results) in [ + (1, 1, 4096, 4096), + (4096, 512, 1024 * 1024 * 1024, 128 * 1024 * 1024), + ] { + let config = ValidatedSettings::parse( + &format!( + "budgets:\n workflowFrames: {frames}\n workflowFramesPerExecution: \ + {local_frames}\n workflowResultBytes: {results}\n \ + workflowResultBytesPerExecution: {local_results}\n" + ), + &BTreeMap::new(), + )?; + assert_eq!(config.budgets().workflow_frames, frames); + assert_eq!(config.budgets().workflow_frames_per_execution, local_frames); + assert_eq!(config.budgets().workflow_result_bytes, results); + assert_eq!( + config.budgets().workflow_result_bytes_per_execution, + local_results + ); + } + Ok(()) + } + + #[test] + fn test_should_enforce_exact_callback_configuration_bounds_and_defaults() + -> Result<(), SettingsError> { + let defaults = RuntimeBudgets::default(); + assert_eq!( + ( + defaults.workflow_callback_tokens, + defaults.workflow_callback_tombstones, + defaults.workflow_callback_commands, + defaults.workflow_callback_workers + ), + (4096, 100_000, 1024, 8) + ); + for (tokens, retained, commands, workers, seconds) in [ + (1, 1, 1, 1, 60), + (65_536, 1_000_000, 100_000, 128, 7_776_000), + ] { + let config = ValidatedSettings::parse( + &format!( + "budgets:\n workflowCallbackTokens: {tokens}\n workflowCallbackTombstones: \ + {retained}\n workflowCallbackCommands: {commands}\n \ + workflowCallbackWorkers: {workers}\n workflowRetentionSeconds: {seconds}\n" + ), + &BTreeMap::new(), + )?; + let budgets = config.budgets(); + assert_eq!( + ( + budgets.workflow_callback_tokens, + budgets.workflow_callback_tombstones, + budgets.workflow_callback_commands, + budgets.workflow_callback_workers, + budgets.workflow_retention_seconds + ), + (tokens, retained, commands, workers, seconds) + ); + } + Ok(()) + } + #[test] fn test_should_apply_environment_over_yaml_and_derive_ipv6_endpoint() -> Result<(), SettingsError> { @@ -701,3 +1286,104 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod docker_tests { + use std::collections::BTreeMap; + + use super::{RuntimeBudgets, ValidatedSettings}; + #[test] + fn test_should_accept_docker_yaml_keys_budgets_and_process_precedence() { + let yaml = "environment:\n LAMBDA_DOCKER_RELAY_IMAGE: local:v1\n LAMBDA_DOCKER_NETWORK: \ + none\n LAMBDA_DOCKER_PULL_POLICY: never\n LAMBDA_DOCKER_CROSS_ARCHITECTURE: \ + yes\nbudgets:\n lambdaDockerPullSeconds: 1\n lambdaDockerImagePins: 32\n \ + lambdaDockerDeploymentJobs: 32\n"; + let settings = ValidatedSettings::parse( + yaml, + &BTreeMap::from([("LAMBDA_DOCKER_RELAY_IMAGE".into(), "override:v2".into())]), + ) + .unwrap(); + assert_eq!( + settings.get("LAMBDA_DOCKER_RELAY_IMAGE"), + Some("override:v2") + ); + assert_eq!( + settings.get("LAMBDA_DOCKER_CROSS_ARCHITECTURE"), + Some("true") + ); + assert_eq!(settings.budgets().lambda_docker_pull_seconds, 1); + assert_eq!(RuntimeBudgets::default().lambda_docker_pull_seconds, 300); + } + #[test] + fn test_should_reject_every_docker_budget_outside_its_supported_range() { + for (key, min, max) in [ + ("lambdaDockerEnvironments", 1_u64, 32_u64), + ("lambdaDockerMemoryBytes", 268_435_456, 1_099_511_627_776), + ("lambdaDockerPullSeconds", 1, 600), + ("lambdaDockerDeploymentJobs", 1, 32), + ("lambdaDockerDeploymentSeconds", 1, 3600), + ("lambdaDockerImagePins", 1, 8192), + ("lambdaDockerIoSeconds", 1, 60), + ("lambdaDockerCodeUploadSeconds", 1, 600), + ("lambdaDockerStartSeconds", 1, 60), + ("lambdaDockerCleanupSeconds", 1, 60), + ] { + for value in [min - 1, max + 1] { + let raw = format!("budgets:\n {key}: {value}\n"); + assert!( + ValidatedSettings::parse(&raw, &BTreeMap::new()).is_err(), + "{key}" + ); + } + for value in [min, max] { + let raw = format!("budgets:\n lambdaDockerDeploymentJobs: 1\n {key}: {value}\n"); + // Avoid duplicate keys when the row itself is deployment jobs. + let raw = if key == "lambdaDockerDeploymentJobs" { + format!("budgets:\n {key}: {value}\n") + } else { + raw + }; + assert!( + ValidatedSettings::parse(&raw, &BTreeMap::new()).is_ok(), + "{key}" + ); + } + } + assert!( + ValidatedSettings::parse( + "budgets:\n lambdaDockerImagePins: 1\n lambdaDockerDeploymentJobs: 2", + &BTreeMap::new() + ) + .is_err() + ); + } +} + +#[cfg(test)] +mod startup_tests { + use std::collections::BTreeMap; + + use super::{RuntimeBudgets, ValidatedSettings}; + #[test] + fn test_should_bound_one_process_startup_budget_independently_of_shutdown() { + assert_eq!(RuntimeBudgets::default().startup_seconds, 600); + for seconds in [0, 3601] { + assert!( + ValidatedSettings::parse( + &format!("budgets:\n startupSeconds: {seconds}\n"), + &BTreeMap::new() + ) + .is_err() + ); + } + for seconds in [1, 3600] { + let settings = ValidatedSettings::parse( + &format!("budgets:\n startupSeconds: {seconds}\n shutdownSeconds: 10\n"), + &BTreeMap::new(), + ) + .unwrap(); + assert_eq!(settings.budgets().startup_seconds, seconds); + assert_eq!(settings.budgets().shutdown_seconds, 10); + } + } +} diff --git a/crates/rustack-core/src/state.rs b/crates/rustack-core/src/state.rs index 16ef0774..d53f8d4d 100644 --- a/crates/rustack-core/src/state.rs +++ b/crates/rustack-core/src/state.rs @@ -1,7 +1,7 @@ //! Multi-account, multi-region state management. //! //! Provides [`AccountRegionStore`], a thread-safe concurrent store that -//! partitions state by AWS account ID and region, matching LocalStack's +//! partitions state by AWS account ID and region, matching `LocalStack`'s //! `AccountRegionBundle` pattern. use std::sync::Arc; diff --git a/crates/rustack-core/src/types.rs b/crates/rustack-core/src/types.rs index 76ee8100..aeb9f3a1 100644 --- a/crates/rustack-core/src/types.rs +++ b/crates/rustack-core/src/types.rs @@ -7,7 +7,7 @@ use std::fmt; pub struct AccountId(String); impl AccountId { - /// Default account ID used by LocalStack. + /// Default account ID used by `LocalStack`. pub const DEFAULT: &str = "000000000000"; /// Create a new account ID from a string. @@ -46,7 +46,7 @@ impl fmt::Display for AccountId { pub struct AwsRegion(String); impl AwsRegion { - /// Default region used by LocalStack. + /// Default region used by `LocalStack`. pub const DEFAULT: &str = "us-east-1"; /// Create a new region. diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 9e932cf7..f16c5259 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -14,25 +14,42 @@ workspace = true rustack-core = { workspace = true } rustack-lambda-model = { workspace = true } rustack-lambda-http = { workspace = true } +rustack-lambda-runtime = { workspace = true } rustack-auth = { workspace = true } squib = { workspace = true } http = { workspace = true } bytes = { workspace = true } +bollard-stubs = { workspace = true } +percent-encoding = { workspace = true } +form_urlencoded = { workspace = true } +typed-builder = { workspace = true } +winnow = { workspace = true } dashmap = { workspace = true } serde = { workspace = true } -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } chrono = { workspace = true } uuid = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } +nix = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } sha2 = { workspace = true } +hex = { workspace = true } base64 = { workspace = true } +secrecy = { workspace = true } +yaml-rust2 = { workspace = true } async-trait = { workspace = true } tempfile = { workspace = true } zip = { workspace = true } +tar = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } http-body-util = { workspace = true } parking_lot = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +rstest = { workspace = true } +tracing-subscriber = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs new file mode 100644 index 00000000..fa4a3e4a --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -0,0 +1,205 @@ +//! Owned, validated deployment artifacts shared by captured execution requests. +//! +//! ZIP leases keep successfully staged code alive through invocation cleanup. +//! Images carry immutable daemon identity rather than an unresolved mutable tag. + +mod execution; +mod image; +mod pinned; + +use std::{ + fmt::{Debug, Formatter, Result as FmtResult}, + fs::{canonicalize, remove_dir_all, symlink_metadata}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use bytes::Bytes; +pub use execution::{ImageExecutionDefaults, ImageExecutionMetadata, ImageExecutionPlan}; +pub use image::{ + Architecture, ArtifactError, ImageMetadata, ImageReference, ResolvedImage, Sha256Digest, +}; +pub(crate) use pinned::{ + ImageLeaseContext, ImageLiveness, PreparationAdmission, PreparationReceipt, +}; +pub use pinned::{PinnedImage, UnpackedImageSize}; +use tempfile::TempDir; + +use crate::storage::compute_sha256; + +pub(crate) const MAX_ZIP_BYTES: usize = 50 * 1024 * 1024; +pub(crate) const MAX_ZIP_BASE64_BYTES: usize = MAX_ZIP_BYTES.div_ceil(3) * 4; + +pub(crate) const MAX_IMAGE_BYTES: u64 = 10 * 1024 * 1024 * 1024; +#[cfg(test)] +use crate::{storage::stage_zip, test_zip::minimal_zip}; + +/// One unambiguous deployment artifact for an invocation. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum DeploymentArtifact { + /// Successfully staged ZIP, with shared lifetime ownership. + Zip(ValidatedArtifact), + /// Live Linux image lease, retained through backend cleanup. + Image(PinnedImage), +} + +/// Image deployment availability; snapshot metadata never grants a live lease. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub enum ImageArtifact { + /// No image has been prepared. + #[default] + Unresolved, + /// Immutable restored metadata requiring availability reacquisition. + Restored(ResolvedImage), + /// Prepared image with shared availability ownership. + Pinned(PinnedImage), +} +impl ImageArtifact { + /// Captured metadata, whether or not runtime availability has been acquired. + /// + /// ``` + /// assert!(rustack_lambda_core::artifact::ImageArtifact::Unresolved.resolved().is_none()); + /// ``` + #[must_use] + pub fn resolved(&self) -> Option<&ResolvedImage> { + match self { + Self::Unresolved => None, + Self::Restored(image) => Some(image), + Self::Pinned(image) => Some(image.resolved()), + } + } + + /// Live artifact ownership, absent for restored metadata alone. + /// + /// ``` + /// assert!(rustack_lambda_core::artifact::ImageArtifact::Unresolved.pinned().is_none()); + /// ``` + #[must_use] + pub fn pinned(&self) -> Option<&PinnedImage> { + match self { + Self::Pinned(image) => Some(image), + Self::Unresolved | Self::Restored(_) => None, + } + } +} + +/// Shared ownership of a ZIP archive and the extracted code it produced. +/// +/// Only the function store can issue a lease after successful staging. +#[derive(Clone)] +#[non_exhaustive] +pub struct ValidatedArtifact(Arc); + +struct ZipStorage { + directory: Option, + root: PathBuf, + extracted: PathBuf, + bytes: Bytes, + sha256: String, +} + +impl Debug for ZipStorage { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.debug_struct("ZipStorage") + .field("sha256", &self.sha256) + .field("archive_bytes", &self.bytes.len()) + .finish_non_exhaustive() + } +} + +impl Debug for ValidatedArtifact { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.debug_struct("ValidatedArtifact") + .field("sha256", &self.0.sha256) + .field("archive_bytes", &self.0.bytes.len()) + .finish_non_exhaustive() + } +} + +impl ValidatedArtifact { + #[cfg(test)] + pub(crate) fn fixture() -> Self { + stage_zip(&std::env::temp_dir(), Bytes::from(minimal_zip(b"fixture"))) + .expect("trusted ZIP unit fixture must stage successfully") + } + pub(crate) fn from_staged( + directory: TempDir, + root: PathBuf, + extracted: PathBuf, + bytes: Bytes, + ) -> Self { + let sha256 = compute_sha256(&bytes); + Self(Arc::new(ZipStorage { + directory: Some(directory), + root, + extracted, + bytes, + sha256, + })) + } + + /// Absolute root of the extracted code, not a user-controlled logical name. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// assert!(artifact.code_root().is_absolute()); + /// # } + /// ``` + #[must_use] + pub fn code_root(&self) -> &Path { + &self.0.extracted + } + + /// Original bytes from the same successfully staged ZIP. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// let shared_bytes = artifact.zip_bytes().clone(); + /// assert_eq!(shared_bytes, *artifact.zip_bytes()); + /// # } + /// ``` + #[must_use] + pub fn zip_bytes(&self) -> &Bytes { + &self.0.bytes + } + + /// Base64-encoded SHA-256 of the original archive. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// assert_eq!(artifact.sha256().len(), 44); + /// # } + /// ``` + #[must_use] + pub fn sha256(&self) -> &str { + &self.0.sha256 + } + + pub(crate) fn storage_root(&self) -> &Path { + &self.0.root + } +} + +impl Drop for ZipStorage { + #[allow(clippy::disallowed_methods)] // Last-lease Drop owns bounded directory cleanup; async IO is unavailable here. + fn drop(&mut self) { + let Some(directory) = self.directory.take() else { + return; + }; + let path = directory.keep(); + let safe = canonicalize(&self.root).is_ok_and(|root| root == self.root) + && symlink_metadata(&self.root) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()) + && symlink_metadata(&path) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()); + if !safe { + tracing::error!("Refusing cleanup of replaced Lambda artifact root"); + return; + } + if let Err(error) = remove_dir_all(path) { + tracing::error!(%error, "Lambda artifact cleanup failed"); + } + } +} diff --git a/crates/rustack-lambda-core/src/artifact/execution.rs b/crates/rustack-lambda-core/src/artifact/execution.rs new file mode 100644 index 00000000..f5c230e5 --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/execution.rs @@ -0,0 +1,384 @@ +//! Validated image defaults retained through capture and snapshot restoration. + +use std::{fmt, sync::Arc}; + +use rustack_lambda_model::types::ImageConfig; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::ArtifactError; +use crate::function_config::validate_image_config; + +/// Primitive image execution metadata; validate with `ImageExecutionDefaults`. +#[derive(Clone, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ImageExecutionMetadata { + /// Inherited entrypoint, before function overrides. + pub entrypoint: Vec, + /// Inherited command, before function overrides. + pub command: Vec, + /// Image-internal working directory; empty normalizes to root. + pub working_directory: String, + /// Ordered inherited environment, including Docker deletion markers. + pub environment: Vec, + /// Image user declaration; container security policy is applied separately. + pub user: String, +} +impl fmt::Debug for ImageExecutionMetadata { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ImageExecutionMetadata()") + } +} + +/// Immutable normalized image defaults, revalidated on deserialization. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(try_from = "ImageExecutionMetadata", into = "ImageExecutionMetadata")] +#[non_exhaustive] +pub struct ImageExecutionDefaults(Arc); + +impl TryFrom for ImageExecutionDefaults { + type Error = ArtifactError; + fn try_from(mut metadata: ImageExecutionMetadata) -> Result { + if metadata.entrypoint.len() == 1 + && metadata.entrypoint.first().is_some_and(String::is_empty) + { + metadata.entrypoint.clear(); + } + if metadata.working_directory.is_empty() { + metadata.working_directory = "/".into(); + } + validate_image_config(&ImageConfig { + entry_point: Some(metadata.entrypoint.clone()), + command: Some(metadata.command.clone()), + working_directory: Some(metadata.working_directory.clone()), + }) + .map_err(|_| ArtifactError::Inspection)?; + validate_environment(&metadata.environment)?; + if metadata.user.len() > 256 + || !metadata.user.bytes().all(|byte| (32..=126).contains(&byte)) + { + return Err(ArtifactError::Inspection); + } + Ok(Self(Arc::new(metadata))) + } +} +impl From for ImageExecutionMetadata { + fn from(defaults: ImageExecutionDefaults) -> Self { + Arc::unwrap_or_clone(defaults.0) + } +} + +impl ImageExecutionDefaults { + /// Borrow normalized defaults; diagnostics must not print their values. + /// + /// ```no_run + /// # fn defaults(defaults: &rustack_lambda_core::artifact::ImageExecutionDefaults) { + /// let environment = &defaults.metadata().environment; + /// # } + /// ``` + #[must_use] + pub fn metadata(&self) -> &ImageExecutionMetadata { + &self.0 + } + + /// Materialize the exact Docker argv/workdir, preserving empty overrides. + /// + /// # Errors + /// Rejects invalid overrides, an empty executable or composed argv bounds. + /// + /// ```no_run + /// # fn plan(defaults: &rustack_lambda_core::artifact::ImageExecutionDefaults) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let plan = defaults.plan(None)?; + /// assert!(!plan.entrypoint().is_empty()); // Includes Docker's reset sentinel. + /// # Ok(()) } + /// ``` + pub fn plan( + &self, + overrides: Option<&ImageConfig>, + ) -> Result { + if let Some(config) = overrides { + validate_image_config(config).map_err(|_| ArtifactError::Inspection)?; + } + let mut entrypoint = overrides + .and_then(|config| config.entry_point.as_ref()) + .unwrap_or(&self.0.entrypoint) + .clone(); + let command = overrides + .and_then(|config| config.command.as_ref()) + .unwrap_or(&self.0.command) + .clone(); + let workdir = overrides + .and_then(|config| config.working_directory.as_ref()) + .unwrap_or(&self.0.working_directory) + .clone(); + if entrypoint.len() == 1 && entrypoint.first().is_some_and(String::is_empty) { + entrypoint.clear(); + } + validate_image_config(&ImageConfig { + entry_point: Some(entrypoint.clone()), + command: Some(command.clone()), + working_directory: Some(workdir.clone()), + }) + .map_err(|_| ArtifactError::Inspection)?; + if entrypoint + .iter() + .chain(&command) + .next() + .is_none_or(String::is_empty) + { + return Err(ArtifactError::Inspection); + } + if entrypoint.is_empty() { + entrypoint.push(String::new()); + } + Ok(ImageExecutionPlan { + entrypoint, + command, + workdir, + }) + } +} + +/// Validated Docker-create fields; no daemon inheritance is needed. +#[derive(Clone)] +#[non_exhaustive] +pub struct ImageExecutionPlan { + entrypoint: Vec, + command: Vec, + workdir: String, +} +impl fmt::Debug for ImageExecutionPlan { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ImageExecutionPlan()") + } +} +impl ImageExecutionPlan { + /// Exact create entrypoint; semantic emptiness uses the `[""]` reset sentinel. + /// + /// ```no_run + /// # fn entry(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let entrypoint = plan.entrypoint().to_vec(); + /// # } + /// ``` + #[must_use] + pub fn entrypoint(&self) -> &[String] { + &self.entrypoint + } + /// Exact create command, including an explicitly empty array. + /// + /// ```no_run + /// # fn command(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let command = plan.command().to_vec(); + /// # } + /// ``` + #[must_use] + pub fn command(&self) -> &[String] { + &self.command + } + /// Exact absolute working directory inside the image. + /// + /// ```no_run + /// # fn workdir(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let workdir = plan.working_directory(); + /// # } + /// ``` + #[must_use] + pub fn working_directory(&self) -> &str { + &self.workdir + } +} + +fn validate_environment(environment: &[String]) -> Result<(), ArtifactError> { + if environment.len() > 128 { + return Err(ArtifactError::Inspection); + } + let mut total = 0_usize; + for entry in environment { + let key = entry.split_once('=').map_or(entry.as_str(), |(key, _)| key); + if entry.len() > 4096 + || entry.contains('\0') + || key.len() > 256 + || !key + .as_bytes() + .first() + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err(ArtifactError::Inspection); + } + total = total + .checked_add(entry.len()) + .filter(|total| *total <= 64 * 1024) + .ok_or(ArtifactError::Inspection)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn metadata() -> ImageExecutionMetadata { + ImageExecutionMetadata::builder() + .entrypoint(vec!["/entry".into()]) + .command(vec!["handler".into()]) + .working_directory("/task".into()) + .environment(vec!["PATH=/bin".into(), "REMOVED".into()]) + .user("1000:1000".into()) + .build() + } + + #[test] + fn test_should_materialize_inherited_and_explicit_empty_docker_argv() { + let defaults = ImageExecutionDefaults::try_from(metadata()).unwrap(); + let inherited = defaults.plan(None).unwrap(); + assert_eq!(inherited.entrypoint(), ["/entry"]); + assert_eq!(inherited.command(), ["handler"]); + assert_eq!(inherited.working_directory(), "/task"); + let clear_command = defaults + .plan(Some(&ImageConfig { + command: Some(vec![]), + ..Default::default() + })) + .unwrap(); + assert_eq!(clear_command.command(), [] as [String; 0]); + assert_eq!(clear_command.entrypoint(), ["/entry"]); + for reset in [vec![], vec![String::new()]] { + let override_config = ImageConfig { + entry_point: Some(reset), + command: Some(vec!["/other".into()]), + working_directory: Some("/tmp".into()), + }; + let plan = defaults.plan(Some(&override_config)).unwrap(); + assert_eq!(plan.entrypoint(), [""]); + assert_eq!(plan.command(), ["/other"]); + assert_eq!(plan.working_directory(), "/tmp"); + } + assert!( + defaults + .plan(Some(&ImageConfig { + entry_point: Some(vec![]), + command: Some(vec![]), + ..Default::default() + })) + .is_err() + ); + } + + #[test] + fn test_should_allow_empty_defaults_with_an_executable_override_and_normalize_root() { + let mut input = metadata(); + input.entrypoint = vec![String::new()]; + input.command.clear(); + input.working_directory.clear(); + let defaults = ImageExecutionDefaults::try_from(input).unwrap(); + assert_eq!(defaults.metadata().entrypoint, [] as [String; 0]); + assert_eq!(defaults.metadata().working_directory, "/"); + assert!(defaults.plan(None).is_err()); + let plan = defaults + .plan(Some(&ImageConfig { + command: Some(vec!["/start".into()]), + ..Default::default() + })) + .unwrap(); + assert_eq!(plan.entrypoint(), [""]); + assert_eq!(plan.command(), ["/start"]); + let mut invalid = metadata(); + invalid.entrypoint.clear(); + invalid.command.clear(); + invalid.working_directory = "relative".into(); + assert!(ImageExecutionDefaults::try_from(invalid).is_err()); + } + + #[rstest::rstest] + #[case("argv_count")] + #[case("argv_item")] + #[case("argv_total")] + #[case("argv_nul")] + #[case("directory")] + #[case("directory_size")] + #[case("env_count")] + #[case("env_item")] + #[case("env_total")] + #[case("env_key")] + #[case("env_key_size")] + #[case("env_nul")] + #[case("user_size")] + #[case("user_control")] + fn test_should_reject_invalid_defaults_before_capture(#[case] field: &str) { + let mut input = metadata(); + match field { + "argv_count" => input.command = vec!["x".into(); 1501], + "argv_item" => input.command = vec!["x".repeat(4097)], + "argv_total" => input.command = vec!["x".repeat(4096); 17], + "argv_nul" => input.entrypoint = vec!["/bin/x\0".into()], + "directory" => input.working_directory = "relative".into(), + "directory_size" => input.working_directory = format!("/{}", "x".repeat(1000)), + "env_count" => input.environment = vec!["X=1".into(); 129], + "env_item" => input.environment = vec![format!("X={}", "x".repeat(4095))], + "env_total" => input.environment = vec![format!("X={}", "x".repeat(4094)); 17], + "env_key" => input.environment = vec!["1KEY=value".into()], + "env_key_size" => input.environment = vec![format!("{}=x", "K".repeat(257))], + "env_nul" => input.environment = vec!["KEY=\0".into()], + "user_size" => input.user = "x".repeat(257), + "user_control" => input.user = "1000\n".into(), + _ => panic!("unknown test field"), + } + assert!(matches!( + ImageExecutionDefaults::try_from(input), + Err(ArtifactError::Inspection) + )); + } + + #[test] + fn test_should_check_composed_argv_and_environment_exact_byte_boundaries() { + let mut input = metadata(); + input.entrypoint = vec!["e".repeat(4096); 16]; + input.command.clear(); + input.environment = vec![format!("X={}", "x".repeat(4094)); 16]; + let defaults = ImageExecutionDefaults::try_from(input).unwrap(); + defaults.plan(None).unwrap(); + assert!( + defaults + .plan(Some(&ImageConfig { + command: Some(vec!["x".into()]), + ..Default::default() + })) + .is_err() + ); + assert!( + defaults + .plan(Some(&ImageConfig { + working_directory: Some(String::new()), + ..Default::default() + })) + .is_err() + ); + } + + #[test] + fn test_should_revalidate_saved_defaults_and_redact_all_debug_surfaces() { + let mut input = metadata(); + input.entrypoint = vec!["secret_argv".into()]; + input.environment = vec!["SECRET=secret_value".into()]; + input.working_directory = "/secret_path".into(); + input.user = "secret_user".into(); + let defaults = ImageExecutionDefaults::try_from(input.clone()).unwrap(); + let saved = serde_json::to_value(&defaults).unwrap(); + let restored: ImageExecutionDefaults = serde_json::from_value(saved.clone()).unwrap(); + assert_eq!(restored.plan(None).unwrap().entrypoint(), ["secret_argv"]); + for shown in [ + format!("{input:?}"), + format!("{defaults:?}"), + format!("{:?}", defaults.plan(None).unwrap()), + ] { + assert!(!shown.contains("secret_")); + } + let mut invalid = saved; + invalid["environment"] = serde_json::json!(["BAD-KEY=value"]); + assert!(serde_json::from_value::(invalid).is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/artifact/image.rs b/crates/rustack-lambda-core/src/artifact/image.rs new file mode 100644 index 00000000..692a3e25 --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/image.rs @@ -0,0 +1,773 @@ +//! Bounded Docker image reference grammar and immutable inspection metadata. + +use std::{fmt, net::Ipv6Addr, str::FromStr}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use typed_builder::TypedBuilder; +use winnow::{ + ModalResult, Parser, + combinator::{alt, repeat, separated}, + token::take_while, +}; + +use super::{ImageExecutionDefaults, MAX_IMAGE_BYTES}; + +/// Closed Lambda architecture, also used to select a Docker platform. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Architecture { + /// Linux amd64 image or `x86_64` function. + #[serde(rename = "x86_64")] + X86_64, + /// Linux aarch64 image or arm64 function. + #[serde(rename = "arm64")] + Arm64, +} + +impl Architecture { + /// Lambda wire spelling of this architecture. + /// + /// ``` + /// use rustack_lambda_core::artifact::Architecture; + /// assert_eq!(Architecture::Arm64.as_str(), "arm64"); + /// ``` + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::X86_64 => "x86_64", + Self::Arm64 => "arm64", + } + } + + /// Linux platform selector for Docker. + /// + /// ``` + /// use rustack_lambda_core::artifact::Architecture; + /// assert_eq!(Architecture::X86_64.docker_platform(), "linux/amd64"); + /// ``` + #[must_use] + pub const fn docker_platform(self) -> &'static str { + match self { + Self::X86_64 => "linux/amd64", + Self::Arm64 => "linux/arm64", + } + } +} + +impl FromStr for Architecture { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + match value { + "x86_64" => Ok(Self::X86_64), + "arm64" => Ok(Self::Arm64), + _ => Err(ArtifactError::Architecture), + } + } +} + +/// Invalid artifact metadata; errors never include the untrusted input. +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[non_exhaustive] +pub enum ArtifactError { + /// Reference does not match the bounded Docker grammar. + #[error("invalid Docker image reference")] + Reference, + /// Digest must be SHA-256 with 64 lowercase hexadecimal digits. + #[error("invalid SHA-256 image digest")] + Digest, + /// Only the two Lambda architectures are supported. + #[error("unsupported Lambda architecture")] + Architecture, + /// Inspection did not describe a supported Linux image. + #[error("invalid Linux image inspection metadata")] + Inspection, +} + +/// A validated Docker repository, optional tag, and optional SHA-256 digest. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct ImageReference(String); + +impl ImageReference { + /// Original validated spelling, passed directly to the daemon without a shell. + /// + /// ``` + /// use rustack_lambda_core::artifact::ImageReference; + /// let image: ImageReference = "localhost:5000/team/worker:v1".parse()?; + /// assert_eq!(image.as_str(), "localhost:5000/team/worker:v1"); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Repository spelling, including an explicit registry, without tag/digest. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "localhost:5000/team/worker:v1".parse()?; + /// assert_eq!(image.repository(), "localhost:5000/team/worker"); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn repository(&self) -> &str { + let name = self + .0 + .split_once('@') + .map_or(self.0.as_str(), |(name, _)| name); + if name + .rsplit('/') + .next() + .is_some_and(|part| part.contains(':')) + { + name.rsplit_once(':').map_or(name, |(name, _)| name) + } else { + name + } + } + + /// Explicit registry component; short local names do not invent provenance. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "worker:v1".parse()?; + /// assert_eq!(image.registry(), None); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn registry(&self) -> Option<&str> { + self.repository() + .split_once('/') + .and_then(|(first, _)| is_registry_component(first).then_some(first)) + } + + /// Requested digest suffix, if present, already checked by the constructor. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "worker:v1".parse()?; + /// assert_eq!(image.digest(), None); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn digest(&self) -> Option<&str> { + self.0.split_once('@').map(|(_, digest)| digest) + } +} + +fn is_registry_component(first: &str) -> bool { + first.contains('.') + || first.contains(':') + || first == "localhost" + || first.starts_with('[') + || first.bytes().any(|byte| byte.is_ascii_uppercase()) +} + +impl FromStr for ImageReference { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + if value.is_empty() || value.len() > 1024 || !value.is_ascii() { + return Err(ArtifactError::Reference); + } + let (name_tag, digest) = match value.split_once('@') { + Some((name, digest)) => (name, Some(digest)), + None => (value, None), + }; + if let Some(digest) = digest { + digest.parse::()?; + } + let last_component = name_tag + .rsplit('/') + .next() + .ok_or(ArtifactError::Reference)?; + let (name, tag) = match last_component.split_once(':') { + Some((_, tag)) => { + let (name, _) = name_tag.rsplit_once(':').ok_or(ArtifactError::Reference)?; + (name, Some(tag)) + } + None => (name_tag, None), + }; + if let Some(tag) = tag { + tag_parser + .parse(tag) + .map_err(|_| ArtifactError::Reference)?; + } + if name.len() > 255 { + return Err(ArtifactError::Reference); + } + let repository = if let Some((first, remaining)) = name.split_once('/') { + if is_registry_component(first) { + validate_registry(first)?; + remaining + } else { + name + } + } else { + name + }; + repository_parser + .parse(repository) + .map_err(|_| ArtifactError::Reference)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom for ImageReference { + type Error = ArtifactError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: ImageReference) -> Self { + value.0 + } +} +impl fmt::Display for ImageReference { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +fn component(input: &mut &str) -> ModalResult<()> { + ( + take_while(1.., |c: char| c.is_ascii_lowercase() || c.is_ascii_digit()), + repeat::<_, _, (), _, _>( + 0.., + ( + alt(("__", "_", ".", take_while(1.., '-'))), + take_while(1.., |c: char| c.is_ascii_lowercase() || c.is_ascii_digit()), + ), + ), + ) + .void() + .parse_next(input) +} +fn repository_parser(input: &mut &str) -> ModalResult<()> { + separated(1.., component, '/').parse_next(input) +} + +fn tag_parser(input: &mut &str) -> ModalResult<()> { + ( + take_while(1, |c: char| c.is_ascii_alphanumeric() || c == '_'), + take_while(0..=127, |c: char| { + c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') + }), + ) + .void() + .parse_next(input) +} + +fn digest_parser(input: &mut &str) -> ModalResult<()> { + ( + "sha256:", + take_while(64, |c: char| c.is_ascii_digit() || matches!(c, 'a'..='f')), + ) + .void() + .parse_next(input) +} + +fn validate_registry(registry: &str) -> Result<(), ArtifactError> { + let (host, port) = if let Some(bracketed) = registry.strip_prefix('[') { + let (host, tail) = bracketed.split_once(']').ok_or(ArtifactError::Reference)?; + if !host + .bytes() + .all(|byte| byte.is_ascii_hexdigit() || byte == b':') + { + return Err(ArtifactError::Reference); + } + host.parse::() + .map_err(|_| ArtifactError::Reference)?; + let port = if tail.is_empty() { + None + } else { + Some(tail.strip_prefix(':').ok_or(ArtifactError::Reference)?) + }; + (None, port) + } else { + let (host, port) = registry + .split_once(':') + .map_or((registry, None), |(host, port)| (host, Some(port))); + (Some(host), port) + }; + if let Some(host) = host { + if host.is_empty() + || host.len() > 253 + || host.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || !label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + || label.starts_with('-') + || label.ends_with('-') + }) + { + return Err(ArtifactError::Reference); + } + } + if let Some(port) = port { + if port.is_empty() + || port.len() > 5 + || !port.bytes().all(|b| b.is_ascii_digit()) + || port.parse::().map_or(true, |port| port == 0) + { + return Err(ArtifactError::Reference); + } + } + Ok(()) +} + +/// Immutable SHA-256 digest, including its algorithm prefix. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct Sha256Digest(String); + +impl Sha256Digest { + /// The validated `sha256:` value. + /// + /// ``` + /// use rustack_lambda_core::artifact::Sha256Digest; + /// let digest: Sha256Digest = format!("sha256:{}", "a".repeat(64)).parse()?; + /// assert_eq!(digest.as_str().len(), 71); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} +impl FromStr for Sha256Digest { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + if value.len() != 71 { + return Err(ArtifactError::Digest); + } + digest_parser + .parse(value) + .map_err(|_| ArtifactError::Digest)?; + Ok(Self(value.to_owned())) + } +} +impl TryFrom for Sha256Digest { + type Error = ArtifactError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: Sha256Digest) -> Self { + value.0 + } +} + +/// Validated primitive fields returned by image inspection. +#[derive(Debug, Clone, TypedBuilder, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ImageMetadata { + /// Caller-supplied deployment reference. + pub requested_reference: ImageReference, + /// Immutable daemon image content ID. + pub image_id: Sha256Digest, + /// Selected platform manifest, separate from an addressable parent image ID. + pub selected_manifest: Option, + /// Registry repository and selected manifest digest, when available. + pub registry_manifest: Option, + /// Inspected operating system, required to equal linux. + pub os: String, + /// Selected Lambda architecture. + pub architecture: Architecture, + /// SHA-256 of the normalized inspected execution metadata. + pub inspection_hash: Sha256Digest, + /// Selected image size reported by the daemon, bounded to 10 GiB locally. + pub size_bytes: u64, + /// Validated inherited execution defaults, retained without re-inspection. + pub execution: ImageExecutionDefaults, +} + +impl ImageMetadata { + /// Compute the canonical fingerprint also verified during snapshot restore. + /// + /// # Errors + /// Rejects an invalid inspected size or bounded metadata serialization. + /// + /// ```no_run + /// # fn hash(metadata: &rustack_lambda_core::artifact::ImageMetadata) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let hash = rustack_lambda_core::artifact::ImageMetadata::execution_fingerprint( + /// &metadata.image_id, metadata.selected_manifest.as_ref(), + /// metadata.architecture, metadata.size_bytes, &metadata.execution)?; + /// # Ok(()) } + /// ``` + pub fn execution_fingerprint( + image_id: &Sha256Digest, + selected_manifest: Option<&Sha256Digest>, + architecture: Architecture, + size: u64, + defaults: &ImageExecutionDefaults, + ) -> Result { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct Fingerprint<'a> { + version: u8, + image_id: &'a Sha256Digest, + descriptor_digest: Option<&'a Sha256Digest>, + architecture: Architecture, + size: u64, + defaults: &'a ImageExecutionDefaults, + } + if !(1..=MAX_IMAGE_BYTES).contains(&size) { + return Err(ArtifactError::Inspection); + } + // Validated defaults contain <=128KiB of argv/environment, whose worst + // JSON escape expansion is sixfold; fixed-size identities keep this <1MiB. + let bytes = serde_json::to_vec(&Fingerprint { + version: 1, + image_id, + descriptor_digest: selected_manifest, + architecture, + size, + defaults, + }) + .map_err(|_| ArtifactError::Inspection)?; + format!("sha256:{}", hex::encode(Sha256::digest(bytes))).parse() + } +} + +/// Immutable Linux image metadata that passed all inspection invariants. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(try_from = "ImageMetadata", into = "ImageMetadata")] +#[non_exhaustive] +pub struct ResolvedImage(ImageMetadata); + +impl TryFrom for ResolvedImage { + type Error = ArtifactError; + fn try_from(metadata: ImageMetadata) -> Result { + if metadata.os != "linux" + || metadata.size_bytes == 0 + || metadata.size_bytes > MAX_IMAGE_BYTES + || metadata + .registry_manifest + .as_ref() + .is_some_and(|image| !image.as_str().contains('@')) + { + return Err(ArtifactError::Inspection); + } + if metadata.inspection_hash + != ImageMetadata::execution_fingerprint( + &metadata.image_id, + metadata.selected_manifest.as_ref(), + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + )? + { + return Err(ArtifactError::Inspection); + } + if let (Some(selected), Some(registry)) = + (&metadata.selected_manifest, &metadata.registry_manifest) + { + if registry.digest() != Some(selected.as_str()) { + return Err(ArtifactError::Inspection); + } + } + Ok(Self(metadata)) + } +} +impl From for ImageMetadata { + fn from(image: ResolvedImage) -> Self { + image.0 + } +} + +impl ResolvedImage { + #[cfg(test)] + pub(crate) fn fixture(architecture: Architecture) -> Self { + let image_id = format!("sha256:{}", "a".repeat(64)) + .parse() + .expect("valid unit digest"); + let execution = super::ImageExecutionMetadata::builder() + .entrypoint(vec!["/bootstrap".into()]) + .command(Vec::new()) + .working_directory("/".into()) + .environment(Vec::new()) + .user("1000".into()) + .build() + .try_into() + .expect("valid execution defaults"); + let hash = + ImageMetadata::execution_fingerprint(&image_id, None, architecture, 1024, &execution) + .expect("unit fingerprint"); + ImageMetadata::builder() + .requested_reference( + "rustack-fixture:unit" + .parse() + .expect("valid unit reference"), + ) + .image_id(image_id) + .selected_manifest(None) + .registry_manifest(None) + .os("linux".into()) + .architecture(architecture) + .inspection_hash(hash) + .size_bytes(1024) + .execution(execution) + .build() + .try_into() + .expect("valid unit image metadata") + } + /// Base64 SHA-256 of canonical inspected execution metadata, independent of a tag. + /// + /// # Errors + /// Rejects a malformed canonical fingerprint before decoding it. + /// + /// ```no_run + /// # fn hash(image: &rustack_lambda_core::artifact::ResolvedImage) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let code_hash = image.code_sha256()?; + /// assert_eq!(code_hash.len(), 44); + /// # Ok(()) } + /// ``` + pub fn code_sha256(&self) -> Result { + let digest = self + .0 + .inspection_hash + .as_str() + .strip_prefix("sha256:") + .ok_or(ArtifactError::Inspection)?; + let bytes = hex::decode(digest).map_err(|_| ArtifactError::Inspection)?; + Ok(STANDARD.encode(bytes)) + } + + /// Immutable inspected metadata; Invoke must use `image_id` rather than a tag. + /// + /// ``` + /// # fn inspect(image: &rustack_lambda_core::artifact::ResolvedImage) { + /// assert_eq!(image.metadata().os, "linux"); + /// # } + /// ``` + #[must_use] + pub fn metadata(&self) -> &ImageMetadata { + &self.0 + } + + /// Registry manifest reference, or Rustack's explicit local-content extension. + /// + /// ``` + /// # fn inspect(image: &rustack_lambda_core::artifact::ResolvedImage) { + /// assert!(!image.resolved_uri().is_empty()); + /// # } + /// ``` + #[must_use] + pub fn resolved_uri(&self) -> String { + self.0.registry_manifest.as_ref().map_or_else( + || format!("local://{}", self.0.image_id.as_str()), + ToString::to_string, + ) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("worker")] + #[case("team/worker:v1")] + #[case("team/foo__bar:Tag_1.2-3")] + #[case("team/foo---bar:_")] + #[case("localhost:5000/worker:latest")] + #[case("registry.example.com/team/worker:V2")] + #[case("Registry/worker:v1")] + #[case("127.0.0.1:65535/worker")] + #[case("[::1]:5000/team/worker:v1")] + #[case("[2001:db8::1]/worker")] + fn test_should_parse_docker_references_and_round_trip_saved_values( + #[case] reference: &str, + ) -> Result<(), Box> { + let parsed: ImageReference = reference.parse()?; + assert_eq!(parsed.as_str(), reference); + let saved = serde_json::to_string(&parsed)?; + assert_eq!(serde_json::from_str::(&saved)?, parsed); + Ok(()) + } + + #[rstest] + #[case("")] + #[case("https://registry.example.com/worker")] + #[case("user@registry.example.com/worker")] + #[case("worker:$(id)")] + #[case("worker;id")] + #[case("worker\n")] + #[case("worker\0")] + #[case("team/UPPER")] + #[case("team//worker")] + #[case("team/../worker")] + #[case("team/foo___bar")] + #[case("team/foo..bar")] + #[case("worker:")] + #[case("worker:.tag")] + #[case("worker:tag:extra")] + #[case("registry.example.com:0/worker")] + #[case("registry.example.com:65536/worker")] + #[case("registry..example.com/worker")] + #[case("registry.example.com:/worker")] + #[case("[::1]extra/worker")] + #[case("[not-ipv6]:5000/worker")] + #[case("[::ffff:192.0.2.1]/worker")] + #[case("[fe80::1%eth0]/worker")] + #[case("worker@sha256:abc")] + fn test_should_reject_invalid_references_at_parse_and_deserialization(#[case] reference: &str) { + assert!(reference.parse::().is_err()); + assert!( + serde_json::from_value::(serde_json::Value::String(reference.into())) + .is_err() + ); + } + + #[test] + fn test_should_enforce_reference_tag_and_repository_byte_limits() -> Result<(), ArtifactError> { + let longest_tag = format!("worker:{}", "t".repeat(128)); + longest_tag.parse::()?; + assert!(format!("{longest_tag}t").parse::().is_err()); + "a".repeat(255).parse::()?; + assert!("a".repeat(256).parse::().is_err()); + assert!("a".repeat(1025).parse::().is_err()); + assert!("wörker".parse::().is_err()); + Ok(()) + } + + #[test] + fn test_should_parse_digest_references_without_confusing_registry_ports_and_tags() + -> Result<(), ArtifactError> { + let digest = format!("sha256:{}", "a".repeat(64)); + for reference in [ + format!("worker@{digest}"), + format!("localhost:5000/team/worker:v1@{digest}"), + format!("[::1]:5000/worker@{digest}"), + ] { + reference.parse::()?; + } + assert!( + format!("worker@{digest}@{digest}") + .parse::() + .is_err() + ); + assert!( + format!("sha256:{}", "A".repeat(64)) + .parse::() + .is_err() + ); + assert!( + format!("sha256:{}", "a".repeat(63)) + .parse::() + .is_err() + ); + assert!( + format!("sha512:{}", "a".repeat(64)) + .parse::() + .is_err() + ); + Ok(()) + } + + #[test] + fn test_should_preserve_local_and_registry_image_identity_through_snapshot_round_trips() + -> Result<(), Box> { + let local = ResolvedImage::fixture(Architecture::Arm64); + assert_eq!( + local.resolved_uri(), + format!("local://sha256:{}", "a".repeat(64)) + ); + let mut metadata = local.metadata().clone(); + let manifest = format!("registry.example.com/worker@sha256:{}", "c".repeat(64)); + metadata.registry_manifest = Some(manifest.parse()?); + let image = ResolvedImage::try_from(metadata)?; + assert_eq!(image.resolved_uri(), manifest); + let saved = serde_json::to_vec(&image)?; + let restored: ResolvedImage = serde_json::from_slice(&saved)?; + assert_eq!(restored.metadata().image_id, image.metadata().image_id); + assert_eq!(restored.metadata().architecture, Architecture::Arm64); + assert_eq!(restored.resolved_uri(), manifest); + Ok(()) + } + + #[test] + fn test_should_revalidate_image_inspection_on_deserialization() + -> Result<(), Box> { + let image = ResolvedImage::fixture(Architecture::X86_64); + let saved = serde_json::to_value(&image)?; + for (field, invalid) in [ + ("os", serde_json::json!("windows")), + ("architecture", serde_json::json!("riscv64")), + ("imageId", serde_json::json!("worker:latest")), + ("registryManifest", serde_json::json!("worker:latest")), + ("sizeBytes", serde_json::json!(0)), + ( + "sizeBytes", + serde_json::json!(10_u64 * 1024 * 1024 * 1024 + 1), + ), + ] { + let mut invalid_image = saved.clone(); + invalid_image + .as_object_mut() + .ok_or("image must be an object")? + .insert(field.into(), invalid); + assert!( + serde_json::from_value::(invalid_image).is_err(), + "{field}" + ); + } + Ok(()) + } + #[rstest::rstest] + #[case("/imageId", serde_json::json!(format!("sha256:{}", "d".repeat(64))))] + #[case("/selectedManifest", serde_json::json!(format!("sha256:{}", "d".repeat(64))))] + #[case("/sizeBytes", serde_json::json!(1025))] + #[case("/architecture", serde_json::json!("arm64"))] + #[case("/execution/command", serde_json::json!(["changed-valid-argv"]))] + #[case("/execution/environment", serde_json::json!(["KEY=changed-valid-value"]))] + #[case("/execution/user", serde_json::json!("1001"))] + #[case("/execution/entrypoint", serde_json::json!(["/other"]))] + #[case("/execution/workingDirectory", serde_json::json!("/other"))] + #[case("/inspectionHash", serde_json::json!(format!("sha256:{}", "f".repeat(64))))] + fn test_should_reject_valid_saved_metadata_mutations_with_a_stale_fingerprint( + #[case] pointer: &str, + #[case] replacement: serde_json::Value, + ) { + let image = ResolvedImage::fixture(Architecture::X86_64); + let mut saved = serde_json::to_value(image).unwrap(); + *saved.pointer_mut(pointer).unwrap() = replacement; + assert!(serde_json::from_value::(saved).is_err()); + } + + #[test] + fn test_should_extract_registry_repository_and_digest_without_confusing_ports() { + for (reference, repository, registry) in [ + ( + "localhost:5000/team/worker:v1", + "localhost:5000/team/worker", + Some("localhost:5000"), + ), + ( + "[::1]:5000/team/worker:v1", + "[::1]:5000/team/worker", + Some("[::1]:5000"), + ), + ("Registry/worker:v1", "Registry/worker", Some("Registry")), + ("team/worker:v1", "team/worker", None), + ("worker:v1", "worker", None), + ] { + let with_digest = format!("{reference}@sha256:{}", "e".repeat(64)); + let image: ImageReference = with_digest.parse().unwrap(); + assert_eq!(image.repository(), repository); + assert_eq!(image.registry(), registry); + assert_eq!( + image.digest(), + Some(format!("sha256:{}", "e".repeat(64)).as_str()) + ); + } + } +} diff --git a/crates/rustack-lambda-core/src/artifact/pinned.rs b/crates/rustack-lambda-core/src/artifact/pinned.rs new file mode 100644 index 00000000..8831b749 --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/pinned.rs @@ -0,0 +1,297 @@ +//! Live image artifacts cannot be reconstructed from snapshot metadata alone. + +use std::{ + fmt, + num::NonZeroU64, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::{OwnedSemaphorePermit, watch}; +use typed_builder::TypedBuilder; + +use super::{ArtifactError, MAX_IMAGE_BYTES, ResolvedImage}; +use crate::docker::DirectoryLease; + +/// Positive daemon-reported unpacked image storage, within the local10GiB quota. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "u64", into = "u64")] +#[non_exhaustive] +pub struct UnpackedImageSize(NonZeroU64); +impl TryFrom for UnpackedImageSize { + type Error = ArtifactError; + fn try_from(value: u64) -> Result { + let value = NonZeroU64::new(value) + .filter(|value| value.get() <= MAX_IMAGE_BYTES) + .ok_or(ArtifactError::Inspection)?; + Ok(Self(value)) + } +} +impl From for u64 { + fn from(size: UnpackedImageSize) -> Self { + size.0.get() + } +} +impl UnpackedImageSize { + /// Number of validated unpacked bytes, independent of compressed size. + /// + /// ``` + /// let size = rustack_lambda_core::artifact::UnpackedImageSize::try_from(4096)?; + /// assert_eq!(size.get(),4096); + /// # Ok::<(),rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } +} + +/// Shared live pin lease; cloning retains availability until joined retirement. +#[derive(Clone)] +#[non_exhaustive] +pub struct PinnedImage(Arc); + +/// Admission retained by the image actor until a consumed handoff or physical retirement. +#[derive(Debug)] +pub(crate) struct PreparationAdmission { + _image: OwnedSemaphorePermit, + _deployment: Option>, +} +impl PreparationAdmission { + pub(crate) fn new( + image: OwnedSemaphorePermit, + deployment: Option>, + ) -> Self { + Self { + _image: image, + _deployment: deployment, + } + } +} +/// Shared acceptance receipt; clearing it releases guards retained above workers. +pub(crate) type PreparationReceipt = watch::Sender>; + +/// Availability view with no pin-release sender or directory/resource ownership. +#[derive(Debug, Clone)] +pub(crate) struct ImageLiveness { + active: Arc, + stop: watch::Receiver, +} +impl ImageLiveness { + pub(crate) fn is_live(&self) -> bool { + self.active.load(Ordering::Acquire) + && !*self.stop.borrow() + && self.stop.has_changed().is_ok() + } + pub(crate) async fn revoked(&self) { + let mut stop = self.stop.clone(); + while self.is_live() { + if stop.changed().await.is_err() { + return; + } + } + } +} + +/// Runtime-owned pins carry the exact acquired directory; fixtures cannot grant it. +#[derive(Debug)] +pub(crate) enum ImageLeaseContext { + #[cfg(test)] + Fixture, + Owned { + directory: DirectoryLease, + preparation: Option, + }, +} + +#[derive(TypedBuilder)] +struct Lease { + resolved: ResolvedImage, + size: UnpackedImageSize, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, + context: ImageLeaseContext, +} +impl Drop for Lease { + fn drop(&mut self) { + self.release.send_replace(()); + } +} +impl fmt::Debug for PinnedImage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("PinnedImage") + .field("image_id", &self.0.resolved.metadata().image_id) + .field("unpacked_bytes", &self.0.size) + .field("live", &self.is_live()) + .finish_non_exhaustive() + } +} +impl PinnedImage { + #[cfg(test)] + pub(crate) fn fixture(resolved: ResolvedImage) -> Self { + let (release, _) = watch::channel(()); + let (_, stop) = watch::channel(false); + let size = UnpackedImageSize::try_from(resolved.metadata().size_bytes) + .expect("validated unit image size"); + Self::new( + resolved, + size, + release, + Arc::new(AtomicBool::new(true)), + stop, + ) + } + + #[cfg(test)] + pub(crate) fn new( + resolved: ResolvedImage, + size: UnpackedImageSize, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, + ) -> Self { + Self::with_context( + resolved, + size, + release, + active, + stop, + ImageLeaseContext::Fixture, + ) + } + + pub(crate) fn with_context( + resolved: ResolvedImage, + size: UnpackedImageSize, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, + context: ImageLeaseContext, + ) -> Self { + Self(Arc::new( + Lease::builder() + .resolved(resolved) + .size(size) + .release(release) + .active(active) + .stop(stop) + .context(context) + .build(), + )) + } + + /// Confirm a validated consumed artifact or a committed deployment CAS. + pub(crate) fn accept_preparation(&self) { + if let ImageLeaseContext::Owned { + preparation: Some(receipt), + .. + } = &self.0.context + { + receipt.send_replace(None); + } + } + + pub(crate) fn liveness(&self) -> ImageLiveness { + ImageLiveness { + active: self.0.active.clone(), + stop: self.0.stop.clone(), + } + } + + pub(crate) fn belongs_to(&self, directory: &DirectoryLease) -> bool { + match &self.0.context { + ImageLeaseContext::Owned { + directory: origin, .. + } => origin.same_instance(directory), + #[cfg(test)] + ImageLeaseContext::Fixture => false, + } + } + + /// Immutable metadata captured by preparation; never resolve a mutable tag. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let id = &image.resolved().metadata().image_id; + /// # } + /// ``` + #[must_use] + pub fn resolved(&self) -> &ResolvedImage { + &self.0.resolved + } + + /// Proven unpacked size used by deployment quota enforcement. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let bytes = image.unpacked_size().get(); + /// # } + /// ``` + #[must_use] + pub fn unpacked_size(&self) -> UnpackedImageSize { + self.0.size + } + + /// Whether the owning manager still permits consumers to use this lease. + /// + /// Shutdown revokes leases after the runtime has joined their consumers. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let available = image.is_live(); + /// # } + /// ``` + #[must_use] + pub fn is_live(&self) -> bool { + self.0.active.load(Ordering::Acquire) && !*self.0.stop.borrow() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::artifact::Architecture; + + #[test] + fn test_should_validate_unpacked_storage_limits_including_deserialization() { + assert!(UnpackedImageSize::try_from(0).is_err()); + assert!(UnpackedImageSize::try_from(MAX_IMAGE_BYTES + 1).is_err()); + assert_eq!( + UnpackedImageSize::try_from(MAX_IMAGE_BYTES).unwrap().get(), + MAX_IMAGE_BYTES + ); + assert!(serde_json::from_str::("0").is_err()); + let size: UnpackedImageSize = serde_json::from_str("4096").unwrap(); + assert_eq!(size.get(), 4096); + } + + #[tokio::test] + async fn test_should_keep_release_channel_until_last_clone_and_revoke_on_stop() { + let (release, mut released) = watch::channel(()); + let (stop, stopping) = watch::channel(false); + let active = Arc::new(AtomicBool::new(true)); + let image = PinnedImage::new( + ResolvedImage::fixture(Architecture::Arm64), + 4096.try_into().unwrap(), + release, + active.clone(), + stopping, + ); + let captured = image.clone(); + drop(image); + assert!(!released.has_changed().unwrap()); + assert!(captured.is_live()); + assert_eq!(captured.unpacked_size().get(), 4096); + assert!(!format!("{captured:?}").contains("entrypoint")); + stop.send_replace(true); + assert!(!captured.is_live()); + drop(captured); + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/config.rs b/crates/rustack-lambda-core/src/config.rs index ed8fbf39..939ba6ba 100644 --- a/crates/rustack-lambda-core/src/config.rs +++ b/crates/rustack-lambda-core/src/config.rs @@ -42,7 +42,7 @@ impl LambdaConfig { /// - `GATEWAY_HOST` (default: `localhost`) /// - `GATEWAY_PORT` (default: `4566`) /// - `LAMBDA_DOCKER_ENABLED` (default: `false` — legacy alias for `LAMBDA_EXECUTOR=docker`). - /// - `LAMBDA_EXECUTOR` (default: `disabled`; unsupported Docker settings fail explicitly). + /// - `LAMBDA_EXECUTOR` (default: `disabled`; Docker requires qualified runtime installation). /// Accepts `disabled`, `auto`, `native`, `docker`, `squib`. Auto uses Squib for Zip functions /// on macOS and native execution otherwise. The native backend runs `provided.*` bootstraps /// (Rust / Go / C++) directly on the host with no Docker requirement. @@ -50,6 +50,11 @@ impl LambdaConfig { /// - `LAMBDA_IDLE_TIMEOUT_SECS` (default: `600`) /// - `LAMBDA_INIT_TIMEOUT_SECS` (default: `5`) /// - `LAMBDA_SQUIB_*` variables documented by [`SquibExecutorConfig`]. + /// + /// # Errors + /// + /// Rejects malformed environment values, invalid budgets, + /// conflicting executor settings or invalid Squib configuration. pub fn from_env() -> Result { Self::from_lookup(|key| match rustack_core::settings::var(key) { Ok(value) => Some(value), @@ -59,6 +64,11 @@ impl LambdaConfig { } /// Parse an explicit environment lookup without changing process globals. + /// + /// # Errors + /// + /// Rejects malformed scalar values, invalid budgets/identifiers, conflicting executor + /// settings or invalid Squib configuration. pub fn from_lookup( lookup: impl Fn(&str) -> Option, ) -> Result { @@ -94,9 +104,6 @@ impl LambdaConfig { if docker_enabled && executor != ExecutorBackend::Docker { return Err(error("conflicting Docker/executor settings")); } - if executor == ExecutorBackend::Docker { - return Err(error("Docker execution is not supported by this build")); - } let mut config = Self { executor, docker_enabled, @@ -196,6 +203,21 @@ mod tests { ); } + #[test] + fn test_should_parse_explicit_docker_without_granting_unqualified_execution() { + for (key, value) in [ + ("LAMBDA_EXECUTOR", "docker"), + ("LAMBDA_DOCKER_ENABLED", "true"), + ] { + let config = + LambdaConfig::from_lookup(|name| (name == key).then(|| value.into())).unwrap(); + assert_eq!(config.executor, ExecutorBackend::Docker); + let provider = crate::provider::RustackLambda::new(config); + assert!(!provider.is_ready()); + assert!(provider.executor().image_available().is_err()); + } + } + #[test] fn test_should_create_default_config() { let config = LambdaConfig::default(); diff --git a/crates/rustack-lambda-core/src/deployment.rs b/crates/rustack-lambda-core/src/deployment.rs new file mode 100644 index 00000000..7cefbee7 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment.rs @@ -0,0 +1,555 @@ +//! Typed deployment transitions and stale-result protection at the store boundary. +//! +//! A retained worker owns each candidate until settlement. Store keys bind resource +//! lifetime, original revision and deployment generation; metadata is never a live lease. + +#[cfg(unix)] +mod manager; +use std::fmt; + +#[cfg(unix)] +pub use manager::{ImageDeploymentConfig, ImageDeploymentManager, ImageDeploymentStatus}; +use rustack_lambda_model::types::ImageConfig; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::{ + artifact::{Architecture, ImageArtifact, ImageReference, PinnedImage}, + error::LambdaServiceError, + identity::FunctionResourceId, + storage::{FunctionRecord, FunctionStore, VersionRecord}, +}; + +/// A nonnil identity for one accepted deployment attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct DeploymentGeneration(Uuid); + +impl TryFrom for DeploymentGeneration { + type Error = LambdaServiceError; + fn try_from(value: Uuid) -> Result { + if value.is_nil() { + return Err(invalid("Deployment generation cannot be nil")); + } + Ok(Self(value)) + } +} +impl TryFrom for DeploymentGeneration { + type Error = LambdaServiceError; + fn try_from(value: String) -> Result { + if value.len() != 36 { + return Err(invalid("Invalid deployment generation")); + } + let id = Uuid::parse_str(&value).map_err(|_| invalid("Invalid deployment generation"))?; + Self::try_from(id) + } +} +impl From for String { + fn from(value: DeploymentGeneration) -> Self { + value.0.to_string() + } +} + +/// Bounded public failure vocabulary; no backend response or secret is retained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub enum ImageDeploymentFailure { + /// The requested content is unavailable. + ImageNotFound, + /// Image metadata or its effective entry configuration is invalid. + InvalidImage, + /// The selected architecture cannot run under the configured policy. + UnsupportedPlatform, + /// Docker is unavailable or its ownership health is degraded. + RuntimeUnavailable, + /// Deployment exceeded its absolute deadline. + DeadlineExceeded, + /// Deployment was cancelled before settlement. + Cancelled, + /// Its retained supervisor failed. + SupervisorFailed, +} +impl ImageDeploymentFailure { + /// Static, bounded API reason with no untrusted backend text. + /// + /// ``` + /// use rustack_lambda_core::deployment::ImageDeploymentFailure; + /// assert!(ImageDeploymentFailure::InvalidImage.reason().len() < 256); + /// ``` + #[must_use] + pub const fn reason(self) -> &'static str { + match self { + Self::ImageNotFound => "Requested image content is unavailable", + Self::InvalidImage => "Image metadata or entry configuration is invalid", + Self::UnsupportedPlatform => "Image platform is unsupported by this runtime", + Self::RuntimeUnavailable => "Image runtime is unavailable", + Self::DeadlineExceeded => "Image deployment deadline exceeded", + Self::Cancelled => "Image deployment was cancelled", + Self::SupervisorFailed => "Image deployment supervisor failed", + } + } + + /// AWS-compatible reason category for the local failure. + /// + /// ``` + /// use rustack_lambda_core::deployment::ImageDeploymentFailure; + /// assert_eq!(ImageDeploymentFailure::InvalidImage.code(), "InvalidImage"); + /// ``` + #[must_use] + pub const fn code(self) -> &'static str { + match self { + Self::ImageNotFound => "ImageDeleted", + Self::InvalidImage | Self::UnsupportedPlatform => "InvalidImage", + Self::RuntimeUnavailable + | Self::DeadlineExceeded + | Self::Cancelled + | Self::SupervisorFailed => "InternalError", + } + } +} + +/// One authoritative state for both function readiness and update progress. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + content = "detail", + rename_all = "camelCase", + deny_unknown_fields +)] +#[non_exhaustive] +pub enum DeploymentState { + /// Usable revision with no failed or pending update. + #[default] + Ready, + /// Initial deployment; no previous executable revision exists. + Creating(DeploymentGeneration), + /// Initial deployment failed. + CreateFailed(ImageDeploymentFailure), + /// Previous revision remains usable while a candidate prepares. + Updating(DeploymentGeneration), + /// Previous revision remains usable after a failed update. + UpdateFailed(ImageDeploymentFailure), +} +impl DeploymentState { + /// Public Lambda function state. + /// + /// ``` + /// assert_eq!(rustack_lambda_core::deployment::DeploymentState::Ready.state(), "Active"); + /// ``` + #[must_use] + pub const fn state(self) -> &'static str { + match self { + Self::Ready | Self::Updating(_) | Self::UpdateFailed(_) => "Active", + Self::Creating(_) => "Pending", + Self::CreateFailed(_) => "Failed", + } + } + + /// Public Lambda update status. + /// + /// ``` + /// assert_eq!(rustack_lambda_core::deployment::DeploymentState::Ready.update_status(), "Successful"); + /// ``` + #[must_use] + pub const fn update_status(self) -> &'static str { + match self { + Self::Ready => "Successful", + Self::Creating(_) | Self::Updating(_) => "InProgress", + Self::CreateFailed(_) | Self::UpdateFailed(_) => "Failed", + } + } + + /// Public failure reason, if a deployment failed. + /// + /// ``` + /// assert!(rustack_lambda_core::deployment::DeploymentState::Ready.failure().is_none()); + /// ``` + #[must_use] + pub const fn failure(self) -> Option { + match self { + Self::CreateFailed(reason) | Self::UpdateFailed(reason) => Some(reason), + _ => None, + } + } + + /// `StateReason` describes initial readiness failure, not a failed update. + pub(crate) const fn creation_failure(self) -> Option { + match self { + Self::CreateFailed(failure) => Some(failure), + _ => None, + } + } + + pub(crate) fn ensure_mutable(self) -> Result<(), LambdaServiceError> { + if matches!(self, Self::Creating(_) | Self::Updating(_)) { + return Err(conflict("A function deployment is already in progress")); + } + Ok(()) + } + + pub(crate) fn restore(legacy: &str, saved: Option) -> Result { + let state = match saved { + Some(state) if state.state() == legacy => state, + Some(_) => return Err(invalid("Snapshot deployment states contradict each other")), + None => match legacy { + "Active" => Self::Ready, + "Pending" => Self::CreateFailed(ImageDeploymentFailure::Cancelled), + "Failed" => Self::CreateFailed(ImageDeploymentFailure::InvalidImage), + _ => return Err(invalid("Unsupported snapshot function state")), + }, + }; + Ok(match state { + Self::Creating(_) => Self::CreateFailed(ImageDeploymentFailure::Cancelled), + Self::Updating(_) => Self::UpdateFailed(ImageDeploymentFailure::Cancelled), + state => state, + }) + } +} + +/// Closed candidate issued by a store transaction, owned by a deployment worker. +/// +/// Debug omits its image reference, entry configuration and environment. +/// +/// Construction is restricted to validated store transactions: +/// +/// ```compile_fail +/// use rustack_lambda_core::deployment::ImageDeployment; +/// let forged = ImageDeployment::builder(); +/// ``` +#[derive(Clone, TypedBuilder)] +#[builder( + builder_method(vis = ""), + build_method(vis = ""), + builder_type(vis = "") +)] +#[non_exhaustive] +pub struct ImageDeployment { + name: String, + resource: FunctionResourceId, + original_revision: String, + generation: DeploymentGeneration, + initial: bool, + candidate: VersionRecord, + architecture: Architecture, + reference: ImageReference, +} +impl fmt::Debug for ImageDeployment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ImageDeployment") + .field("resource", &self.resource) + .field("generation", &self.generation) + .field("initial", &self.initial) + .finish_non_exhaustive() + } +} +impl ImageDeployment { + fn new( + record: &FunctionRecord, + candidate: VersionRecord, + initial: bool, + ) -> Result { + crate::resolver::FunctionName::parse(&record.name)?; + if record.latest.revision_id.len() != 36 + || Uuid::parse_str(&record.latest.revision_id).is_err() + { + return Err(invalid("Invalid deployment revision identity")); + } + let architecture = crate::function_config::validate_execution_settings(&candidate)?; + let reference = candidate + .image_uri + .as_deref() + .ok_or_else(|| invalid("Image reference is required"))? + .parse() + .map_err(|_| invalid("Invalid image reference"))?; + Ok(Self::builder() + .name(record.name.clone()) + .resource(record.resource_id) + .original_revision(record.latest.revision_id.clone()) + .generation(DeploymentGeneration(Uuid::new_v4())) + .initial(initial) + .candidate(candidate) + .architecture(architecture) + .reference(reference) + .build()) + } + + /// Reference captured at acceptance for bounded image preparation. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let reference = job.reference(); + /// # } + /// ``` + #[must_use] + pub fn reference(&self) -> &ImageReference { + &self.reference + } + + /// Validated requested architecture for this candidate. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let architecture = job.architecture(); + /// # } + /// ``` + #[must_use] + pub const fn architecture(&self) -> Architecture { + self.architecture + } + + /// Complete entry overrides; omitted and empty arrays remain distinct. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let overrides = job.image_config(); + /// # } + /// ``` + #[must_use] + pub fn image_config(&self) -> Option<&ImageConfig> { + self.candidate.image_config.as_ref() + } + + #[cfg(unix)] + pub(crate) fn is_current(&self, store: &FunctionStore) -> Result { + Ok(store + .get(&self.name)? + .as_ref() + .is_some_and(|record| self.matches(record))) + } + + fn matches(&self, record: &FunctionRecord) -> bool { + let state = if self.initial { + DeploymentState::Creating(self.generation) + } else { + DeploymentState::Updating(self.generation) + }; + record.resource_id == self.resource + && record.latest.revision_id == self.original_revision + && record.latest.deployment == state + } +} + +impl FunctionStore { + /// Insert an initial Pending image deployment after the caller reserves work. + /// + /// # Errors + /// Rejects duplicate/invalid records and non-image or already resolved candidates. + /// + /// ```no_run + /// # fn accept(store: &rustack_lambda_core::storage::FunctionStore, record: rustack_lambda_core::storage::FunctionRecord) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let candidate = store.insert_image_deployment(record)?; + /// # Ok(()) } + /// ``` + pub fn insert_image_deployment( + &self, + record: FunctionRecord, + ) -> Result { + self.reserve_image_create(record).map(|(job, _)| job) + } + + pub(crate) fn reserve_image_create( + &self, + mut record: FunctionRecord, + ) -> Result<(ImageDeployment, FunctionRecord), LambdaServiceError> { + if record.latest.package_type != "Image" + || !matches!(record.latest.image, ImageArtifact::Unresolved) + || record.latest.deployment != DeploymentState::Ready + || !record.versions.is_empty() + { + return Err(invalid( + "Initial image candidate must be unresolved and unpublished", + )); + } + record.latest.code_sha256.clear(); + record.latest.code_size = 0; + let job = ImageDeployment::new(&record, record.latest.clone(), true)?; + record.latest.deployment = DeploymentState::Creating(job.generation); + self.insert(record.clone())?; + Ok((job, record)) + } + + /// Reserve an image update while preserving the previous usable revision. + /// + /// # Errors + /// Rejects stale resource/revision keys, pending work, package changes or invalid settings. + /// + /// ```no_run + /// # fn accept(store: &rustack_lambda_core::storage::FunctionStore, record: &rustack_lambda_core::storage::FunctionRecord) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let job = store.begin_image_update(&record.name, record.resource_id, + /// &record.latest.revision_id, "worker:v2".parse().map_err(|_| rustack_lambda_core::error::LambdaServiceError::DockerNotAvailable)?, + /// rustack_lambda_core::artifact::Architecture::Arm64)?; + /// # Ok(()) } + /// ``` + pub fn begin_image_update( + &self, + name: &str, + resource: FunctionResourceId, + expected: &str, + reference: ImageReference, + architecture: Architecture, + ) -> Result { + self.reserve_image_update(name, resource, expected, reference, architecture, None) + .map(|(job, _)| job) + } + + pub(crate) fn reserve_image_update( + &self, + name: &str, + resource: FunctionResourceId, + expected: &str, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result<(ImageDeployment, FunctionRecord), LambdaServiceError> { + self.try_update_if_revision(name, None, |record| { + check_revision(requested, &record.latest.revision_id)?; + if record.resource_id != resource || record.latest.revision_id != expected { + return Err(conflict("Function resource changed")); + } + record.latest.deployment.ensure_mutable()?; + if record.latest.package_type != "Image" { + return Err(invalid("Deployment package type cannot change")); + } + let mut candidate = record.latest.clone(); + candidate.image_uri = Some(reference.into()); + candidate.architectures = vec![architecture.as_str().into()]; + candidate.image = ImageArtifact::Unresolved; + candidate.code_sha256.clear(); + candidate.code_size = 0; + candidate.deployment = DeploymentState::Ready; + let initial = matches!(record.latest.deployment, DeploymentState::CreateFailed(_)); + let job = ImageDeployment::new(record, candidate.clone(), initial)?; + if initial { + record.latest = candidate; + } + record.latest.deployment = if initial { + DeploymentState::Creating(job.generation) + } else { + DeploymentState::Updating(job.generation) + }; + Ok((job, record.clone())) + }) + } + + /// Atomically settle a candidate only while all accepted identity keys match. + /// + /// Stale/duplicate/deleted results return false and cannot mutate live metadata. + /// The caller must retain and join the worker independently of this transaction. + /// + /// # Errors + /// Rejects a revoked/mismatching lease or an invalid candidate without committing it. + /// + /// ```no_run + /// # fn finish(store: &rustack_lambda_core::storage::FunctionStore, job: &rustack_lambda_core::deployment::ImageDeployment) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// store.settle_image_deployment(job, Err(rustack_lambda_core::deployment::ImageDeploymentFailure::Cancelled))?; + /// # Ok(()) } + /// ``` + pub fn settle_image_deployment( + &self, + job: &ImageDeployment, + outcome: Result, + ) -> Result { + let Some(before) = self.get(&job.name)? else { + return Ok(false); + }; + if !job.matches(&before) { + return Ok(false); + } + let receipt = outcome.as_ref().ok().cloned(); + let prepared = match outcome { + Ok(image) => { + if !image.is_live() + || image.resolved().metadata().requested_reference != job.reference + || image.resolved().metadata().architecture != job.architecture + { + return Err(invalid( + "Prepared image does not match deployment candidate", + )); + } + image + .resolved() + .metadata() + .execution + .plan(job.image_config()) + .map_err(|_| invalid("Invalid effective image entry configuration"))?; + let mut candidate = job.candidate.clone(); + candidate.code_sha256 = image + .resolved() + .code_sha256() + .map_err(|_| invalid("Invalid image fingerprint"))?; + candidate.code_size = image.unpacked_size().get(); + candidate.image = ImageArtifact::Pinned(image); + candidate.deployment = DeploymentState::Ready; + candidate.revision_id = Uuid::new_v4().to_string(); + candidate.last_modified = crate::provider::now_iso8601(); + Ok(candidate) + } + Err(failure) => Err(failure), + }; + let committed = match self.update(&job.name, |record| { + if !job.matches(record) { + return false; + } + match prepared { + Ok(candidate) => record.latest = candidate, + Err(failure) => { + record.latest.deployment = if job.initial { + DeploymentState::CreateFailed(failure) + } else { + DeploymentState::UpdateFailed(failure) + } + } + } + true + }) { + Err(LambdaServiceError::FunctionNotFound { .. }) => Ok(false), + result => result, + }?; + if committed { + if let Some(image) = receipt { + image.accept_preparation(); + } + } + Ok(committed) + } +} + +pub(crate) fn check_revision( + requested: Option<&str>, + actual: &str, +) -> Result<(), LambdaServiceError> { + if let Some(requested) = requested { + if requested.is_empty() + || requested.len() > 100 + || !requested + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + { + return Err(invalid("Invalid RevisionId")); + } + if requested != actual { + return Err(LambdaServiceError::PreconditionFailed); + } + } + Ok(()) +} + +fn invalid(message: &str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} +fn conflict(message: &str) -> LambdaServiceError { + LambdaServiceError::ResourceConflict { + message: message.into(), + } +} + +#[cfg(test)] +#[path = "deployment_tests.rs"] +mod tests; +#[cfg(test)] +pub(crate) use tests::record as image_record_fixture; diff --git a/crates/rustack-lambda-core/src/deployment/manager.rs b/crates/rustack-lambda-core/src/deployment/manager.rs new file mode 100644 index 00000000..a250ff8f --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment/manager.rs @@ -0,0 +1,709 @@ +//! Bounded deployment admission, retained child joins and keyed failure recovery. +//! +//! Owner Drop intentionally leaves its signalled root running on the captured +//! runtime to drain. The root owns child joins; abnormal root Drop transfers +//! them and their reservations to retained emergency cleanup. Joined shutdown, +//! not diagnostic counts, proves this owner's settlement. The separate image +//! owner still owns critical mutations after a preparation future is cancelled. + +use std::{ + collections::HashMap, + future::Future, + mem, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use http::StatusCode; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, watch}, + task::{Id, JoinError, JoinSet}, + time::{Instant, sleep_until, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ImageDeployment, ImageDeploymentFailure}; +use crate::{ + artifact::{Architecture, ImageReference, PinnedImage}, + docker::{DockerError, ImageLeasePort}, + error::LambdaServiceError, + executor::ExecutorError, + invocation::CleanupTask, + provider::DockerArchiveProfile, + storage::{FunctionRecord, FunctionStore, FunctionStoreSnapshot, PreparedImport, image_error}, +}; + +/// Validated accepted-deployment capacity and absolute preparation budget. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ImageDeploymentConfig { + jobs: u8, + budget: Duration, +} +impl Default for ImageDeploymentConfig { + fn default() -> Self { + Self { + jobs: 8, + budget: Duration::from_secs(300), + } + } +} +impl ImageDeploymentConfig { + /// Validate one bounded deployment owner; capacity includes queued and running jobs. + /// + /// # Errors + /// Rejects capacity outside1..32 or a budget outside1..3,600 seconds. + /// + /// ``` + /// let config = rustack_lambda_core::deployment::ImageDeploymentConfig::new( + /// 8, std::time::Duration::from_secs(300))?; + /// # Ok::<(),rustack_lambda_core::error::LambdaServiceError>(()) + /// ``` + pub fn new(jobs: u8, budget: Duration) -> Result { + if !(1..=32).contains(&jobs) + || budget < Duration::from_secs(1) + || budget > Duration::from_secs(3600) + { + return Err(invalid("Invalid deployment capacity or deadline")); + } + Ok(Self { jobs, budget }) + } +} + +/// Diagnostic counts; use joined quiescence/shutdown for settlement evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ImageDeploymentStatus { + /// Queued/running jobs whose capacity has not yet been reaped. + pub accepted: usize, + /// Whether both owners currently permit new preparation. + pub accepting: bool, + /// A deployment supervisor failure permanently closed this owner. + pub failed: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Mode { + Running, + Drain, + Cancel, +} +#[derive(Debug, Default)] +struct Health { + failed: AtomicBool, + #[cfg(test)] + crash_root: AtomicBool, + #[cfg(test)] + crash_worker: AtomicBool, +} +fn signal(control: &watch::Sender, mode: Mode) { + control.send_if_modified(|current| { + if *current < mode { + *current = mode; + true + } else { + false + } + }); +} +fn unavailable() -> LambdaServiceError { + LambdaServiceError::ResourceNotReady { + message: "Image deployment owner is unavailable".into(), + } +} +fn invalid(message: &str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} +fn failed() -> ExecutorError { + ExecutorError::CleanupFailed("Image deployment supervisor failed".into()) +} + +#[derive(Debug, TypedBuilder)] +struct Pending { + job: ImageDeployment, + store: Arc, + health: Arc, + control: watch::Sender, + #[builder(default)] + settled: AtomicBool, +} +impl Pending { + fn finish( + &self, + outcome: Result, + ) -> Result<(), LambdaServiceError> { + if !self.settled.load(Ordering::Acquire) { + self.store.settle_image_deployment(&self.job, outcome)?; + self.settled.store(true, Ordering::Release); + } + Ok(()) + } + fn fail_owner(&self) { + self.health.failed.store(true, Ordering::Release); + signal(&self.control, Mode::Cancel); + } +} +impl Drop for Pending { + fn drop(&mut self) { + if self.settled.load(Ordering::Acquire) { + return; + } + let reason = if self.health.failed.load(Ordering::Acquire) + || *self.control.borrow() != Mode::Cancel + { + self.fail_owner(); + ImageDeploymentFailure::SupervisorFailed + } else { + ImageDeploymentFailure::Cancelled + }; + if self.finish(Err(reason)).is_err() { + self.fail_owner(); + tracing::error!("Abandoned image deployment could not be settled"); + } + } +} + +#[derive(Debug)] +struct Envelope { + id: Uuid, + pending: Arc, + slot: OwnedSemaphorePermit, + deadline: Instant, +} +#[derive(Debug)] +struct Row { + pending: Arc, + _slot: Arc, +} +#[derive(Debug)] +struct Reservation { + queue: mpsc::OwnedPermit, + slot: OwnedSemaphorePermit, + deadline: Instant, +} + +/// Own accepted image deployments through actual worker joins and keyed settlement. +#[derive(Debug, TypedBuilder)] +#[builder( + builder_method(vis = ""), + builder_type(vis = ""), + build_method(vis = "") +)] +#[non_exhaustive] +pub struct ImageDeploymentManager { + store: Arc, + preparation: ImageLeasePort, + config: ImageDeploymentConfig, + capacity: Arc, + sender: mpsc::Sender, + control: watch::Sender, + health: Arc, + task: CleanupTask, + emergency: Arc, +} +impl ImageDeploymentManager { + /// Start a retained owner using an already qualified image preparation port. + /// + /// # Errors + /// Rejects unavailable preparation, excess capacity or lack of an entered runtime. + /// This does not probe or authorize the Docker execution backend. + /// + /// ```no_run + /// # fn start(store:std::sync::Arc, port:rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let owner = rustack_lambda_core::deployment::ImageDeploymentManager::start(store, port, + /// rustack_lambda_core::deployment::ImageDeploymentConfig::default())?; + /// # Ok(()) } + /// ``` + pub fn start( + store: Arc, + preparation: ImageLeasePort, + config: ImageDeploymentConfig, + ) -> Result { + preparation + .available() + .map_err(|_| LambdaServiceError::DockerNotAvailable)?; + if config.jobs > preparation.preparation_jobs() { + return Err(invalid("Deployment jobs exceed image preparation capacity")); + } + let runtime = Handle::try_current().map_err(|_| unavailable())?; + let (sender, inbox) = mpsc::channel(usize::from(config.jobs)); + let (control, _) = watch::channel(Mode::Running); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let actor = Actor::builder() + .inbox(inbox) + .control(control.clone()) + .preparation(preparation.clone()) + .health(health.clone()) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + let task = CleanupTask::default(); + task.start_on(&runtime, move || actor.run()); + Ok(Self::builder() + .store(store) + .preparation(preparation) + .config(config) + .capacity(Arc::new(Semaphore::new(usize::from(config.jobs)))) + .sender(sender) + .control(control) + .health(health) + .task(task) + .emergency(emergency) + .build()) + } + + fn available(&self) -> Result<(), LambdaServiceError> { + if *self.control.borrow() != Mode::Running || self.health.failed.load(Ordering::Acquire) { + return Err(unavailable()); + } + self.preparation + .available() + .map_err(|_| LambdaServiceError::DockerNotAvailable) + } + fn reserve(&self) -> Result { + self.available()?; + let deadline = Instant::now() + .checked_add(self.config.budget) + .ok_or_else(unavailable)?; + let slot = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| LambdaServiceError::TooManyRequests)?; + let queue = self + .sender + .clone() + .try_reserve_owned() + .map_err(|_| unavailable())?; + self.available()?; + Ok(Reservation { + queue, + slot, + deadline, + }) + } + fn accept( + &self, + reservation: Reservation, + job: ImageDeployment, + accepted: FunctionRecord, + ) -> FunctionRecord { + let pending = Arc::new( + Pending::builder() + .job(job) + .store(self.store.clone()) + .health(self.health.clone()) + .control(self.control.clone()) + .build(), + ); + reservation.queue.send(Envelope { + id: Uuid::new_v4(), + pending, + slot: reservation.slot, + deadline: reservation.deadline, + }); + accepted + } + + /// Reserve work before inserting an initial Pending image function. + /// + /// # Errors + /// Rejects invalid/duplicate resources, unavailable owners or exhausted capacity. + /// The returned snapshot is captured before enqueue; dropping it does not cancel accepted work. + /// + /// ```no_run + /// # fn create(owner:&rustack_lambda_core::deployment::ImageDeploymentManager, record:rustack_lambda_core::storage::FunctionRecord) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let pending = owner.create(record)?; + /// # Ok(()) } + /// ``` + pub fn create(&self, record: FunctionRecord) -> Result { + let reservation = self.reserve()?; + let (job, accepted) = self.store.reserve_image_create(record)?; + Ok(self.accept(reservation, job, accepted)) + } + + /// Reserve an update while preserving previous code and configuration. + /// + /// # Errors + /// Rejects stale resources/revisions, invalid settings, pending work or unavailable capacity. + /// + /// ```no_run + /// # fn update(owner:&rustack_lambda_core::deployment::ImageDeploymentManager, before:&rustack_lambda_core::storage::FunctionRecord) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let reference = "worker:v2".parse().map_err(|_| rustack_lambda_core::error::LambdaServiceError::DockerNotAvailable)?; + /// owner.update(before, reference, rustack_lambda_core::artifact::Architecture::Arm64)?; + /// # Ok(()) } + /// ``` + pub fn update( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + ) -> Result { + self.update_with_revision(before, reference, architecture, None) + } + + pub(crate) fn update_with_revision( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result { + super::check_revision(requested, &before.latest.revision_id)?; + let reservation = self.reserve()?; + let (job, accepted) = self.store.reserve_image_update( + &before.name, + before.resource_id, + &before.latest.revision_id, + reference, + architecture, + requested, + )?; + Ok(self.accept(reservation, job, accepted)) + } + + /// Observe bounded admission; counts alone never prove clean settlement. + /// + /// ```no_run + /// # fn inspect(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let count = owner.status().accepted; + /// # } + /// ``` + #[must_use] + pub fn status(&self) -> ImageDeploymentStatus { + ImageDeploymentStatus { + accepted: usize::from(self.config.jobs) + .saturating_sub(self.capacity.available_permits()), + accepting: self.available().is_ok(), + failed: self.health.failed.load(Ordering::Acquire), + } + } + + pub(crate) fn cancel(&self) { + signal(&self.control, Mode::Cancel); + } + + async fn join(&self, deadline: Instant) -> Result<(), ExecutorError> { + if Instant::now() >= deadline { + return Err(ExecutorError::DeadlineExceeded); + } + let result = timeout_at(deadline, async { + let primary = self.task.wait().await; + let emergency = self.emergency.wait().await; + primary.and(emergency) + }) + .await + .map_err(|_| ExecutorError::DeadlineExceeded)?; + if Instant::now() >= deadline { + return Err(ExecutorError::DeadlineExceeded); + } + result + } + pub(crate) async fn prepare_restore( + &self, + snapshot: FunctionStoreSnapshot, + deadline: Instant, + profile: DockerArchiveProfile, + ) -> Result, LambdaServiceError> { + self.restore_available()?; + self.store + .prepare_docker_snapshot(snapshot, &self.preparation, deadline, profile) + .await + } + + pub(crate) fn restore_available(&self) -> Result<(), LambdaServiceError> { + let status = self.status(); + if !status.accepting || status.failed || status.accepted != 0 { + return Err(invalid( + "Image restore requires an idle healthy deployment owner", + )); + } + self.preparation.available().map_err(image_error) + } + + /// Close admission and drain accepted workers without cancelling preparation. + /// + /// # Errors + /// Returns deadline or cached supervision failure while retaining joins. + /// The separate image owner must also join its critical mutations and cleanup. + /// + /// ```no_run + /// # async fn stop(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let result = owner.quiesce(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn quiesce(&self, deadline: Instant) -> Result<(), ExecutorError> { + signal(&self.control, Mode::Drain); + self.join(deadline).await + } + + /// Cancel accepted preparations, settle records and join this owner's workers. + /// + /// # Errors + /// Returns deadline or cached supervision failure; cancelled waiters retain joins. + /// + /// ```no_run + /// # async fn stop(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let result = owner.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn shutdown(&self, deadline: Instant) -> Result<(), ExecutorError> { + signal(&self.control, Mode::Cancel); + self.join(deadline).await + } +} +impl Drop for ImageDeploymentManager { + fn drop(&mut self) { + signal(&self.control, Mode::Cancel); + } +} + +#[derive(Debug, TypedBuilder)] +struct Actor { + inbox: mpsc::Receiver, + preparation: ImageLeasePort, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default)] + jobs: JoinSet>, + #[builder(default)] + rows: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + fn fail(&self) { + self.health.failed.store(true, Ordering::Release); + signal(&self.control, Mode::Cancel); + } + async fn run(mut self) -> Result<(), ExecutorError> { + let mut control = self.control.subscribe(); + let mut closed = false; + loop { + if *control.borrow() != Mode::Running { + self.inbox.close(); + } + if closed && self.jobs.is_empty() { + break; + } + tokio::select! { + biased; + _ = control.changed() => {}, + joined = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => if let Some(joined) = joined { self.reap(&joined); }, + envelope = self.inbox.recv(), if !closed => if let Some(envelope) = envelope { + if *control.borrow() == Mode::Cancel { + if envelope.pending.finish(Err(ImageDeploymentFailure::Cancelled)).is_err() { self.fail(); } + } else { self.start(envelope); } + } else { + closed = true; + if *control.borrow() == Mode::Running { signal(&self.control, Mode::Cancel); } + }, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(failed()) + } else { + Ok(()) + } + } + fn start(&mut self, envelope: Envelope) { + #[cfg(test)] + assert!( + !self.health.crash_root.swap(false, Ordering::AcqRel), + "injected deployment root failure" + ); + let Envelope { + id, + pending, + slot, + deadline, + } = envelope; + let slot = Arc::new(slot); + self.rows.insert( + id, + Row { + pending: pending.clone(), + _slot: slot.clone(), + }, + ); + let preparation = self.preparation.clone(); + let control = self.control.subscribe(); + #[cfg(test)] + let health = self.health.clone(); + let worker = self.jobs.spawn(async move { + #[cfg(test)] + assert!( + !health.crash_worker.swap(false, Ordering::AcqRel), + "injected deployment worker failure" + ); + let outcome = prepare(&pending, preparation, control, deadline, slot).await; + let outcome = if pending.health.failed.load(Ordering::Acquire) { + Err(ImageDeploymentFailure::SupervisorFailed) + } else { + outcome + }; + if pending.finish(outcome).is_err() + && pending + .finish(Err(ImageDeploymentFailure::InvalidImage)) + .is_err() + { + return Err(failed()); + } + Ok(id) + }); + self.index.insert(worker.id(), id); + } + fn reap(&mut self, joined: &Result<(Id, Result), JoinError>) { + let task = match joined { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(id) = self.index.remove(&task) else { + self.fail(); + return; + }; + let Some(row) = self.rows.remove(&id) else { + self.fail(); + return; + }; + if !matches!(joined, Ok((_, Ok(returned))) if *returned == id) + || !row.pending.settled.load(Ordering::Acquire) + { + self.fail(); + if row + .pending + .finish(Err(ImageDeploymentFailure::SupervisorFailed)) + .is_err() + { + tracing::error!("Failed image worker could not be settled"); + } + } + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + if self.rows.is_empty() { + return; + } + self.fail(); + let rows = mem::take(&mut self.rows); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + let mut clean = true; + for row in rows.into_values() { + clean &= row + .pending + .finish(Err(ImageDeploymentFailure::SupervisorFailed)) + .is_ok(); + } + if clean { Ok(()) } else { Err(failed()) } + }); + } +} + +async fn prepare( + pending: &Pending, + preparation: ImageLeasePort, + mut control: watch::Receiver, + deadline: Instant, + admission: Arc, +) -> Result { + let mut changes = pending.store.changes(); + let preparation = preparation.prepare_deployment( + pending.job.reference(), + pending.job.architecture(), + pending.job.image_config(), + deadline, + admission, + ); + tokio::pin!(preparation); + let result = loop { + check(pending, &control, deadline)?; + match next(preparation.as_mut(), &mut control, &mut changes, deadline).await? { + Progress::Prepared(result) => break result.map_err(|error| map_failure(&error)), + Progress::Changed => {} + } + }; + check(pending, &control, deadline)?; + result +} +#[derive(Debug)] +enum Progress { + Prepared(Result), + Changed, +} +async fn next( + preparation: Pin<&mut impl Future>>, + control: &mut watch::Receiver, + changes: &mut watch::Receiver<()>, + deadline: Instant, +) -> Result { + tokio::select! { + biased; + () = sleep_until(deadline) => Err(ImageDeploymentFailure::DeadlineExceeded), + _ = control.wait_for(|mode| *mode == Mode::Cancel) => Err(ImageDeploymentFailure::Cancelled), + result = preparation => Ok(Progress::Prepared(result)), + notice = changes.changed() => notice.map(|()| Progress::Changed).map_err(|_| ImageDeploymentFailure::SupervisorFailed), + } +} + +fn check( + pending: &Pending, + control: &watch::Receiver, + deadline: Instant, +) -> Result<(), ImageDeploymentFailure> { + if Instant::now() >= deadline { + return Err(ImageDeploymentFailure::DeadlineExceeded); + } + if *control.borrow() == Mode::Cancel { + return Err(ImageDeploymentFailure::Cancelled); + } + if !pending + .job + .is_current(&pending.store) + .map_err(|_| ImageDeploymentFailure::SupervisorFailed)? + { + return Err(ImageDeploymentFailure::Cancelled); + } + Ok(()) +} +fn map_failure(error: &DockerError) -> ImageDeploymentFailure { + match error { + DockerError::Status(StatusCode::NOT_FOUND) => ImageDeploymentFailure::ImageNotFound, + DockerError::Deadline => ImageDeploymentFailure::DeadlineExceeded, + DockerError::Cancelled => ImageDeploymentFailure::Cancelled, + DockerError::Worker => ImageDeploymentFailure::SupervisorFailed, + DockerError::Configuration + | DockerError::Schema + | DockerError::Image(_) + | DockerError::Json(_) + | DockerError::Protocol + | DockerError::PullDenied + | DockerError::PullFailed + | DockerError::TooLarge => ImageDeploymentFailure::InvalidImage, + _ => ImageDeploymentFailure::RuntimeUnavailable, + } +} + +#[cfg(test)] +#[path = "manager_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/deployment/manager_tests.rs b/crates/rustack-lambda-core/src/deployment/manager_tests.rs new file mode 100644 index 00000000..a6dbb3be --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment/manager_tests.rs @@ -0,0 +1,817 @@ +use std::{ + future::Future, + task::{Context, Poll, Waker}, +}; + +use async_trait::async_trait; +use rstest::rstest; +use rustack_lambda_model::{ + input::{ + CreateFunctionInput, PublishVersionInput, UpdateFunctionCodeInput, + UpdateFunctionConfigurationInput, + }, + types::FunctionCode, +}; +use tokio::time::timeout; + +use super::*; +use crate::{ + artifact::ArtifactError, + config::LambdaConfig, + deployment::{DeploymentState, tests::record}, + docker::{ + ImageLeaseManager, + image_fixture::{Command, FakePins, Mode as DaemonMode}, + }, + executor::{Executor, ExecutorBackend, InvokeRequest, InvokeResponse, NoopExecutor}, + invocation::InvocationContext, + provider::RustackLambda, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn until(condition: impl Fn() -> bool) { + timeout(Duration::from_secs(10), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +fn latest(store: &FunctionStore) -> crate::storage::VersionRecord { + store.get("worker").unwrap().unwrap().latest +} +fn named(name: &str) -> FunctionRecord { + let mut record = record(); + record.name = name.into(); + record.arn = format!("arn:aws:lambda:us-east-1:000000000000:function:{name}"); + record +} +struct Fixture { + root: tempfile::TempDir, + store: Arc, + fake: FakePins, + images: ImageLeaseManager, + manager: ImageDeploymentManager, +} +impl Fixture { + async fn new(mode: DaemonMode, jobs: u8, budget: Duration) -> Self { + let fake = FakePins::new(mode).await; + let images = fake.owner(8, jobs); + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let manager = ImageDeploymentManager::start( + store.clone(), + images.port(), + ImageDeploymentConfig::new(jobs, budget).unwrap(), + ) + .unwrap(); + Self { + root, + store, + fake, + images, + manager, + } + } + async fn idle(&self) { + until(|| self.manager.status().accepted == 0).await; + } + async fn failed_preparation(&self, reason: ImageDeploymentFailure) { + until(|| latest(&self.store).deployment == DeploymentState::CreateFailed(reason)).await; + assert_eq!(self.manager.status().accepted, 1); + } + async fn ready(&self) { + let pending = self.manager.create(record()).unwrap(); + assert_eq!(pending.latest.deployment.state(), "Pending"); + self.idle().await; + assert_eq!(latest(&self.store).deployment, DeploymentState::Ready); + } + async fn close(self, failed: bool) { + let Self { + root, + store, + fake, + mut images, + manager, + } = self; + assert_eq!(manager.shutdown(deadline()).await.is_err(), failed); + drop(manager); + drop(store); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; + drop(root); + } +} + +#[tokio::test] +async fn test_should_keep_accepted_preparation_owned_after_response_is_discarded() { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + drop(fixture.manager.create(record()).unwrap()); + fixture.idle().await; + let active = latest(&fixture.store); + assert_eq!(active.deployment, DeploymentState::Ready); + assert_eq!(active.code_size, 4096); + assert!(active.image.pinned().unwrap().is_live()); + assert!(!format!("{:?}", fixture.manager.status()).contains("private")); + drop(active); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_reject_capacity_before_resource_insertion_and_drain_after_cancellation() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(10)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(matches!( + fixture.manager.create(named("other")), + Err(LambdaServiceError::TooManyRequests) + )); + assert!(!fixture.store.contains("other").unwrap()); + assert!(fixture.manager.shutdown(Instant::now()).await.is_err()); + fixture + .failed_preparation(ImageDeploymentFailure::Cancelled) + .await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); + assert!(fixture.images.port().status().pins > 0); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.idle().await; + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_preserve_old_revision_during_update_and_after_preparation_failure() { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + fixture.ready().await; + let before = fixture.store.get("worker").unwrap().unwrap(); + let accepted = fixture + .manager + .update(&before, "missing:v2".parse().unwrap(), Architecture::Arm64) + .unwrap(); + assert_eq!(accepted.latest.deployment.update_status(), "InProgress"); + assert_eq!(accepted.latest.code_sha256, before.latest.code_sha256); + fixture.idle().await; + let failed = latest(&fixture.store); + assert_eq!( + failed.deployment, + DeploymentState::UpdateFailed(ImageDeploymentFailure::ImageNotFound) + ); + assert_eq!(failed.revision_id, before.latest.revision_id); + assert_eq!(failed.code_sha256, before.latest.code_sha256); + drop(failed); + drop(accepted); + drop(before); + let before = fixture.store.get("worker").unwrap().unwrap(); + fixture.fake.commands.send(Command::Retarget).await.unwrap(); + fixture + .manager + .update(&before, "worker:v1".parse().unwrap(), Architecture::Arm64) + .unwrap(); + fixture.idle().await; + assert_eq!(latest(&fixture.store).deployment, DeploymentState::Ready); + assert_ne!( + latest(&fixture.store).code_sha256, + before.latest.code_sha256 + ); + drop(before); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_keep_deployment_deadline_absolute_and_leave_critical_tag_cleanup_owned() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(1)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + // Notifications must recheck identity without restarting the preparation future. + for _ in 0..4 { + fixture + .store + .update("worker", |record| { + record.tags.insert("progress".into(), "value".into()); + }) + .unwrap(); + } + fixture + .failed_preparation(ImageDeploymentFailure::DeadlineExceeded) + .await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::DeadlineExceeded) + ); + assert!(fixture.images.port().status().pins > 0); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.idle().await; + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_cancel_deleted_generation_without_waiting_for_its_deployment_deadline() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(300)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + fixture.store.remove("worker").unwrap(); + let mut replacement = record(); + replacement.latest.description = "replacement".into(); + let resource = replacement.resource_id; + fixture.store.insert(replacement).unwrap(); + fixture.manager.quiesce(deadline()).await.unwrap(); + assert_eq!(fixture.manager.status().accepted, 1); + assert_eq!( + fixture.store.get("worker").unwrap().unwrap().resource_id, + resource + ); + assert_eq!(latest(&fixture.store).description, "replacement"); + assert!(latest(&fixture.store).image.pinned().is_none()); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.idle().await; + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_return_the_atomic_acceptance_receipt_after_immediate_deletion() { + let fixture = Fixture::new(DaemonMode::Normal, 1, Duration::from_secs(10)).await; + let reservation = fixture.manager.reserve().unwrap(); + let (job, receipt) = fixture.store.reserve_image_create(record()).unwrap(); + fixture.store.remove("worker").unwrap(); + let accepted = fixture.manager.accept(reservation, job, receipt); + assert_eq!(accepted.latest.deployment.state(), "Pending"); + fixture.idle().await; + assert!(!fixture.manager.status().failed); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 0); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_retain_join_when_drain_or_shutdown_waiter_is_abandoned() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(10)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let mut waiter = Box::pin(fixture.manager.quiesce(deadline())); + assert!(matches!( + waiter + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + drop(waiter); + assert!(!fixture.manager.status().accepting); + assert_eq!(fixture.manager.status().accepted, 1); + let mut waiter = Box::pin(fixture.manager.shutdown(deadline())); + let _ = waiter + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())); + drop(waiter); + fixture + .failed_preparation(ImageDeploymentFailure::Cancelled) + .await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.idle().await; + fixture.close(false).await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_settle_failed_workers_or_queued_root_work_and_cache_failure( + #[case] root: bool, +) { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + if root { + fixture + .manager + .health + .crash_root + .store(true, Ordering::Release); + } else { + fixture + .manager + .health + .crash_worker + .store(true, Ordering::Release); + } + fixture.manager.create(record()).unwrap(); + fixture.idle().await; + assert!(fixture.manager.status().failed); + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::SupervisorFailed) + ); + assert!(fixture.manager.shutdown(deadline()).await.is_err()); + assert!(fixture.manager.shutdown(deadline()).await.is_err()); + assert!(fixture.manager.create(named("other")).is_err()); + fixture.close(true).await; +} + +#[tokio::test] +async fn test_should_recover_joined_rows_after_root_drop_outside_an_entered_runtime() { + let fake = FakePins::new(DaemonMode::Never).await; + let mut images = fake.owner(4, 1); + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let job = store.insert_image_deployment(record()).unwrap(); + let health = Arc::new(Health::default()); + let (control, _) = watch::channel(Mode::Running); + let emergency = Arc::new(CleanupTask::default()); + let (_sender, inbox) = mpsc::channel(1); + let mut actor = Actor::builder() + .inbox(inbox) + .preparation(images.port()) + .control(control.clone()) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(emergency.clone()) + .build(); + let capacity = Arc::new(Semaphore::new(1)); + actor.start(Envelope { + id: Uuid::new_v4(), + pending: Arc::new( + Pending::builder() + .job(job) + .store(store.clone()) + .health(health.clone()) + .control(control) + .build(), + ), + slot: capacity.clone().acquire_owned().await.unwrap(), + deadline: deadline(), + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let thread = std::thread::spawn(move || drop(actor)); + tokio::task::spawn_blocking(move || thread.join().unwrap()) + .await + .unwrap(); + emergency.wait().await.unwrap(); + assert!(health.failed.load(Ordering::Acquire)); + assert_eq!(capacity.available_permits(), 0); + assert_eq!( + latest(&store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::SupervisorFailed) + ); + fake.commands.send(Command::Commit).await.unwrap(); + drop(store); + images.shutdown(deadline()).await.unwrap(); + assert_eq!(capacity.available_permits(), 1); + fake.close().await; +} + +#[test] +fn test_should_validate_deployment_budgets_and_map_bounded_failure_classes() { + assert!(ImageDeploymentConfig::new(0, Duration::from_secs(1)).is_err()); + assert!(ImageDeploymentConfig::new(33, Duration::from_secs(1)).is_err()); + assert!(ImageDeploymentConfig::new(1, Duration::from_nanos(1)).is_err()); + assert!(ImageDeploymentConfig::new(1, Duration::from_secs(3601)).is_err()); + assert_eq!( + map_failure(&DockerError::Status(StatusCode::NOT_FOUND)), + ImageDeploymentFailure::ImageNotFound + ); + assert_eq!( + map_failure(&DockerError::Quarantined), + ImageDeploymentFailure::RuntimeUnavailable + ); + assert_eq!( + map_failure(&DockerError::Protocol), + ImageDeploymentFailure::InvalidImage + ); + for error in [ + DockerError::PullDenied, + DockerError::PullFailed, + DockerError::Image(ArtifactError::Inspection), + ] { + assert_eq!(map_failure(&error), ImageDeploymentFailure::InvalidImage); + } + assert_eq!( + map_failure(&DockerError::Worker), + ImageDeploymentFailure::SupervisorFailed + ); +} + +#[derive(Debug)] +struct ControlPlaneExecutor; +#[async_trait] +impl Executor for ControlPlaneExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn available(&self, _: &InvokeRequest) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported( + "Control-plane fixture does not execute code".into(), + )) + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported( + "Control-plane fixture does not execute code".into(), + )) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +fn input(name: &str) -> CreateFunctionInput { + CreateFunctionInput { + function_name: name.into(), + role: "arn:aws:iam::000000000000:role/test".into(), + package_type: Some("Image".into()), + architectures: Some(vec!["arm64".into()]), + code: FunctionCode { + image_uri: Some("worker:v1".into()), + ..Default::default() + }, + ..Default::default() + } +} +async fn wait_provider(provider: &RustackLambda) { + until(|| { + let config = provider.get_function_configuration("worker", None).unwrap(); + config.last_update_status.as_deref() != Some("InProgress") + }) + .await; + // Give the joined root a chance to reap the completed worker before another admission. + tokio::task::yield_now().await; +} + +#[tokio::test] +async fn test_should_route_provider_image_create_update_status_and_explicit_revision_preconditions() +{ + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(8, 2); + let root = tempfile::tempdir().unwrap(); + let config = LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }; + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + config, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(2, Duration::from_secs(10)).unwrap(), + ) + .unwrap(); + let pending = provider.create_function(input("worker")).await.unwrap(); + assert_eq!(pending.state.as_deref(), Some("Pending")); + assert!(pending.code_sha256.is_none()); + wait_provider(&provider).await; + let before = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(before.state.as_deref(), Some("Active")); + assert!(before.code_sha256.is_some()); + assert!(matches!( + provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v1".into()), + revision_id: Some("stale".into()), + ..Default::default() + } + ) + .await, + Err(LambdaServiceError::PreconditionFailed) + )); + assert!(matches!( + provider.update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + revision_id: Some("stale".into()), + ..Default::default() + } + ), + Err(LambdaServiceError::PreconditionFailed) + )); + fake.commands.send(Command::Retarget).await.unwrap(); + let updating = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v1".into()), + revision_id: before.revision_id.clone(), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(updating.state.as_deref(), Some("Active")); + assert_eq!(updating.last_update_status.as_deref(), Some("InProgress")); + assert_eq!(updating.code_sha256, before.code_sha256); + wait_provider(&provider).await; + let after = provider.get_function_configuration("worker", None).unwrap(); + assert_ne!(after.code_sha256, before.code_sha256); + assert!(matches!( + provider.publish_version( + "worker", + &PublishVersionInput { + revision_id: before.revision_id, + ..Default::default() + } + ), + Err(LambdaServiceError::PreconditionFailed) + )); + let published = provider + .publish_version("worker", &PublishVersionInput::default()) + .unwrap(); + assert_eq!(published.version.as_deref(), Some("1")); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_unqualified_image_deployment_before_resource_mutation() { + let provider = RustackLambda::new(LambdaConfig::default()); + assert!(matches!( + provider.create_function(input("worker")).await, + Err(LambdaServiceError::DockerNotAvailable) + )); + assert!(!provider.store().contains("worker").unwrap()); + assert!(NoopExecutor::new().image_available().is_err()); + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + assert!( + RustackLambda::new(LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(1)).unwrap() + ) + .is_err() + ); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_poll_ready_preparation_ahead_of_continuously_ready_unrelated_changes() { + let (_stop, mut control) = watch::channel(Mode::Running); + let (publisher, mut changes) = watch::channel(()); + for _ in 0..16 { + publisher.send_replace(()); + let image = + PinnedImage::fixture(crate::artifact::ResolvedImage::fixture(Architecture::Arm64)); + let prepared = std::future::ready(Ok(image)); + tokio::pin!(prepared); + assert!(matches!( + next(prepared.as_mut(), &mut control, &mut changes, deadline()).await, + Ok(Progress::Prepared(Ok(_))) + )); + } +} + +#[tokio::test] +async fn test_should_accept_a_long_overall_budget_while_bounding_each_docker_read() { + let fixture = Fixture::new(DaemonMode::Normal, 1, Duration::from_secs(3600)).await; + fixture.ready().await; + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_keep_signalled_cleanup_running_when_owner_is_dropped_off_runtime() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(300)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let Fixture { + root, + store, + fake, + mut images, + manager, + } = fixture; + let thread = std::thread::spawn(move || drop(manager)); + tokio::task::spawn_blocking(move || thread.join().unwrap()) + .await + .unwrap(); + until(|| { + latest(&store).deployment + == DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + }) + .await; + fake.commands.send(Command::Commit).await.unwrap(); + drop(store); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; + drop(root); +} + +#[tokio::test] +async fn test_should_mark_an_empty_abnormally_dropped_root_as_failed() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let health = Arc::new(Health::default()); + let (control, _) = watch::channel(Mode::Running); + let (_sender, inbox) = mpsc::channel(1); + let actor = Actor::builder() + .inbox(inbox) + .preparation(images.port()) + .control(control.clone()) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(Arc::new(CleanupTask::default())) + .build(); + drop(actor); + assert!(health.failed.load(Ordering::Acquire)); + assert_eq!(*control.borrow(), Mode::Cancel); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_fail_quiescence_at_one_deadline_then_cancel_without_losing_ownership() { + let fake = FakePins::new(DaemonMode::Never).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(300)).unwrap(), + ) + .unwrap(); + provider.create_function(input("worker")).await.unwrap(); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(provider.quiesce(Duration::from_millis(10)).await.is_err()); + until(|| { + latest(provider.store()).deployment + == DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + }) + .await; + fake.commands.send(Command::Commit).await.unwrap(); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_deployment_capacity_larger_than_the_preparation_owner() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + assert!( + ImageDeploymentManager::start( + Arc::new(FunctionStore::new(root.path())), + images.port(), + ImageDeploymentConfig::new(2, Duration::from_secs(1)).unwrap() + ) + .is_err() + ); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_validate_image_code_dry_run_without_preparation_or_state_changes() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(10)).unwrap(), + ) + .unwrap(); + provider.create_function(input("worker")).await.unwrap(); + wait_provider(&provider).await; + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + let response = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("missing:new-image".into()), + dry_run: Some(true), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(response.last_update_status.as_deref(), Some("Successful")); + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.absent.load(Ordering::Acquire), 0); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_keep_cancelled_deployment_admission_after_outer_worker_join_until_image_io_retires() + { + let fake = FakePins::new(DaemonMode::Never).await; + let mut images = fake.owner(4, 1); + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let job = store.insert_image_deployment(record()).unwrap(); + let health = Arc::new(Health::default()); + let (control, _) = watch::channel(Mode::Running); + let (sender, inbox) = mpsc::channel(1); + let mut actor = Actor::builder() + .inbox(inbox) + .preparation(images.port()) + .control(control.clone()) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(Arc::new(CleanupTask::default())) + .build(); + let capacity = Arc::new(Semaphore::new(1)); + let pending = Arc::new( + Pending::builder() + .job(job) + .store(store.clone()) + .health(health) + .control(control) + .build(), + ); + actor.start(Envelope { + id: Uuid::new_v4(), + pending: pending.clone(), + slot: capacity.clone().acquire_owned().await.unwrap(), + deadline: deadline(), + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + store.remove("worker").unwrap(); + until(|| pending.settled.load(Ordering::Acquire)).await; + let joined = timeout(Duration::from_secs(2), actor.jobs.join_next_with_id()) + .await + .unwrap() + .unwrap(); + actor.reap(&joined); + assert!(actor.rows.is_empty()); + assert_eq!(capacity.available_permits(), 0); + assert!(capacity.clone().try_acquire_owned().is_err()); + assert_eq!(images.port().status().preparing, 1); + fake.commands.send(Command::Commit).await.unwrap(); + until(|| images.port().status().pins == 0).await; + assert_eq!(capacity.available_permits(), 1); + assert!(!store.contains("worker").unwrap()); + drop(sender); + timeout(Duration::from_secs(2), actor.run()) + .await + .unwrap() + .unwrap(); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[path = "manager_tests/restore.rs"] +mod restore; diff --git a/crates/rustack-lambda-core/src/deployment/manager_tests/restore.rs b/crates/rustack-lambda-core/src/deployment/manager_tests/restore.rs new file mode 100644 index 00000000..7731d055 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment/manager_tests/restore.rs @@ -0,0 +1,637 @@ +//! Complete-candidate restore must not publish partial resources or live placeholders. + +use std::{ + collections::BTreeMap, + io::{Cursor, Read, Write}, + sync::atomic::AtomicBool, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; +use serde_json::{Value, json}; +use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + +use super::*; +use crate::{ + artifact::{ImageArtifact, ImageMetadata, ResolvedImage}, + docker::{DockerExecutorError, ImageResolver}, + provider::{ + DockerArchiveProfile, LambdaSnapshot, + provided_zip_tests::{input as zip_input, package as zip_package}, + }, + storage::{ + AliasRecord, EventSourceMappingRecord, LayerRecord, LayerStoreSnapshot, compute_sha256, + }, +}; + +struct ImportFixture { + fake: FakePins, + images: ImageLeaseManager, + provider: Arc, + _root: tempfile::TempDir, +} +impl ImportFixture { + async fn new(mode: DaemonMode, old_pin: bool) -> Self { + Self::with_executor(mode, old_pin, Arc::new(ControlPlaneExecutor)).await + } + async fn with_executor(mode: DaemonMode, old_pin: bool, executor: Arc) -> Self { + let fake = FakePins::new(mode).await; + let images = fake.owner(8, 2); + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let mut old = named("old"); + if old_pin { + let image = images + .port() + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + old.latest.code_sha256 = image.resolved().code_sha256().unwrap(); + old.latest.code_size = image.unpacked_size().get(); + old.latest.image = ImageArtifact::Pinned(image); + } else { + old.latest.deployment = + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled); + } + store.insert(old).unwrap(); + let provider = Arc::new( + RustackLambda::with_executor( + store, + LambdaConfig { + executor: ExecutorBackend::Docker, + ..LambdaConfig::default() + }, + executor, + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(2, Duration::from_secs(10)).unwrap(), + ) + .unwrap(), + ); + provider.layer_store().import_snapshot(LayerStoreSnapshot { + layers: vec![LayerRecord { + name: "old-layer".into(), + layer_arn: "arn:aws:lambda:us-east-1:000000000000:layer:old-layer".into(), + versions: BTreeMap::new(), + next_version: 1, + }], + }); + let mapping: EventSourceMappingRecord = serde_json::from_value(json!({ + "uuid":"old-mapping","eventSourceArn":"arn:aws:sqs:us-east-1:000000000000:old", + "functionArn":"arn:aws:lambda:us-east-1:000000000000:function:old","enabled":false, + "batchSize":1,"maximumBatchingWindowInSeconds":0,"functionResponseTypes":[], + "state":"Disabled","stateTransitionReason":"fixture","lastModified":0,"lastProcessingResult":"None" + })).unwrap(); + provider.esm_store().create(mapping); + Self { + fake, + images, + provider, + _root: root, + } + } + async fn saved(&self) -> ResolvedImage { + ImageResolver::new(self.fake.wire.port()) + .resolve_local( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap() + } + fn state(&self) -> Value { + serde_json::to_value(self.provider.export_snapshot()).unwrap() + } + async fn close(mut self) { + self.provider.shutdown_before(deadline()).await.unwrap(); + drop(self.provider); + self.images.shutdown(deadline()).await.unwrap(); + self.fake.close().await; + } +} +fn saved_record(name: &str, saved: &ResolvedImage) -> FunctionRecord { + let mut value = named(name); + value.latest.code_sha256 = saved.code_sha256().unwrap(); + value.latest.code_size = 4096; + value.latest.image = ImageArtifact::Restored(saved.clone()); + value +} +fn snapshot(records: Vec) -> LambdaSnapshot { + let store = FunctionStore::new("unused-image-snapshot-source"); + for record in records { + store.insert(record).unwrap(); + } + LambdaSnapshot { + functions: store.export_snapshot(), + ..LambdaSnapshot::default() + } +} + +#[tokio::test] +async fn test_should_restore_latest_published_and_alias_with_one_shared_exact_lease() { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let saved = fixture.saved().await; + let mut record = saved_record("restored", &saved); + record.tags.insert("retained".into(), "yes".into()); + let mut published = record.latest.clone(); + published.version = "1".into(); + record.versions.insert(1, published); + record.next_version = 2; + record.aliases.insert( + "prod".into(), + AliasRecord { + name: "prod".into(), + function_version: "1".into(), + description: "saved".into(), + routing_config: None, + revision_id: Uuid::new_v4().to_string(), + }, + ); + let source = snapshot(vec![record]); + let expected = serde_json::to_value(&source).unwrap(); + fixture.fake.commands.send(Command::Retarget).await.unwrap(); + fixture + .provider + .import_snapshot_before(source, deadline()) + .await + .unwrap(); + until(|| fixture.images.port().status().pins == 1).await; + assert_eq!(fixture.state(), expected); + fixture.provider.validate_runtime_restore().unwrap(); + let restored = fixture.provider.store().get("restored").unwrap().unwrap(); + assert!(restored.latest.image.pinned().unwrap().is_live()); + assert!( + restored + .versions + .get(&1) + .unwrap() + .image + .pinned() + .unwrap() + .is_live() + ); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 2); + drop(restored); + fixture.close().await; +} + +#[rstest] +#[case("duplicate")] +#[case("memory")] +#[case("hash")] +#[case("missing-metadata")] +#[case("empty-plan")] +#[case("alias")] +#[case("region")] +#[case("account")] +#[tokio::test] +async fn test_should_validate_all_records_scope_and_plans_before_any_pin(#[case] corruption: &str) { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let saved = fixture.saved().await; + let mut source = snapshot(vec![saved_record("a", &saved), saved_record("z", &saved)]); + let before = fixture.state(); + let last = source.functions.functions.last_mut().unwrap(); + match corruption { + "memory" => last.latest.memory_size = 0, + "hash" => last.latest.code_sha256 = "mismatch".into(), + "missing-metadata" => last.latest.resolved_image = None, + "empty-plan" => { + last.latest.image_config = Some(rustack_lambda_model::types::ImageConfig { + entry_point: Some(vec![]), + command: Some(vec![]), + ..Default::default() + }); + } + "alias" => { + last.aliases.insert( + "prod".into(), + AliasRecord { + name: "prod".into(), + function_version: "7".into(), + description: String::new(), + routing_config: None, + revision_id: Uuid::new_v4().to_string(), + }, + ); + } + "region" => last.arn = last.arn.replace("us-east-1", "us-west-2"), + "account" => last.arn = last.arn.replace("000000000000", "111111111111"), + "duplicate" => source + .functions + .functions + .push(source.functions.functions.first().unwrap().clone()), + _ => panic!("fixed corruption table"), + } + assert!( + fixture + .provider + .import_snapshot_before(source, deadline()) + .await + .is_err() + ); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fixture.state(), before); + assert!( + fixture + .provider + .store() + .get("old") + .unwrap() + .unwrap() + .latest + .image + .pinned() + .unwrap() + .is_live() + ); + fixture.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_drop_prepared_candidates_on_later_size_or_missing_content_failure( + #[case] missing: bool, +) { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let saved = fixture.saved().await; + let mut source = snapshot(vec![saved_record("a", &saved), saved_record("z", &saved)]); + let last = source.functions.functions.last_mut().unwrap(); + if missing { + let mut metadata: ImageMetadata = saved.into(); + metadata.image_id = format!("sha256:{}", "e".repeat(64)).parse().unwrap(); + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + metadata.selected_manifest.as_ref(), + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + ) + .unwrap(); + let absent: ResolvedImage = metadata.try_into().unwrap(); + last.latest.code_sha256 = absent.code_sha256().unwrap(); + last.latest.resolved_image = Some(absent); + } else { + last.latest.code_size = 4097; + } + let before = fixture.state(); + let result = fixture + .provider + .import_snapshot_before(source, deadline()) + .await; + if missing { + assert!( + matches!(result, Err(LambdaServiceError::ImageRestore { source }) if matches!(*source, DockerError::Status(StatusCode::NOT_FOUND))) + ); + } else { + assert!(matches!( + result, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + } + until(|| fixture.images.port().status().pins == 1).await; + assert_eq!(fixture.state(), before); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 2); + assert_eq!(fixture.fake.stats.deletes.load(Ordering::Acquire), 1); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_reject_expired_commit_after_complete_pin_preparation_without_old_resource_changes() + { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let saved = fixture.saved().await; + let before = fixture.state(); + let source = snapshot(vec![saved_record("new", &saved)]); + let candidate = fixture + .provider + .store() + .prepare_docker_snapshot( + source.functions, + &fixture.images.port(), + deadline(), + DockerArchiveProfile::ImageOnly, + ) + .await + .unwrap(); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 2); + assert!( + matches!(candidate.commit_before(Instant::now()), Err(LambdaServiceError::ImageRestore { source }) if matches!(*source, DockerError::Deadline)) + ); + until(|| fixture.images.port().status().pins == 1).await; + assert_eq!(fixture.state(), before); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_retain_uncertain_restore_mutation_after_caller_cancellation_without_publication() + { + let fixture = ImportFixture::new(DaemonMode::Never, false).await; + let saved = fixture.saved().await; + let before = fixture.state(); + let source = snapshot(vec![saved_record("new", &saved)]); + let provider = fixture.provider.clone(); + let caller = + tokio::spawn(async move { provider.import_snapshot_before(source, deadline()).await }); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(fixture.images.port().status().pins, 1); + assert_eq!(fixture.state(), before); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.images.port().wait_empty(deadline()).await.unwrap(); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fixture.fake.stats.deletes.load(Ordering::Acquire), 1); + assert_eq!(fixture.state(), before); + fixture.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_restore_failed_and_interrupted_creation_without_an_executable_pin( + #[case] interrupted: bool, +) { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let mut value = named("failed"); + value.latest.deployment = if interrupted { + DeploymentState::Creating(Uuid::new_v4().to_string().try_into().unwrap()) + } else { + DeploymentState::CreateFailed(ImageDeploymentFailure::ImageNotFound) + }; + fixture + .provider + .import_snapshot_before(snapshot(vec![value]), deadline()) + .await + .unwrap(); + fixture.images.port().wait_empty(deadline()).await.unwrap(); + fixture.provider.validate_runtime_restore().unwrap(); + let restored = fixture.provider.store().get("failed").unwrap().unwrap(); + assert!(matches!( + restored.latest.deployment, + DeploymentState::CreateFailed(_) + )); + assert!(restored.latest.image.pinned().is_none()); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_export_borrowed_records_with_the_existing_owned_snapshot_schema() { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + let expected = fixture.state(); + let actual = fixture + .provider + .export_snapshot_before(deadline()) + .await + .unwrap(); + assert_eq!(serde_json::to_value(actual).unwrap(), expected); + fixture.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_deep_live_policy_and_oversized_ancillary_export_before_clone( + #[case] deep: bool, +) { + let fixture = ImportFixture::new(DaemonMode::Normal, true).await; + if deep { + let mut condition = Value::Null; + for _ in 0..50_000 { + condition = Value::Array(vec![condition]); + } + fixture + .provider + .store() + .update("old", |record| { + record + .policy + .statements + .push(crate::storage::PolicyStatement { + sid: "deep".into(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:InvokeFunction".into(), + resource: "*".into(), + condition: Some(condition), + }); + }) + .unwrap(); + } else { + let mut layer: LayerRecord = serde_json::from_value(json!({"name":"oversized","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:oversized","nextVersion":2, + "versions":{"1":{"version":1,"description":"","compatibleRuntimes":[],"compatibleArchitectures":[],"licenseInfo":null,"codeSha256":"unused","codeSize":0,"createdDate":"2026-10-27T00:00:00Z","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:oversized","layerVersionArn":"arn:aws:lambda:us-east-1:000000000000:layer:oversized:1","policy":{"statements":[]}}}})).unwrap(); + layer.versions.get_mut(&1).unwrap().description = "x".repeat(20 * 1024 * 1024); + fixture + .provider + .layer_store() + .import_snapshot(LayerStoreSnapshot { + layers: vec![layer], + }); + } + let error = fixture + .provider + .export_snapshot_before(deadline()) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidParameter { .. })); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fixture.provider.store().len(), 1); + fixture.close().await; +} + +#[derive(Debug)] +struct ManagedZipExecutor(Arc); +#[async_trait] +impl Executor for ManagedZipExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn docker_zip_architecture(&self) -> Option { + Some(Architecture::Arm64) + } + fn zip_available(&self, architecture: Architecture) -> Result<(), ExecutorError> { + if !self.0.load(Ordering::Acquire) || architecture != Architecture::Arm64 { + return Err(DockerExecutorError::Unavailable.into()); + } + Ok(()) + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported( + "archive metadata fixture".into(), + )) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +async fn managed_fixture(mode: DaemonMode) -> (ImportFixture, Arc, LambdaSnapshot) { + let live = Arc::new(AtomicBool::new(true)); + let fixture = + ImportFixture::with_executor(mode, false, Arc::new(ManagedZipExecutor(live.clone()))).await; + let mut input = zip_input(); + input.publish = Some(true); + fixture.provider.create_function(input).await.unwrap(); + let source = fixture + .provider + .export_snapshot_before(deadline()) + .await + .unwrap(); + (fixture, live, source) +} +#[tokio::test] +async fn test_should_restore_original_zip_with_valid_long_budget_and_fresh_private_artifacts() { + let (fixture, _live, source) = managed_fixture(DaemonMode::Normal).await; + let old = fixture + .provider + .store() + .get("provided-control") + .unwrap() + .unwrap(); + let expected = fixture.state(); + fixture + .provider + .import_snapshot_before(source, Instant::now() + Duration::from_secs(3600)) + .await + .unwrap(); + let loaded = fixture + .provider + .store() + .get("provided-control") + .unwrap() + .unwrap(); + assert_eq!(loaded.latest.code_sha256, old.latest.code_sha256); + assert_eq!(loaded.latest.revision_id, old.latest.revision_id); + assert_ne!(loaded.latest.code_path, old.latest.code_path); + assert_eq!(fixture.state(), expected); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 0); + drop(loaded); + drop(old); + fixture.close().await; +} +#[rstest] +#[case::non_elf(false)] +#[case::wrong_elf_target(true)] +#[tokio::test] +async fn test_should_classify_checksum_valid_bad_elf_as_input_before_any_image_pin( + #[case] wrong_target: bool, +) { + let (fixture, _live, mut source) = managed_fixture(DaemonMode::Normal).await; + let image = snapshot(vec![saved_record("a-image", &fixture.saved().await)]); + source.functions.functions.extend(image.functions.functions); + let mut bad = zip_package(false); + if wrong_target { + let bytes = STANDARD.decode(zip_package(true)).unwrap(); + let mut zip = ZipArchive::new(Cursor::new(bytes)).unwrap(); + let mut body = Vec::new(); + zip.by_name("bootstrap") + .unwrap() + .read_to_end(&mut body) + .unwrap(); + body.get_mut(18..20) + .unwrap() + .copy_from_slice(&62_u16.to_le_bytes()); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + ) + .unwrap(); + writer.write_all(&body).unwrap(); + bad = STANDARD.encode(writer.finish().unwrap().into_inner()); + } + let bytes = STANDARD.decode(&bad).unwrap(); + let row = &mut source + .functions + .functions + .iter_mut() + .find(|row| row.name == "provided-control") + .unwrap() + .latest; + row.code_size = u64::try_from(bytes.len()).unwrap(); + row.code_sha256 = compute_sha256(&bytes); + row.zip_bytes_base64 = Some(bad); + let before = fixture.state(); + let error = fixture + .provider + .import_snapshot_before(source, deadline()) + .await + .unwrap_err(); + assert!(matches!( + &error, + LambdaServiceError::InvalidParameter { .. } + )); + let public: LambdaError = error.into(); + assert_eq!(public.code, LambdaErrorCode::InvalidParameterValueException); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 0); + assert_eq!(fixture.state(), before); + fixture.close().await; +} +#[tokio::test] +async fn test_should_preserve_complete_old_zip_store_when_later_image_size_proof_fails() { + let (fixture, _live, mut source) = managed_fixture(DaemonMode::Normal).await; + let mut image = saved_record("z-image", &fixture.saved().await); + image.latest.code_size = 4097; + source + .functions + .functions + .extend(snapshot(vec![image]).functions.functions); + let before = fixture.state(); + assert!(matches!( + fixture + .provider + .import_snapshot_before(source, deadline()) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + fixture.images.port().wait_empty(deadline()).await.unwrap(); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fixture.fake.stats.deletes.load(Ordering::Acquire), 1); + assert_eq!(fixture.state(), before); + fixture.close().await; +} +#[tokio::test] +async fn test_should_fence_zip_capability_revocation_after_mixed_pin_preparation() { + let (fixture, live, mut source) = managed_fixture(DaemonMode::Never).await; + source.functions.functions.extend( + snapshot(vec![saved_record("z-image", &fixture.saved().await)]) + .functions + .functions, + ); + let before = fixture.state(); + let provider = fixture.provider.clone(); + let caller = + tokio::spawn(async move { provider.import_snapshot_before(source, deadline()).await }); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + live.store(false, Ordering::Release); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + assert!(caller.await.unwrap().is_err()); + fixture.images.port().wait_empty(deadline()).await.unwrap(); + assert_eq!(fixture.state(), before); + assert_eq!(fixture.fake.stats.deletes.load(Ordering::Acquire), 1); + fixture.close().await; +} diff --git a/crates/rustack-lambda-core/src/deployment_input.rs b/crates/rustack-lambda-core/src/deployment_input.rs new file mode 100644 index 00000000..fc11439d --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_input.rs @@ -0,0 +1,253 @@ +//! Bounded deployment DTO parsing with one borrowed inline-code exception. +//! +//! Pure blocking parse tasks, completed results and active dispatch retain one +//! of four permits until their bounded input work is released. An +//! abandoned caller intentionally leaves only bounded CPU/memory work running; +//! parsing cannot commit metadata, filesystem or Docker effects. + +use std::collections::BTreeMap; + +use bytes::Bytes; +use rustack_core::{ + http::append_bounded, + json::{JsonLimits, validate}, +}; +use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use tokio::{ + sync::{Semaphore, SemaphorePermit}, + task::spawn_blocking, +}; + +use crate::{artifact::MAX_ZIP_BASE64_BYTES, error::LambdaServiceError}; + +const BODY_BYTES: usize = 96 * 1024 * 1024; +const META_BYTES: usize = 1024 * 1024; +const STRING_BYTES: usize = 16 * 1024; +const KEY_BYTES: usize = 256; +const DEPTH: usize = 8; +const MEMBERS: usize = 4096; +// A fixed aggregate safety ceiling for large deployment parsing. Request +// admission remains separately operator-configurable at the HTTP boundary. +static PARSERS: Semaphore = Semaphore::const_new(4); + +#[derive(Debug, Clone, Copy)] +pub(crate) enum Profile { + Create, + Code, + Configuration, +} + +fn invalid() -> LambdaError { + LambdaError::invalid_parameter("Invalid or unsupported deployment request") +} + +pub(crate) async fn decode( + body: &Bytes, + profile: Profile, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + let body = body.clone(); + run(&PARSERS, move || parse(&body, profile)).await +} + +async fn run( + pool: &'static Semaphore, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + let permit = pool.try_acquire().map_err(|_| { + LambdaError::new( + LambdaErrorCode::TooManyRequestsException, + "Deployment parser capacity exhausted", + ) + })?; + run_admitted(permit, work).await +} + +async fn run_admitted( + permit: SemaphorePermit<'static>, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + spawn_blocking(move || { + let value = work()?; + Ok((value, permit)) + }) + .await + .map_err(|_| LambdaError::service_error("Deployment parser failed"))? +} + +fn parse(body: &[u8], profile: Profile) -> Result { + lexical(body)?; + // Borrow raw values, so large strings are not allocated before identifying + // the only field allowed to exceed the ordinary metadata allowance. + let fields: BTreeMap = + serde_json::from_slice(body).map_err(|_| invalid())?; + let blob = match profile { + Profile::Code => fields.get("ZipFile").copied(), + Profile::Create => match fields.get("Code") { + Some(code) => { + let code: BTreeMap = + serde_json::from_str(code.get()).map_err(|_| invalid())?; + code.get("ZipFile").copied() + } + None => None, + }, + Profile::Configuration => None, + } + .filter(|value| value.get().as_bytes().first() == Some(&b'"')); + let mut metadata = Vec::new(); + if let Some(blob) = blob { + let raw = blob.get().as_bytes(); + if raw + .len() + .checked_sub(2) + .is_none_or(|size| size > MAX_ZIP_BASE64_BYTES) + { + return Err(invalid()); + } + // Borrowed RawValue is a span in this very input (serde_json's slice + // deserializer contract). Checked integer offsets never dereference a + // raw pointer; bounded slice access re-establishes the range. + let start = raw + .as_ptr() + .addr() + .checked_sub(body.as_ptr().addr()) + .ok_or_else(invalid)?; + let end = start.checked_add(raw.len()).ok_or_else(invalid)?; + append(&mut metadata, body.get(..start).ok_or_else(invalid)?)?; + append(&mut metadata, b"\"\"")?; + append(&mut metadata, body.get(end..).ok_or_else(invalid)?)?; + } else { + append(&mut metadata, body)?; + } + let limits = JsonLimits::new(META_BYTES, DEPTH, MEMBERS, STRING_BYTES, KEY_BYTES) + .map_err(|_| invalid())?; + validate(&metadata, limits).map_err(|_| invalid())?; + serde_json::from_slice(body).map_err(|_| invalid()) +} +fn append(metadata: &mut Vec, bytes: &[u8]) -> Result<(), LambdaError> { + append_bounded(metadata, bytes, 1024 * 1024).map_err(|_| invalid()) +} + +/// Bound structure and encoded keys before borrowed-map key materialization. +fn lexical(body: &[u8]) -> Result<(), LambdaError> { + if body.len() > BODY_BYTES { + return Err(invalid()); + } + let mut cursor = 0usize; + let mut depth = 0usize; + let mut delimiters = 0usize; + while let Some(byte) = body.get(cursor) { + match byte { + b'"' => { + let start = cursor; + cursor = quoted_end(body, cursor)?; + let following = body + .get(cursor..) + .ok_or_else(invalid)? + .iter() + .find(|byte| !byte.is_ascii_whitespace()); + if following == Some(&b':') && cursor.saturating_sub(start) > KEY_BYTES * 6 + 2 { + return Err(invalid()); + } + continue; + } + b'{' | b'[' => { + depth = depth + .checked_add(1) + .filter(|depth| *depth <= DEPTH) + .ok_or_else(invalid)?; + } + b'}' | b']' => { + depth = depth.checked_sub(1).ok_or_else(invalid)?; + } + b',' | b':' => { + delimiters = delimiters + .checked_add(1) + .filter(|count| *count <= MEMBERS * 2) + .ok_or_else(invalid)?; + } + _ => {} + } + cursor = cursor.checked_add(1).ok_or_else(invalid)?; + } + Ok(()) +} +fn quoted_end(body: &[u8], start: usize) -> Result { + let mut cursor = start.checked_add(1).ok_or_else(invalid)?; + loop { + match body.get(cursor) { + Some(b'"') => return cursor.checked_add(1).ok_or_else(invalid), + Some(b'\\') => cursor = cursor.checked_add(2).ok_or_else(invalid)?, + Some(_) => cursor = cursor.checked_add(1).ok_or_else(invalid)?, + None => return Err(invalid()), + } + } +} + +/// Check standard canonical base64 without allocating decoded archive contents. +pub(crate) async fn validate_inline( + encoded: String, + admitted: Option>, +) -> Result, LambdaServiceError> { + let result = if let Some(permit) = admitted { + run_admitted(permit, move || inline(&encoded)).await + } else { + run(&PARSERS, move || inline(&encoded)).await + }; + result + .map(|((), permit)| permit) + .map_err(|error| match error.code { + LambdaErrorCode::TooManyRequestsException => LambdaServiceError::TooManyRequests, + LambdaErrorCode::ServiceException => LambdaServiceError::Internal { + message: "Deployment parser failed".into(), + }, + _ => LambdaServiceError::InvalidParameter { + message: "Invalid inline code encoding".into(), + }, + }) +} + +fn inline(encoded: &str) -> Result<(), LambdaError> { + if encoded.len() > MAX_ZIP_BASE64_BYTES || !encoded.len().is_multiple_of(4) { + return Err(invalid()); + } + let data = encoded.trim_end_matches('='); + let padding = encoded.len().checked_sub(data.len()).ok_or_else(invalid)?; + if padding > 2 { + return Err(invalid()); + } + let mut last = 0; + for byte in data.bytes() { + last = match byte { + b'A'..=b'Z' => byte.checked_sub(b'A'), + b'a'..=b'z' => byte + .checked_sub(b'a') + .and_then(|value| value.checked_add(26)), + b'0'..=b'9' => byte + .checked_sub(b'0') + .and_then(|value| value.checked_add(52)), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } + .ok_or_else(invalid)?; + } + if (padding == 2 && last & 15 != 0) || (padding == 1 && last & 3 != 0) { + return Err(invalid()); + } + let decoded = encoded + .len() + .checked_div(4) + .and_then(|groups| groups.checked_mul(3)) + .and_then(|bytes| bytes.checked_sub(padding)) + .ok_or_else(invalid)?; + if decoded > crate::artifact::MAX_ZIP_BYTES { + return Err(invalid()); + } + Ok(()) +} + +#[cfg(test)] +#[path = "deployment_input_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/deployment_input_tests.rs b/crates/rustack-lambda-core/src/deployment_input_tests.rs new file mode 100644 index 00000000..d72704c9 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_input_tests.rs @@ -0,0 +1,185 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use rustack_lambda_model::input::{ + CreateFunctionInput, UpdateFunctionCodeInput, UpdateFunctionConfigurationInput, +}; +use serde_json::json; +use tokio::sync::oneshot; + +use super::*; + +#[test] +fn test_should_allow_large_zip_only_at_the_selected_deployment_path() { + let zip = "A".repeat(2 * 1024 * 1024); + let code = serde_json::to_vec(&json!({"ZipFile":zip,"DryRun":true})).unwrap(); + let parsed: UpdateFunctionCodeInput = parse(&code, Profile::Code).unwrap(); + assert_eq!(parsed.zip_file.as_deref(), Some(zip.as_str())); + let create = + serde_json::to_vec(&json!({"FunctionName":"worker","Role":"role","Code":{"ZipFile":zip}})) + .unwrap(); + let parsed: CreateFunctionInput = parse(&create, Profile::Create).unwrap(); + assert_eq!(parsed.code.zip_file.as_deref(), Some(zip.as_str())); + assert!(parse::(&code, Profile::Configuration).is_err()); + let wrong = serde_json::to_vec(&json!({"Environment":{"Variables":{"ZipFile":zip}}})).unwrap(); + assert!(parse::(&wrong, Profile::Configuration).is_err()); +} + +#[rstest] +#[case(br#"{"ZipFile":"AA==","ZipFile":"AA=="}"#)] +#[case(br#"{"ZipFile":"AA==","\u005aipFile":"AA=="}"#)] +#[case(br#"{"DryRun":false,"DryRun":true}"#)] +#[case(br#"{"Architectures":[],"DryRun":"true"}"#)] +#[case(br#"{"DryRun":true,"SourceKMSKeyArn":"private-marker"}"#)] +#[case(br#"{"ZipFile":true}"#)] +#[case(br#"{"ZipFile":"unfinished}"#)] +fn test_should_reject_ambiguous_unknown_or_malformed_code_bodies(#[case] bytes: &[u8]) { + let error = parse::(bytes, Profile::Code).unwrap_err(); + assert_eq!(error.message, "Invalid or unsupported deployment request"); + assert!(!format!("{error:?}").contains("private-marker")); +} + +#[test] +fn test_should_recognize_escaped_code_keys_and_reject_nested_duplicate_environment_keys() { + let input = br#"{"FunctionName":"worker","Role":"role","\u0043ode":{"\u005aipFile":"AA=="}}"#; + assert_eq!( + parse::(input, Profile::Create) + .unwrap() + .code + .zip_file + .as_deref(), + Some("AA==") + ); + let duplicate = br#"{"Environment":{"Variables":{"TOKEN":"first","\u0054OKEN":"second"}}}"#; + assert!(parse::(duplicate, Profile::Configuration).is_err()); +} + +#[test] +fn test_should_reject_oversized_keys_strings_depth_and_metadata_before_dispatch() { + let body = serde_json::to_vec(&json!({"Description":"x".repeat(STRING_BYTES+1)})).unwrap(); + assert!(parse::(&body, Profile::Configuration).is_err()); + let key = format!("{{\"{}\":0}}", "x".repeat(KEY_BYTES * 6 + 1)); + assert!(lexical(key.as_bytes()).is_err()); + let nested = format!("{}0{}", "[".repeat(DEPTH + 1), "]".repeat(DEPTH + 1)); + assert!(lexical(nested.as_bytes()).is_err()); + let members = format!("[{}0]", "0,".repeat(MEMBERS * 2 + 1)); + assert!(lexical(members.as_bytes()).is_err()); + let metadata = format!("{{{}\"DryRun\":true}}", " ".repeat(META_BYTES)); + assert!(parse::(metadata.as_bytes(), Profile::Code).is_err()); +} + +#[rstest] +#[case("", true)] +#[case("AA==", true)] +#[case("AAA=", true)] +#[case("AAAA", true)] +#[case("AB==", false)] +#[case("AAB=", false)] +#[case("A===", false)] +#[case("AA=A", false)] +#[case("A", false)] +#[case("AAAA\n", false)] +fn test_should_validate_canonical_base64_without_decoding(#[case] data: &str, #[case] valid: bool) { + assert_eq!(inline(data).is_ok(), valid); +} + +#[test] +fn test_should_accept_exact_zip_size_and_reject_the_first_extra_decoded_byte() { + let bytes = vec![0u8; crate::artifact::MAX_ZIP_BYTES]; + let encoded = STANDARD.encode(bytes); + assert!(inline(&encoded).is_ok()); + let excess = "A".repeat(MAX_ZIP_BASE64_BYTES); + assert!(inline(&excess).is_err()); +} + +#[tokio::test] +async fn test_should_retain_a_parser_permit_after_its_response_waiter_is_dropped() { + static POOL: Semaphore = Semaphore::const_new(1); + let (begun, started) = oneshot::channel(); + let (release, blocked) = std::sync::mpsc::channel(); + let waiter = tokio::spawn(run(&POOL, move || { + begun.send(()).unwrap(); + blocked.recv().unwrap(); + Ok(()) + })); + started.await.unwrap(); + waiter.abort(); + let _ = waiter.await; + assert!(matches!( + run(&POOL, || Ok(())).await.unwrap_err().code, + LambdaErrorCode::TooManyRequestsException + )); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while POOL.available_permits() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(run(&POOL, || Ok(())).await.is_ok()); +} + +#[tokio::test] +async fn test_should_surface_a_pure_parser_worker_panic_without_losing_capacity() { + static POOL: Semaphore = Semaphore::const_new(1); + let error = run::<()>(&POOL, || panic!("injected pure parser panic")) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::ServiceException); + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_apply_strict_nested_deployment_dtos() { + for body in [ + json!({"ImageConfig":{"Unsupported":true}}), + json!({"Environment":{"Unknown":"private-marker"}}), + json!({"VpcConfig":{"Unsupported":true}}), + json!({"LoggingConfig":{"Unsupported":true}}), + ] { + assert!( + parse::( + &serde_json::to_vec(&body).unwrap(), + Profile::Configuration + ) + .is_err() + ); + } + let invalid = + json!({"FunctionName":"worker","Role":"role","Code":{"Unknown":"private-marker"}}); + assert!( + parse::(&serde_json::to_vec(&invalid).unwrap(), Profile::Create) + .is_err() + ); +} + +#[tokio::test] +async fn test_should_keep_capacity_with_a_completed_parse_result_until_it_is_consumed() { + static POOL: Semaphore = Semaphore::const_new(1); + let completed = run(&POOL, || Ok(7)).await.unwrap(); + assert_eq!(completed.0, 7); + assert_eq!(POOL.available_permits(), 0); + assert_eq!( + run(&POOL, || Ok(())).await.unwrap_err().code, + LambdaErrorCode::TooManyRequestsException + ); + drop(completed); + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_reuse_admission_for_all_concurrent_inline_validators() { + static POOL: Semaphore = Semaphore::const_new(4); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let permit = POOL.try_acquire().unwrap(); + tasks.spawn(validate_inline("AA==".into(), Some(permit))); + } + let mut results = Vec::new(); + while let Some(result) = tasks.join_next().await { + results.push(result.unwrap().unwrap()); + } + assert_eq!(POOL.available_permits(), 0); + drop(results); + assert_eq!(POOL.available_permits(), 4); +} diff --git a/crates/rustack-lambda-core/src/deployment_tests.rs b/crates/rustack-lambda-core/src/deployment_tests.rs new file mode 100644 index 00000000..53ca96fa --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_tests.rs @@ -0,0 +1,689 @@ +use std::{ + collections::{BTreeMap, HashMap}, + sync::{Arc, atomic::AtomicBool}, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use rustack_lambda_model::{ + input::{PublishVersionInput, UpdateFunctionCodeInput, UpdateFunctionConfigurationInput}, + types::ImageConfig, +}; +use tokio::sync::watch; + +use super::*; +use crate::{ + artifact::{ImageMetadata, ResolvedImage}, + config::LambdaConfig, + provider::RustackLambda, + storage::{FunctionStoreSnapshot, PolicyDocument}, +}; + +fn image( + reference: &str, + letter: char, + architecture: Architecture, +) -> (PinnedImage, watch::Receiver<()>, watch::Sender) { + let mut metadata: ImageMetadata = ResolvedImage::fixture(architecture).into(); + metadata.requested_reference = reference.parse().unwrap(); + metadata.image_id = format!("sha256:{}", letter.to_string().repeat(64)) + .parse() + .unwrap(); + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + None, + architecture, + metadata.size_bytes, + &metadata.execution, + ) + .unwrap(); + let (release, released) = watch::channel(()); + let (stop, stopped) = watch::channel(false); + let image = PinnedImage::new( + metadata.try_into().unwrap(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped, + ); + (image, released, stop) +} +pub(crate) fn record() -> FunctionRecord { + FunctionRecord { + resource_id: FunctionResourceId::new(), + name: "worker".into(), + arn: "arn:aws:lambda:us-east-1:000000000000:function:worker".into(), + latest: VersionRecord { + version: "$LATEST".into(), + runtime: None, + handler: None, + role: "arn:aws:iam::000000000000:role/test".into(), + description: "old description".into(), + timeout: 3, + memory_size: 128, + environment: HashMap::from([("SECRET".into(), "private value".into())]), + package_type: "Image".into(), + code_path: None, + image_uri: Some("worker:v1".into()), + image: ImageArtifact::Unresolved, + zip_bytes: None, + deployment: DeploymentState::Ready, + last_modified: "2026-10-25T00:00:00.000+0000".into(), + architectures: vec!["arm64".into()], + ephemeral_storage_size: 512, + code_sha256: String::new(), + code_size: 0, + revision_id: Uuid::new_v4().to_string(), + layers: Vec::new(), + vpc_config: None, + dead_letter_config: None, + tracing_config: None, + image_config: None, + logging_config: None, + snap_start: None, + }, + versions: BTreeMap::new(), + next_version: 1, + aliases: HashMap::new(), + policy: PolicyDocument::default(), + tags: HashMap::new(), + code_signing_config_arn: None, + url_config: None, + reserved_concurrent_executions: None, + event_invoke_configs: HashMap::new(), + created_at: "2026-10-25T00:00:00.000+0000".into(), + } +} +fn store() -> (tempfile::TempDir, FunctionStore) { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + (root, store) +} +fn ready(store: &FunctionStore) -> FunctionRecord { + let job = store.insert_image_deployment(record()).unwrap(); + store + .settle_image_deployment(&job, Ok(image("worker:v1", 'a', Architecture::Arm64).0)) + .unwrap(); + store.get("worker").unwrap().unwrap() +} +fn update(store: &FunctionStore, reference: &str) -> ImageDeployment { + let before = store.get("worker").unwrap().unwrap(); + store + .begin_image_update( + "worker", + before.resource_id, + &before.latest.revision_id, + reference.parse().unwrap(), + Architecture::Arm64, + ) + .unwrap() +} +fn current(store: &FunctionStore) -> VersionRecord { + store.get("worker").unwrap().unwrap().latest +} + +#[test] +fn test_should_reserve_pending_and_settle_with_content_hash_and_unpacked_size() { + let (_root, store) = store(); + let mut before = record(); + before.latest.code_sha256 = "unverified tag hash".into(); + before.latest.code_size = 999; + let old_revision = before.latest.revision_id.clone(); + let resource = before.resource_id; + let job = store.insert_image_deployment(before).unwrap(); + assert_eq!(current(&store).deployment.state(), "Pending"); + assert_eq!(current(&store).code_sha256, ""); + assert_eq!(current(&store).code_size, 0); + assert_eq!(current(&store).deployment.update_status(), "InProgress"); + let (image, _, _) = image("worker:v1", 'a', Architecture::Arm64); + let hash = image + .resolved() + .metadata() + .inspection_hash + .as_str() + .strip_prefix("sha256:") + .unwrap(); + let expected = STANDARD.encode(hex::decode(hash).unwrap()); + assert!(!format!("{job:?}").contains("private value")); + assert!(!format!("{job:?}").contains("worker:v1")); + assert!(store.settle_image_deployment(&job, Ok(image)).unwrap()); + let after = store.get("worker").unwrap().unwrap(); + assert_eq!(after.resource_id, resource); + assert_eq!(after.latest.deployment, DeploymentState::Ready); + assert_ne!(after.latest.revision_id, old_revision); + assert_eq!(after.latest.code_sha256, expected); + assert_eq!(after.latest.code_size, 4096); // Metadata Size is only1024. + assert!(after.latest.image.pinned().unwrap().is_live()); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::SupervisorFailed)) + .unwrap() + ); +} + +#[test] +fn test_should_preserve_previous_revision_while_updating_and_after_failure() { + let (_root, store) = store(); + let before = ready(&store).latest; + let job = update(&store, "worker:v2"); + let pending = current(&store); + assert_eq!(pending.deployment.state(), "Active"); + assert_eq!(pending.deployment.update_status(), "InProgress"); + assert_eq!(pending.revision_id, before.revision_id); + assert_eq!(pending.image_uri, before.image_uri); + assert_eq!(pending.code_sha256, before.code_sha256); + assert_eq!( + pending.image.resolved().unwrap().metadata().image_id, + before.image.resolved().unwrap().metadata().image_id + ); + assert!( + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::InvalidImage)) + .unwrap() + ); + let failed = current(&store); + assert_eq!( + failed.deployment, + DeploymentState::UpdateFailed(ImageDeploymentFailure::InvalidImage) + ); + assert_eq!(failed.revision_id, before.revision_id); + assert_eq!(failed.image_uri, before.image_uri); + assert_eq!(failed.code_sha256, before.code_sha256); + assert_eq!(failed.environment, before.environment); + let retry = update(&store, "worker:v2"); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); + assert!( + store + .settle_image_deployment(&retry, Ok(image("worker:v2", 'b', Architecture::Arm64).0)) + .unwrap() + ); + assert_eq!(current(&store).image_uri.as_deref(), Some("worker:v2")); + assert_ne!(current(&store).code_sha256, before.code_sha256); +} + +#[test] +fn test_should_retry_failed_creation_without_claiming_a_previous_active_revision() { + let (_root, store) = store(); + let initial = store.insert_image_deployment(record()).unwrap(); + assert!( + store + .settle_image_deployment(&initial, Err(ImageDeploymentFailure::ImageNotFound)) + .unwrap() + ); + assert_eq!(current(&store).deployment.state(), "Failed"); + let retry = update(&store, "worker:fixed"); + assert_eq!(current(&store).deployment.state(), "Pending"); + assert_eq!(current(&store).image_uri.as_deref(), Some("worker:fixed")); + assert!( + store + .settle_image_deployment( + &retry, + Ok(image("worker:fixed", 'b', Architecture::Arm64).0) + ) + .unwrap() + ); + assert_eq!(current(&store).deployment, DeploymentState::Ready); +} + +#[test] +fn test_should_reject_stale_resource_revision_and_generation_independently() { + let (_root, store) = store(); + let before = ready(&store); + assert!( + store + .begin_image_update( + "worker", + FunctionResourceId::new(), + &before.latest.revision_id, + "worker:v2".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + assert!( + store + .begin_image_update( + "worker", + before.resource_id, + "stale", + "worker:v2".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + let job = update(&store, "worker:v2"); + assert!( + store + .begin_image_update( + "worker", + before.resource_id, + &before.latest.revision_id, + "worker:v3".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + store + .update("worker", |record| { + record.latest.revision_id = Uuid::new_v4().to_string(); + }) + .unwrap(); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); + store.remove("worker").unwrap(); + let mut recreated = before; + recreated.resource_id = FunctionResourceId::new(); + recreated.latest.deployment = DeploymentState::Updating(job.generation); + store.insert(recreated).unwrap(); + assert!( + !store + .settle_image_deployment(&job, Ok(image("worker:v2", 'b', Architecture::Arm64).0)) + .unwrap() + ); + store.remove("worker").unwrap(); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); +} + +#[rstest] +#[case("other:v1", Architecture::Arm64, false)] +#[case("worker:v1", Architecture::X86_64, false)] +#[case("worker:v1", Architecture::Arm64, true)] +fn test_should_reject_mismatching_or_revoked_preparation_without_partial_commit( + #[case] reference: &str, + #[case] architecture: Architecture, + #[case] revoke: bool, +) { + let (_root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let pending = current(&store).deployment; + let (image, _, stop) = image(reference, 'a', architecture); + if revoke { + stop.send_replace(true); + } + assert!(store.settle_image_deployment(&job, Ok(image)).is_err()); + assert_eq!(current(&store).deployment, pending); + assert!(current(&store).image.pinned().is_none()); + assert!( + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::InvalidImage)) + .unwrap() + ); +} + +#[test] +fn test_should_drop_stale_result_lease_without_modifying_a_new_generation() { + let (_root, store) = store(); + ready(&store); + let job = update(&store, "worker:v2"); + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap(); + let replacement = update(&store, "worker:v3"); + let (image, released, _) = image("worker:v2", 'b', Architecture::Arm64); + assert!(!store.settle_image_deployment(&job, Ok(image)).unwrap()); + assert!(released.has_changed().is_err()); + assert_eq!( + current(&store).deployment, + DeploymentState::Updating(replacement.generation) + ); +} + +#[test] +fn test_should_rollback_every_change_when_a_fallible_store_mutation_fails() { + let (_root, store) = store(); + let before = ready(&store); + let result: Result<(), _> = + store.try_update_if_revision("worker", Some(&before.latest.revision_id), |record| { + record.latest.environment.clear(); + record.latest.description = "must not commit".into(); + record.latest.revision_id = "changed".into(); + Err(invalid("injected validation failure")) + }); + assert!(result.is_err()); + let after = current(&store); + assert_eq!(after.environment, before.latest.environment); + assert_eq!(after.description, before.latest.description); + assert_eq!(after.revision_id, before.latest.revision_id); +} + +#[tokio::test] +async fn test_should_derive_public_status_and_guard_mutations_and_publication() { + let (root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let pending = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(pending.state.as_deref(), Some("Pending")); + assert!(pending.code_sha256.is_none()); + assert_eq!(pending.last_update_status.as_deref(), Some("InProgress")); + assert!( + provider + .publish_version("worker", &PublishVersionInput::default()) + .is_err() + ); + assert!( + provider + .update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + description: Some("no".into()), + ..Default::default() + } + ) + .is_err() + ); + assert!( + provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v2".into()), + ..Default::default() + } + ) + .await + .is_err() + ); + assert!( + provider + .store() + .settle_image_deployment(&job, Err(ImageDeploymentFailure::ImageNotFound)) + .unwrap() + ); + let failed = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(failed.state.as_deref(), Some("Failed")); + assert_eq!( + failed.state_reason.as_deref(), + Some(ImageDeploymentFailure::ImageNotFound.reason()) + ); + assert_eq!(failed.state_reason_code.as_deref(), Some("ImageDeleted")); + assert_eq!(failed.last_update_status.as_deref(), Some("Failed")); + assert!( + provider + .publish_version("worker", &PublishVersionInput::default()) + .is_err() + ); + drop(provider); + drop(root); +} + +#[tokio::test] +async fn test_should_keep_published_leases_and_report_resolved_uri() { + let (_root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let (image, released, _) = image("worker:v1", 'a', Architecture::Arm64); + store.settle_image_deployment(&job, Ok(image)).unwrap(); + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let published = provider + .publish_version("worker", &PublishVersionInput::default()) + .unwrap(); + assert_eq!(published.version.as_deref(), Some("1")); + let code = provider + .get_function("worker", Some("1")) + .unwrap() + .code + .unwrap(); + assert_eq!(code.image_uri.as_deref(), Some("worker:v1")); + assert!( + code.resolved_image_uri + .unwrap() + .starts_with("local://sha256:") + ); + let update = update(provider.store(), "worker:v2"); + provider + .store() + .settle_image_deployment(&update, Ok(image_fixture("worker:v2", 'b'))) + .unwrap(); + assert!(!released.has_changed().unwrap()); + provider.delete_function("worker", Some("1")).await.unwrap(); + assert!(released.has_changed().is_err()); +} +fn image_fixture(reference: &str, letter: char) -> PinnedImage { + image(reference, letter, Architecture::Arm64).0 +} + +#[tokio::test] +async fn test_should_rollback_invalid_effective_configuration_and_guard_publish_keys() { + let (_root, store) = store(); + let before = ready(&store).latest; + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + assert!( + provider + .update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + description: Some("must not commit".into()), + image_config: Some(ImageConfig { + entry_point: Some(Vec::new()), + command: Some(Vec::new()), + working_directory: None + }), + ..Default::default() + } + ) + .is_err() + ); + assert_eq!(current(provider.store()).description, before.description); + assert_eq!(current(provider.store()).revision_id, before.revision_id); + for input in [ + PublishVersionInput { + revision_id: Some("stale".into()), + ..Default::default() + }, + PublishVersionInput { + code_sha256: Some("wrong".into()), + ..Default::default() + }, + ] { + assert!(provider.publish_version("worker", &input).is_err()); + } + assert!( + provider + .store() + .get("worker") + .unwrap() + .unwrap() + .versions + .is_empty() + ); +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_restore_in_progress_snapshots_as_failed_without_inventing_a_lease( + #[case] updating: bool, +) { + let (_root, store) = store(); + if updating { + ready(&store); + update(&store, "worker:v2"); + } else { + store.insert_image_deployment(record()).unwrap(); + } + let json = serde_json::to_vec(&store.export_snapshot()).unwrap(); + let restore_root = tempfile::tempdir().unwrap(); + let restored = FunctionStore::new(restore_root.path()); + restored + .import_snapshot(serde_json::from_slice(&json).unwrap()) + .await + .unwrap(); + let after = current(&restored); + assert_eq!( + after.deployment, + if updating { + DeploymentState::UpdateFailed(ImageDeploymentFailure::Cancelled) + } else { + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + } + ); + assert!(after.image.pinned().is_none()); + assert_eq!(after.image.resolved().is_some(), updating); +} + +#[tokio::test] +async fn test_should_reject_contradictory_snapshot_states_and_accept_legacy_active_metadata() { + let (_root, store) = store(); + ready(&store); + let baseline = current(&store).revision_id; + let mut snapshot = store.export_snapshot(); + snapshot.functions.first_mut().unwrap().latest.state = "Failed".into(); + assert!(store.import_snapshot(snapshot).await.is_err()); + assert_eq!(current(&store).revision_id, baseline); + let mut legacy = serde_json::to_value(store.export_snapshot()).unwrap(); + legacy["functions"][0]["latest"] + .as_object_mut() + .unwrap() + .remove("deployment"); + let restore_root = tempfile::tempdir().unwrap(); + let restored = FunctionStore::new(restore_root.path()); + restored + .import_snapshot(serde_json::from_value::(legacy).unwrap()) + .await + .unwrap(); + assert_eq!(current(&restored).deployment, DeploymentState::Ready); + assert!(matches!( + current(&restored).image, + ImageArtifact::Restored(_) + )); +} + +#[test] +fn test_should_reject_nil_invalid_and_unknown_deployment_state_values() { + assert!(DeploymentGeneration::try_from(Uuid::nil()).is_err()); + for value in [Uuid::nil().to_string(), "x".repeat(2048), "invalid".into()] { + assert!( + serde_json::from_value::(serde_json::Value::String(value)) + .is_err() + ); + } + assert!(serde_json::from_str::(r#"{"kind":"ready","extra":true}"#).is_err()); + let state = DeploymentState::Creating(DeploymentGeneration(Uuid::new_v4())); + assert_eq!( + serde_json::from_slice::(&serde_json::to_vec(&state).unwrap()).unwrap(), + state + ); +} + +#[rstest] +#[case("hash")] +#[case("zero")] +#[case("oversize")] +#[tokio::test] +async fn test_should_reject_tampered_resolved_snapshot_metadata_atomically( + #[case] corruption: &str, +) { + let (_root, store) = store(); + let before = ready(&store).latest; + let mut snapshot = store.export_snapshot(); + let version = &mut snapshot.functions.first_mut().unwrap().latest; + match corruption { + "hash" => version.code_sha256 = "forged".into(), + "zero" => version.code_size = 0, + "oversize" => version.code_size = crate::artifact::MAX_IMAGE_BYTES + 1, + _ => unreachable!("trusted case"), + } + assert!(store.import_snapshot(snapshot).await.is_err()); + let after = current(&store); + assert_eq!(after.revision_id, before.revision_id); + assert_eq!(after.code_sha256, before.code_sha256); + assert!(after.image.pinned().unwrap().is_live()); +} + +#[test] +fn test_should_reject_a_live_image_size_that_disagrees_with_its_proof() { + let (_root, store) = store(); + let before = ready(&store).latest; + assert!( + store + .update("worker", |record| record.latest.code_size += 1) + .is_err() + ); + assert_eq!(current(&store).code_size, before.code_size); +} + +#[test] +fn test_should_commit_only_one_of_two_concurrent_completion_results() { + let (_root, store) = store(); + ready(&store); + let job = update(&store, "worker:v2"); + let barrier = std::sync::Barrier::new(2); + let wins = std::thread::scope(|threads| { + let handles: Vec<_> = ['b', 'c'] + .into_iter() + .map(|letter| { + let store = &store; + let job = &job; + let barrier = &barrier; + threads.spawn(move || { + let prepared = image("worker:v2", letter, Architecture::Arm64).0; + barrier.wait(); + store.settle_image_deployment(job, Ok(prepared)).unwrap() + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .filter(|won| *won) + .count() + }); + assert_eq!(wins, 1); + assert_eq!(current(&store).deployment, DeploymentState::Ready); +} + +#[tokio::test] +async fn test_should_preserve_failed_readiness_when_restoring_legacy_zip_progress() { + let (_root, store) = store(); + let mut source = record(); + source.latest.package_type = "Zip".into(); + source.latest.runtime = Some("provided.al2023".into()); + source.latest.handler = Some("bootstrap".into()); + source.latest.image_uri = None; + store.insert(source).unwrap(); + let mut snapshot = store.export_snapshot(); + let version = &mut snapshot.functions.first_mut().unwrap().latest; + version.state = "Pending".into(); + version.deployment = None; + store.import_snapshot(snapshot).await.unwrap(); + assert_eq!( + current(&store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); +} + +#[tokio::test] +async fn test_should_reject_image_update_publication_before_mutating_the_previous_revision() { + let (_root, store) = store(); + let before = ready(&store).latest; + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let outcome = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v2".into()), + publish: Some(true), + ..Default::default() + }, + ) + .await; + assert!(matches!( + outcome, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let after = current(provider.store()); + assert_eq!(after.revision_id, before.revision_id); + assert_eq!(after.code_sha256, before.code_sha256); + assert_eq!(after.deployment, DeploymentState::Ready); +} diff --git a/crates/rustack-lambda-core/src/diagnostics.rs b/crates/rustack-lambda-core/src/diagnostics.rs new file mode 100644 index 00000000..235635be --- /dev/null +++ b/crates/rustack-lambda-core/src/diagnostics.rs @@ -0,0 +1,38 @@ +//! Bounded actual invocation ownership, sampled without business admission. +//! +//! Counters are independently sampled; they are not a transaction spanning two +//! semaphores. They contain no function identity, request data or credentials. + +use serde::Serialize; + +pub(crate) const ACCEPTED_LIMIT: usize = 128; +pub(crate) const EXECUTION_LIMIT: usize = 32; +pub(crate) const DEFAULT_FUNCTION_LIMIT: usize = 8; + +/// Limits used by the real invocation supervisor and its admission semaphores. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct InvocationWorkLimits { + /// Maximum accepted calls, including asynchronous waiting and cleanup. + pub accepted_invocations: usize, + /// Maximum owned global execution permits, including backend pool waiting. + pub active_executions: usize, + /// Per-function execution limit when no reserved override is installed. + pub default_function_executions: usize, +} + +/// Actual independently sampled ownership counters from one Lambda provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct InvocationWorkStatus { + /// Accepted-work permits retained until execution/cleanup settles. + pub accepted_invocations: usize, + /// Global execution permits retained through backend waiting and cleanup. + pub active_executions: usize, + /// Whether the supervisor still accepts ordinary invocation submissions. + pub accepting: bool, + /// The same limits used to construct/acquire actual admission capacity. + pub limits: InvocationWorkLimits, +} diff --git a/crates/rustack-lambda-core/src/docker/binding.rs b/crates/rustack-lambda-core/src/docker/binding.rs new file mode 100644 index 00000000..e397ea85 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/binding.rs @@ -0,0 +1,159 @@ +//! Persist installation identity and retain exact directory ownership on IO. + +use std::{fmt, sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use serde::Deserialize; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + DirectoryLease, DockerError, DockerPort, Operation, RawReply, RawRequest, check_deadline, + decode, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) struct DaemonIdentity(Uuid); + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct IdentityResponse { + #[serde(rename = "ID")] + id: String, +} + +impl fmt::Debug for IdentityResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("IdentityResponse()") + } +} + +impl fmt::Debug for DaemonIdentity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DaemonIdentity()") + } +} + +impl TryFrom<&str> for DaemonIdentity { + type Error = DockerError; + + fn try_from(value: &str) -> Result { + if value.len() != 36 { + return Err(DockerError::Protocol); + } + let id = Uuid::try_parse(value).map_err(|_| DockerError::Protocol)?; + if id.is_nil() || id.hyphenated().to_string() != value { + return Err(DockerError::Protocol); + } + Ok(Self(id)) + } +} + +impl DaemonIdentity { + pub(super) fn from_uuid(id: Uuid) -> Result { + if id.is_nil() { + return Err(DockerError::Protocol); + } + Ok(Self(id)) + } + + pub(super) fn uuid(self) -> Uuid { + self.0 + } + + pub(super) fn decode(reply: RawReply, deadline: Instant) -> Result { + let response: IdentityResponse = decode(reply, deadline)?; + Self::try_from(response.id.as_str()) + } +} + +#[derive(Debug)] +pub(super) struct DaemonBinding { + pub(super) identity: DaemonIdentity, + pub(super) directory: DirectoryLease, +} + +impl DockerPort { + /// Persist this daemon's installation identity and retain the exact lease. + /// + /// The returned port checks identity on every actual operation connection. + /// First binding requires empty container/image journals; existing bindings + /// never silently move to another daemon. This does not grant readiness. + /// + /// # Errors + /// Rejects invalid budgets/identity, mismatched or unsafe durable state, + /// unprovable historical effects, IO deadlines and wire failures. + /// + /// ```no_run + /// # async fn bind(port: &rustack_lambda_core::docker::DockerPort, directory: &rustack_lambda_core::docker::DirectoryLease) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let bound = port.bind_directory(directory, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(directory.clone()); + /// let mut images = rustack_lambda_core::docker::ImageLeaseManager::start(bound, scope, Default::default())?; + /// images.shutdown(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn bind_directory( + &self, + directory: &DirectoryLease, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DockerError::Configuration); + } + let identity = async { + let reply = self + .raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation: None, + operation: Operation::Info, + method: Method::GET, + target: "/info".into(), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline, + ) + .await?; + DaemonIdentity::decode(reply, deadline) + } + .await + .map_err(|source| DockerError::IdentityCheck(Arc::new(source)))?; + directory.bind_daemon(identity, deadline).await?; + check_deadline(deadline)?; + let mut bound = self.clone(); + bound.raw.binding = Some(Arc::new(DaemonBinding { + identity, + directory: directory.clone(), + })); + Ok(bound) + } + + pub(super) fn require_directory(&self, directory: &DirectoryLease) -> Result<(), DockerError> { + if self + .raw + .binding + .as_ref() + .is_some_and(|binding| binding.directory.same_instance(directory)) + { + return Ok(()); + } + #[cfg(test)] + if self.raw.binding.is_none() && directory.synthetic() { + return Ok(()); + } + Err(DockerError::DaemonBinding) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/binding/tests.rs b/crates/rustack-lambda-core/src/docker/binding/tests.rs new file mode 100644 index 00000000..2189a8d1 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/binding/tests.rs @@ -0,0 +1,283 @@ +use std::time::Duration; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use rstest::rstest; +use tokio::{ + fs::read, + spawn, + time::{Instant, timeout}, +}; + +use super::DaemonIdentity; +use crate::docker::{ + DirectoryLease, DirectoryLeaseError, DockerError, EnvironmentConfig, EnvironmentManager, + ImageLeaseConfig, ImageLeaseManager, ImagePinScope, Operation, RawRequest, + mutation::MutationGuard, + reconcile_containers, reconcile_image_pins, + tests::{DAEMON_ID, Daemon}, +}; + +mod connection; +mod real; +mod recovery; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} + +#[rstest] +#[case("")] +#[case("00000000-0000-0000-0000-000000000000")] +#[case("44CA5378-c28c-4bac-b6ed-2b298f02ecb5")] +#[case("44ca5378c28c4bacb6ed2b298f02ecb5")] +#[case("{44ca5378-c28c-4bac-b6ed-2b298f02ecb5}")] +#[case("44ca5378-c28c-4bac-b6ed-2b298f02ecb5\n")] +#[case("33Z5:BE33:KSMC:S4GY:GRDO:EH5E")] +#[case("éééééééééééééééééé")] +fn test_should_reject_noncanonical_or_unproven_identity(#[case] value: &str) { + assert!(matches!( + DaemonIdentity::try_from(value), + Err(DockerError::Protocol) + )); +} + +#[test] +fn test_should_accept_and_redact_canonical_installation_identity() { + let identity = DaemonIdentity::try_from(DAEMON_ID).unwrap(); + assert_eq!(identity.uuid().to_string(), DAEMON_ID); + assert!(!format!("{identity:?}").contains(DAEMON_ID)); + assert_eq!( + format!("{:?}", DockerError::DaemonBinding), + "Docker daemon binding failed" + ); +} + +#[tokio::test] +async fn test_should_require_exact_bound_lease_for_runtime_and_recovery() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let other_root = tempfile::tempdir().unwrap(); + let other = DirectoryLease::acquire(other_root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let raw = wire.port(); + assert!(matches!( + ImageLeaseManager::start( + raw.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default() + ), + Err(DockerError::DaemonBinding) + )); + assert!( + EnvironmentManager::start(raw.clone(), directory.clone(), EnvironmentConfig::default()) + .is_err() + ); + assert!( + reconcile_containers(&directory, &raw, deadline()) + .await + .is_err() + ); + assert!(matches!( + reconcile_image_pins(&directory, &raw, deadline()).await, + Err(DockerError::DaemonBinding) + )); + let bound = raw.bind_directory(&directory, deadline()).await.unwrap(); + assert!(bound.require_directory(&other).is_err()); + assert!( + reconcile_containers(&other, &bound, deadline()) + .await + .is_err() + ); + assert!( + reconcile_image_pins(&other, &bound, deadline()) + .await + .is_err() + ); + assert_eq!( + reconcile_containers(&directory, &bound, deadline()) + .await + .unwrap() + .remaining, + 0 + ); + assert_eq!( + reconcile_image_pins(&directory, &bound, deadline()) + .await + .unwrap() + .remaining, + 0 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + let mut images = ImageLeaseManager::start( + bound.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + ) + .unwrap(); + images.shutdown(deadline()).await.unwrap(); + drop(images); + let mut environments = EnvironmentManager::start( + bound.clone(), + directory.clone(), + EnvironmentConfig::default(), + ) + .unwrap(); + environments.shutdown(deadline()).await.unwrap(); + drop(environments); + drop(directory); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(bound); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let bound = raw.bind_directory(&next, deadline()).await.unwrap(); + bound.require_directory(&next).unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case("44ca5378-c28c-4bac-b6ed-2b298f02ecb6", false)] +#[case("invalid", true)] +#[tokio::test] +async fn test_should_refuse_changed_peer_before_mutation_issuance( + #[case] id: &str, + #[case] malformed: bool, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let before = read(root.path().join(".rustack-lambda-docker/daemon.v1")) + .await + .unwrap(); + daemon.identity.send_replace(id.to_owned()); + let guard = MutationGuard::new(); + let receipt = guard.receipt(); + let result = bound + .raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation: Some(guard), + operation: Operation::TagPin, + method: Method::POST, + target: "/v1.53/images/test/tag".into(), + body: Bytes::new(), + expected: StatusCode::CREATED, + }, + deadline(), + ) + .await; + assert!(matches!(result, Err(DockerError::IdentityCheck(source)) if + if malformed { matches!(source.as_ref(), DockerError::Protocol) } + else { matches!(source.as_ref(), DockerError::DaemonBinding) })); + assert!(receipt.definitely_unissued()); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!( + read(root.path().join(".rustack-lambda-docker/daemon.v1")) + .await + .unwrap(), + before + ); + daemon.identity.send_replace(DAEMON_ID.into()); + let key = "a".repeat(64).parse().unwrap(); + let (result, ()) = tokio::join!(bound.inspect_container(&key, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert!(matches!( + result, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + )); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_refuse_changed_installation_before_rebinding_directory() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + drop( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ); + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + daemon + .identity + .send_replace("44ca5378-c28c-4bac-b6ed-2b298f02ecb6".into()); + assert!(matches!( + wire.port().bind_directory(&next, deadline()).await, + Err(DockerError::Ownership(_)) + )); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_exact_directory_through_abandoned_inflight_request() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let task = spawn(async move { + bound + .inspect_container(&"a".repeat(64).parse().unwrap(), deadline()) + .await + }); + let held = daemon.next().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + drop(directory); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + held.reply(404, b"{}").await; + wire.shutdown().await.unwrap(); + let _next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/binding/tests/connection.rs b/crates/rustack-lambda-core/src/docker/binding/tests/connection.rs new file mode 100644 index 00000000..7edb0e7f --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/binding/tests/connection.rs @@ -0,0 +1,125 @@ +use std::{sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use rstest::rstest; +use serde_json::{json, to_vec}; +use tokio::{ + io::AsyncWriteExt, + net::UnixListener, + spawn, + time::{advance, pause, timeout}, +}; + +use super::deadline; +use crate::docker::{ + DirectoryLease, DockerError, Operation, Owner, RawRequest, + binding::{DaemonBinding, DaemonIdentity}, + mutation::MutationGuard, + tests::{DAEMON_ID, read_request}, +}; + +#[rstest] +#[case::same_connection(0)] +#[case::checked_peer_closed(1)] +#[case::invalid_json(2)] +#[case::unavailable(3)] +#[case::expired_before_issue(4)] +#[case::missing_identity(5)] +#[case::null_identity(6)] +#[tokio::test] +async fn test_should_fence_issue_on_the_checked_connection(#[case] mode: u8) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let identity = DaemonIdentity::try_from(DAEMON_ID).unwrap(); + directory.bind_daemon(identity, deadline()).await.unwrap(); + let path = root.path().join("fixture.sock"); + let listener = UnixListener::bind(&path).unwrap(); + let (mut owner, mut raw) = Owner::start(path); + raw.binding = Some(Arc::new(DaemonBinding { + identity, + directory, + })); + let guard = MutationGuard::new(); + let receipt = guard.receipt(); + let request_port = raw.clone(); + let task = spawn(async move { + request_port + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation: Some(guard), + operation: Operation::TagPin, + method: Method::POST, + target: "/v1.53/images/proof/tag".into(), + body: Bytes::new(), + expected: StatusCode::CREATED, + }, + deadline(), + ) + .await + }); + let (stream, _) = timeout(Duration::from_secs(5), listener.accept()) + .await + .unwrap() + .unwrap(); + let mut incoming = read_request(stream).await.unwrap(); + assert!(incoming.head.starts_with("GET /info ")); + assert!( + incoming + .head + .to_ascii_lowercase() + .contains("connection: keep-alive") + ); + let body = match mode { + 2 => b"{".to_vec(), + 5 => b"{}".to_vec(), + 6 => br#"{"ID":null}"#.to_vec(), + _ => to_vec(&json!({"ID": DAEMON_ID})).unwrap(), + }; + if mode == 4 { + pause(); + advance(Duration::from_secs(20)).await; + } + if mode == 1 { + incoming.reply(200, &body).await; + } else { + let status = if mode == 3 { 500 } else { 200 }; + let head = format!( + "HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nConnection: keep-alive\r\n\r\n", + body.len() + ); + // Expiry can close the checked connection before the fixture writes. + let _ = incoming.stream.write_all(head.as_bytes()).await; + let _ = incoming.stream.write_all(&body).await; + if mode == 0 { + let actual = read_request(incoming.stream).await.unwrap(); + assert!(actual.head.starts_with("POST /v1.53/images/proof/tag ")); + actual.reply(201, b"").await; + } else { + assert!(read_request(incoming.stream).await.is_err()); + } + } + let result = task.await.unwrap(); + if mode == 0 { + assert!(result.is_ok()); + assert!(!receipt.definitely_unissued()); + } else { + assert!(result.is_err()); + assert!(receipt.definitely_unissued()); + if mode == 4 { + assert!(matches!(result, Err(DockerError::Deadline))); + } + } + assert!( + timeout(Duration::from_millis(20), listener.accept()) + .await + .is_err(), + "checked connection was replayed on another socket" + ); + owner.shutdown().await.unwrap(); +} diff --git a/crates/rustack-lambda-core/src/docker/binding/tests/real.rs b/crates/rustack-lambda-core/src/docker/binding/tests/real.rs new file mode 100644 index 00000000..526fb56e --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/binding/tests/real.rs @@ -0,0 +1,77 @@ +use std::{env, time::Duration}; + +use anyhow::{Context, Ok as anyhow_ok, Result, ensure}; +use tokio::{fs::read, time::Instant}; + +use crate::docker::{DirectoryLease, DockerError, DockerWire}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} + +#[tokio::test] +#[ignore = "requires two explicit distinct Docker daemons"] +async fn test_should_bind_real_installation_across_reacquisition_and_reject_other_daemon() +-> Result<()> { + let socket = + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit primary Docker socket")?; + let other = env::var("RUSTACK_OTHER_DOCKER_SOCKET").context("explicit other Docker socket")?; + let root = tempfile::tempdir()?; + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let mut second = None; + let exercise = async { + let directory = DirectoryLease::acquire(root.path().into(), deadline()).await?; + let bound = wire.port().bind_directory(&directory, deadline()).await?; + let first_id = bound + .info(deadline()) + .await? + .id + .context("engine identity")?; + let previous_owner = directory.owner_id(); + let previous_run = directory.run_id(); + drop(bound); + drop(directory); + wire.shutdown().await?; + wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let next = DirectoryLease::acquire(root.path().into(), deadline()).await?; + ensure!( + next.owner_id() == previous_owner && next.run_id() != previous_run, + "directory identities did not follow restart contract" + ); + let bound = wire.port().bind_directory(&next, deadline()).await?; + ensure!( + bound.info(deadline()).await?.id.as_deref() == Some(first_id.as_str()), + "installation identity changed" + ); + let path = root.path().join(".rustack-lambda-docker/daemon.v1"); + let before = read(&path).await?; + second = Some(DockerWire::connect(format!("unix://{other}").parse()?, deadline()).await?); + let port = second.as_ref().context("second wire")?.port(); + ensure!( + port.info(deadline()).await?.id.as_deref() != Some(first_id.as_str()), + "explicit daemons are not distinct installations" + ); + ensure!( + matches!( + port.bind_directory(&next, deadline()).await, + Err(DockerError::Ownership(_)) + ), + "different daemon was accepted" + ); + ensure!( + read(path).await? == before, + "failed binding changed authoritative state" + ); + anyhow_ok(()) + } + .await; + let second_cleanup = match second.as_mut() { + Some(owner) => owner.shutdown().await, + None => Ok(()), + }; + let first_cleanup = wire.shutdown().await; + exercise?; + second_cleanup?; + first_cleanup?; + Ok(()) +} diff --git a/crates/rustack-lambda-core/src/docker/binding/tests/recovery.rs b/crates/rustack-lambda-core/src/docker/binding/tests/recovery.rs new file mode 100644 index 00000000..c2c33203 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/binding/tests/recovery.rs @@ -0,0 +1,168 @@ +use std::{ + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use http::StatusCode; +use rstest::rstest; +use tokio::{ + fs::read, + time::{Instant, sleep, timeout}, +}; + +use super::deadline; +use crate::{ + artifact::Architecture, + docker::{ + DirectoryLease, DockerError, EnvironmentError, EnvironmentPlan, ImageLeaseConfig, + ImageLeaseManager, ImagePinScope, + lease::tests::{FakePins, Mode}, + ownership::{ContainerRecord, ImageRecord}, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + reconcile_containers, reconcile_image_pins, + tests::Daemon, + }, +}; + +#[rstest] +#[case::known_container(true)] +#[case::committed_alias(false)] +#[tokio::test] +async fn test_should_never_treat_identity_404_as_resource_absence(#[case] container: bool) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + if container { + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let record = ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(); + directory + .prepare_container(record.clone(), deadline()) + .await + .unwrap(); + directory + .know_container(record, "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + } else { + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let record = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + directory + .prepare_image(record.clone(), deadline()) + .await + .unwrap(); + directory.commit_image(record, deadline()).await.unwrap(); + } + let path = root + .path() + .join(".rustack-lambda-docker") + .join(if container { + "containers.v1" + } else { + "pins.v1" + }); + let before = read(&path).await.unwrap(); + drop(bound); + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let bound = wire.port().bind_directory(&next, deadline()).await.unwrap(); + daemon.identity_status.send_replace(404); + if container { + assert!( + matches!(reconcile_containers(&next, &bound, deadline()).await, + Err(EnvironmentError::Docker(DockerError::IdentityCheck(source))) + if matches!(source.as_ref(), DockerError::Status(StatusCode::NOT_FOUND))) + ); + assert_eq!(next.container_records(deadline()).await.unwrap().len(), 1); + } else { + assert!( + matches!(reconcile_image_pins(&next, &bound, deadline()).await, + Err(DockerError::IdentityCheck(source)) + if matches!(source.as_ref(), DockerError::Status(StatusCode::NOT_FOUND))) + ); + assert_eq!(next.image_records(deadline()).await.unwrap().len(), 1); + } + assert_eq!(read(path).await.unwrap(), before); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_live_pin_capacity_when_cleanup_identity_returns_404() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let fake = FakePins::new(Mode::Normal).await; + let bound = fake + .wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start( + bound, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::new(1, 1, Duration::from_secs(1)).unwrap(), + ) + .unwrap(); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + fake.identity_status.send_replace(404); + drop(image); + timeout(Duration::from_secs(5), async { + while port.status().quarantined == 0 { + assert_eq!(port.status().pins, 1); + sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + assert!( + port.wait_empty(Instant::now() + Duration::from_millis(100)) + .await + .is_err() + ); + assert_eq!(port.status().pins, 1); + assert!(port.status().quarantined > 0); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + fake.identity_status.send_replace(200); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + fake.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/code.rs b/crates/rustack-lambda-core/src/docker/code.rs new file mode 100644 index 00000000..5eba5318 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code.rs @@ -0,0 +1,364 @@ +//! Immutable ZIP-to-TAR conversion with retained transfer-buffer admission. +//! +//! A canceled observer leaves only a bounded, read-only blocking computation. +//! Its owned source and reservation survive until the computation/output drops; +//! conversion performs no filesystem reads or daemon operations. Source-lease +//! retirement retains its normal last-owner cleanup. + +use std::{ + fmt, + io::{self, Cursor, Read, Write}, + path::Path, + sync::{Arc, LazyLock}, +}; + +use bytes::Bytes; +use tar::{Builder, EntryType, Header}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, + task::spawn_blocking, + time::{Instant, timeout_at}, +}; +use zip::ZipArchive; + +use super::{DockerError, check_deadline}; +use crate::{ + artifact::{Architecture, MAX_ZIP_BYTES, ValidatedArtifact}, + storage::{MAX_EXTRACTED_SIZE, ZipLayout, validate_zip_directory, validated_zip_path}, +}; + +pub(super) const MAX_CODE_ARCHIVE_BYTES: usize = 282 * 1024 * 1024; +static CONVERSIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(2))); + +struct Storage { + bytes: Bytes, + architecture: Architecture, + source: ValidatedArtifact, + _reservation: OwnedSemaphorePermit, +} + +/// Validated provided-runtime code, ready for the fixed Docker upload endpoint. +/// +/// Clones share bytes, original code ownership and one global buffer reservation. +/// There is no unchecked byte constructor, host-path bind or TAR extraction. +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerCodeArchive(Arc); + +impl fmt::Debug for DockerCodeArchive { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DockerCodeArchive") + .field("bytes", &self.len()) + .field("architecture", &self.architecture()) + .finish_non_exhaustive() + } +} + +impl DockerCodeArchive { + /// Convert original staged ZIP bytes, checking the captured Linux ELF target. + /// + /// # Errors + /// Returns invalid code, byte/allocation limits, expired deadline, closed + /// admission or worker failure. No partial artifact or daemon effect is + /// produced. Two global slots remain held until the last transfer owner drops. + /// + /// ```no_run + /// # async fn prepare(source: rustack_lambda_core::artifact::ValidatedArtifact) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// use rustack_lambda_core::{artifact::Architecture, docker::DockerCodeArchive}; + /// let code = DockerCodeArchive::prepare(source, Architecture::Arm64, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; + /// assert!(!code.is_empty()); + /// # Ok(()) } + /// ``` + pub async fn prepare( + source: ValidatedArtifact, + architecture: Architecture, + deadline: Instant, + ) -> Result { + with_reservation(CONVERSIONS.clone(), deadline, move |reservation| { + convert(source, architecture, reservation, deadline) + }) + .await + } + + /// Encoded TAR bytes, including headers, padding and terminal blocks. + /// + /// ```no_run + /// # fn inspect(code: &rustack_lambda_core::docker::DockerCodeArchive) { + /// assert!(code.len() <= 282 * 1024 * 1024); + /// # } + /// ``` + #[must_use] + pub fn len(&self) -> usize { + self.0.bytes.len() + } + + /// Whether this transfer is empty; successful code archives are never empty. + /// + /// ```no_run + /// # fn inspect(code: &rustack_lambda_core::docker::DockerCodeArchive) { + /// assert!(!code.is_empty()); + /// # } + /// ``` + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.bytes.is_empty() + } + + /// The architecture positively checked in the root bootstrap ELF header. + /// + /// ```no_run + /// # fn inspect(code: &rustack_lambda_core::docker::DockerCodeArchive) { + /// assert!(matches!(code.architecture(), + /// rustack_lambda_core::artifact::Architecture::Arm64 | + /// rustack_lambda_core::artifact::Architecture::X86_64)); + /// # } + /// ``` + #[must_use] + pub fn architecture(&self) -> Architecture { + self.0.architecture + } + + /// Base64 SHA-256 of the original ZIP, independent of transfer normalization. + /// + /// ```no_run + /// # fn inspect(code: &rustack_lambda_core::docker::DockerCodeArchive) { + /// assert_eq!(code.code_sha256().len(), 44); + /// # } + /// ``` + #[must_use] + pub fn code_sha256(&self) -> &str { + self.0.source.sha256() + } + + pub(super) fn bytes(&self) -> &Bytes { + &self.0.bytes + } +} + +async fn with_reservation( + admission: Arc, + deadline: Instant, + work: impl FnOnce(OwnedSemaphorePermit) -> Result + Send + 'static, +) -> Result { + check_deadline(deadline)?; + let reservation = timeout_at(deadline, admission.acquire_owned()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Stopped)?; + check_deadline(deadline)?; + let result = timeout_at(deadline, spawn_blocking(move || work(reservation))) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)??; + check_deadline(deadline)?; + Ok(result) +} + +fn convert( + source: ValidatedArtifact, + architecture: Architecture, + reservation: OwnedSemaphorePermit, + deadline: Instant, +) -> Result { + check_deadline(deadline)?; + if source.zip_bytes().len() > MAX_ZIP_BYTES { + return Err(DockerError::TooLarge); + } + let mut zip = ZipArchive::new(Cursor::new(source.zip_bytes().as_ref())) + .map_err(|_| DockerError::InvalidCode)?; + if zip.len() > 10_000 || zip.has_duplicate_names() { + return Err(DockerError::InvalidCode); + } + let mut output = Builder::new(BoundedWriter { + bytes: Vec::new(), + deadline, + limit: MAX_CODE_ARCHIVE_BYTES, + }); + let mut layout = ZipLayout::default(); + let mut extracted = 0u64; + let mut bootstrap = false; + for index in 0..zip.len() { + check_deadline(deadline)?; + let mut entry = zip.by_index(index).map_err(|_| DockerError::InvalidCode)?; + let path = validated_zip_path(&entry).map_err(|_| DockerError::InvalidCode)?; + if entry.is_dir() { + validate_zip_directory(&mut entry).map_err(|_| DockerError::InvalidCode)?; + } + let directories = layout + .admit(&path, entry.is_dir()) + .map_err(|_| DockerError::InvalidCode)?; + for directory in directories { + let mut header = Header::new_gnu(); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(0); + header.set_entry_type(EntryType::Directory); + header.set_mode(0o555); + output + .append_data(&mut header, directory, io::empty()) + .map_err(|error| archive_io(error.kind()))?; + } + if entry.is_dir() { + continue; + } + let remaining = MAX_EXTRACTED_SIZE.saturating_sub(extracted); + if entry.size() > remaining { + return Err(DockerError::TooLarge); + } + let executable = entry.unix_mode().is_some_and(|mode| mode & 0o111 != 0); + let mut prefix = Vec::new(); + if path == Path::new("bootstrap") { + if !executable { + return Err(DockerError::InvalidCode); + } + prefix.resize(64, 0); + entry + .read_exact(&mut prefix) + .map_err(|_| DockerError::InvalidCode)?; + validate_elf(&prefix, architecture)?; + bootstrap = true; + } + let declared = entry.size(); + let mut reader = CheckedReader { + inner: Cursor::new(prefix).chain(&mut entry), + actual: 0, + remaining: remaining.saturating_add(1), + deadline, + }; + let mut header = Header::new_gnu(); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(declared); + header.set_entry_type(EntryType::Regular); + header.set_mode(if executable { 0o555 } else { 0o444 }); + output + .append_data(&mut header, path, &mut reader) + .map_err(|error| archive_io(error.kind()))?; + if reader.actual != declared { + return Err(DockerError::InvalidCode); + } + extracted = extracted + .checked_add(reader.actual) + .filter(|total| *total <= MAX_EXTRACTED_SIZE) + .ok_or(DockerError::TooLarge)?; + } + if !bootstrap { + return Err(DockerError::InvalidCode); + } + let writer = output + .into_inner() + .map_err(|error| archive_io(error.kind()))?; + check_deadline(deadline)?; + drop(zip); + Ok(DockerCodeArchive(Arc::new(Storage { + bytes: Bytes::from(writer.bytes), + architecture, + source, + _reservation: reservation, + }))) +} + +fn validate_elf(header: &[u8], architecture: Architecture) -> Result<(), DockerError> { + let machine: u16 = match architecture { + Architecture::X86_64 => 62, + Architecture::Arm64 => 183, + }; + if header.len() != 64 + || header.get(..7) != Some(b"\x7fELF\x02\x01\x01") + || !matches!(header.get(16..18), Some([2 | 3, 0])) + || header.get(18..20) != Some(machine.to_le_bytes().as_slice()) + || header.get(20..24) != Some(1u32.to_le_bytes().as_slice()) + || header.get(52..54) != Some(64u16.to_le_bytes().as_slice()) + { + return Err(DockerError::InvalidCode); + } + Ok(()) +} + +struct CheckedReader { + inner: R, + actual: u64, + remaining: u64, + deadline: Instant, +} +impl Read for CheckedReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + check_io(self.deadline)?; + let maximum = usize::try_from(self.remaining.min(8192)) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidData))? + .min(buffer.len()); + if maximum == 0 && !buffer.is_empty() { + return Err(io::Error::from(io::ErrorKind::FileTooLarge)); + } + let count = self.inner.read( + buffer + .get_mut(..maximum) + .ok_or(io::ErrorKind::InvalidInput)?, + )?; + let count64 = u64::try_from(count).map_err(|_| io::ErrorKind::InvalidData)?; + self.remaining = self.remaining.saturating_sub(count64); + self.actual = self + .actual + .checked_add(count64) + .ok_or(io::ErrorKind::InvalidData)?; + check_io(self.deadline)?; + Ok(count) + } +} + +struct BoundedWriter { + bytes: Vec, + deadline: Instant, + limit: usize, +} +impl Write for BoundedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + check_io(self.deadline)?; + let needed = self + .bytes + .len() + .checked_add(bytes.len()) + .filter(|total| *total <= self.limit) + .ok_or(io::ErrorKind::FileTooLarge)?; + if needed > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(needed) + .min(self.limit); + self.bytes + .try_reserve_exact(capacity.saturating_sub(self.bytes.len())) + .map_err(|_| io::ErrorKind::OutOfMemory)?; + } + self.bytes.extend_from_slice(bytes); + check_io(self.deadline)?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + check_io(self.deadline) + } +} + +fn check_io(deadline: Instant) -> io::Result<()> { + if Instant::now() >= deadline { + Err(io::ErrorKind::TimedOut.into()) + } else { + Ok(()) + } +} + +fn archive_io(kind: io::ErrorKind) -> DockerError { + match kind { + io::ErrorKind::TimedOut => DockerError::Deadline, + io::ErrorKind::FileTooLarge => DockerError::TooLarge, + io::ErrorKind::OutOfMemory => DockerError::Allocation, + _ => DockerError::InvalidCode, + } +} + +#[cfg(test)] +pub(in crate::docker) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/code/tests.rs b/crates/rustack-lambda-core/src/docker/code/tests.rs new file mode 100644 index 00000000..7357271f --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code/tests.rs @@ -0,0 +1,544 @@ +use std::{ + io::{Cursor, Read, Write}, + path::Path, + sync::{Arc, mpsc as blocking}, + time::Duration, +}; + +use bytes::Bytes; +use rstest::rstest; +use tar::Archive; +use tempfile::TempDir; +use tokio::{ + fs, + sync::{Semaphore, oneshot}, + time::{Instant, timeout}, +}; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +use super::{ + BoundedWriter, CheckedReader, DockerCodeArchive, convert, validate_elf, with_reservation, +}; +use crate::{ + artifact::{Architecture, ValidatedArtifact}, + docker::DockerError, + error::LambdaServiceError, + storage::FunctionStore, +}; + +pub(in crate::docker) fn elf(architecture: Architecture) -> Vec { + let mut bytes = vec![0; 64]; + bytes + .get_mut(..7) + .unwrap() + .copy_from_slice(b"\x7fELF\x02\x01\x01"); + bytes + .get_mut(16..18) + .unwrap() + .copy_from_slice(&2u16.to_le_bytes()); + let machine: u16 = match architecture { + Architecture::Arm64 => 183, + Architecture::X86_64 => 62, + }; + bytes + .get_mut(18..20) + .unwrap() + .copy_from_slice(&machine.to_le_bytes()); + bytes + .get_mut(20..24) + .unwrap() + .copy_from_slice(&1u32.to_le_bytes()); + bytes + .get_mut(52..54) + .unwrap() + .copy_from_slice(&64u16.to_le_bytes()); + bytes +} + +pub(in crate::docker) fn zip(entries: &[(&str, &[u8], u32)]) -> Vec { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (name, bytes, mode) in entries { + writer + .start_file( + *name, + SimpleFileOptions::default() + .unix_permissions(*mode) + .compression_method(CompressionMethod::Stored), + ) + .unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap().into_inner() +} + +// Deliberately bypass staging only for converter defense-in-depth controls. +pub(in crate::docker) async fn source(bytes: Vec) -> (TempDir, ValidatedArtifact) { + let root = tempfile::tempdir().unwrap(); + let owned = tempfile::tempdir_in(root.path()).unwrap(); + let extracted = owned.path().join("extracted"); + fs::create_dir(&extracted).await.unwrap(); + let artifact = ValidatedArtifact::from_staged( + owned, + root.path().to_owned(), + extracted, + Bytes::from(bytes), + ); + (root, artifact) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} + +pub(in crate::docker) async fn prepare(source: ValidatedArtifact) -> DockerCodeArchive { + prepare_with(source, Arc::new(Semaphore::new(2))).await +} + +pub(in crate::docker) async fn prepare_with( + source: ValidatedArtifact, + admission: Arc, +) -> DockerCodeArchive { + let deadline = deadline(); + with_reservation(admission, deadline, move |permit| { + convert(source, Architecture::Arm64, permit, deadline) + }) + .await + .unwrap() +} + +#[rstest] +#[case(Architecture::Arm64)] +#[case(Architecture::X86_64)] +#[tokio::test] +async fn test_should_transfer_original_bytes_with_deterministic_headers_and_long_paths( + #[case] architecture: Architecture, +) { + let bootstrap = elf(architecture); + let path = format!("{}{}", "long/".repeat(203), "v".repeat(9)); + assert_eq!(path.len(), 1024); + let (_root, artifact) = source(zip(&[ + ("bootstrap", &bootstrap, 0o755), + (&path, b"original", 0o644), + ])) + .await; + fs::write(artifact.code_root().join("bootstrap"), b"mutated host tree") + .await + .unwrap(); + let hash = artifact.sha256().to_owned(); + let first = { + let artifact = artifact.clone(); + with_reservation(Arc::new(Semaphore::new(1)), deadline(), move |permit| { + convert(artifact, architecture, permit, deadline()) + }) + .await + .unwrap() + }; + let second = with_reservation(Arc::new(Semaphore::new(1)), deadline(), move |permit| { + convert(artifact, architecture, permit, deadline()) + }) + .await + .unwrap(); + assert_eq!(first.bytes(), second.bytes()); + assert_eq!(first.code_sha256(), hash); + assert_eq!(first.architecture(), architecture); + assert!(!format!("{first:?}").contains("original")); + let mut archive = Archive::new(Cursor::new(first.bytes())); + let mut count = 0; + let mut directories = 0; + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + let name = entry.path().unwrap().to_string_lossy().into_owned(); + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).unwrap(); + assert_eq!(entry.header().uid().unwrap(), 0); + assert_eq!(entry.header().gid().unwrap(), 0); + assert_eq!(entry.header().mtime().unwrap(), 0); + if entry.header().entry_type().is_dir() { + assert!(Path::new(&path).starts_with(Path::new(&name))); + assert_eq!(entry.header().mode().unwrap(), 0o555); + assert_eq!(bytes, [] as [u8; 0]); + directories += 1; + continue; + } + match name.as_str() { + "bootstrap" => { + assert_eq!(bytes, bootstrap); + assert_eq!(entry.header().mode().unwrap(), 0o555); + } + name if name == path => { + assert_eq!(bytes, b"original"); + assert_eq!(entry.header().mode().unwrap(), 0o444); + } + _ => panic!("unexpected transfer member"), + } + count += 1; + } + assert_eq!(count, 2); + assert_eq!(directories, 203); +} + +#[rstest] +#[case::magic(0, 0)] +#[case::class(4, 1)] +#[case::endian(5, 2)] +#[case::ident_version(6, 2)] +#[case::object_type(16, 1)] +#[case::object_type_high(17, 1)] +#[case::machine(18, 62)] +#[case::machine_high(19, 1)] +#[case::version(20, 2)] +#[case::version_high(23, 1)] +#[case::header_size(52, 0)] +#[tokio::test] +async fn test_should_reject_invalid_elf_before_returning_a_transfer( + #[case] offset: usize, + #[case] value: u8, +) { + let mut bytes = elf(Architecture::Arm64); + *bytes.get_mut(offset).unwrap() = value; + let (_root, artifact) = source(zip(&[("bootstrap", &bytes, 0o755)])).await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); +} + +#[rstest] +#[case::script(b"#!/bin/sh\nexit 0", 0o755)] +#[case::truncated(b"\x7fELF", 0o755)] +#[case::mach_o(b"\xcf\xfa\xed\xfe", 0o755)] +#[case::not_executable(b"unused", 0o644)] +#[tokio::test] +async fn test_should_reject_non_executable_bootstrap_forms( + #[case] bytes: &[u8], + #[case] mode: u32, +) { + let (_root, artifact) = source(zip(&[("bootstrap", bytes, mode)])).await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); +} + +#[tokio::test] +async fn test_should_accept_pie_but_require_the_complete_current_header() { + let mut bytes = elf(Architecture::Arm64); + *bytes.get_mut(16).unwrap() = 3; + validate_elf(&bytes, Architecture::Arm64).unwrap(); + assert!(validate_elf(bytes.get(..63).unwrap(), Architecture::Arm64).is_err()); + let (_root, artifact) = source(zip(&[("data", b"no bootstrap", 0o644)])).await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); +} + +#[rstest] +#[case::traversal("../outside")] +#[case::absolute("/outside")] +#[case::backslash("folder\\outside")] +#[tokio::test] +async fn test_should_revalidate_hostile_zip_paths(#[case] name: &str) { + let bootstrap = elf(Architecture::Arm64); + let (_root, artifact) = source(zip(&[ + ("bootstrap", &bootstrap, 0o755), + (name, b"hostile", 0o644), + ])) + .await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); +} + +#[rstest] +#[case::symlink(0o120_777)] +#[case::fifo(0o010_644)] +#[case::character_device(0o020_644)] +#[case::block_device(0o060_644)] +#[case::socket(0o140_644)] +#[case::setuid(0o104_755)] +#[tokio::test] +async fn test_should_revalidate_zip_special_file_declarations(#[case] mode: u32) { + let bootstrap = elf(Architecture::Arm64); + let mut bytes = zip(&[("bootstrap", &bootstrap, 0o755)]); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(central + 4..central + 6) + .unwrap() + .copy_from_slice(&((3u16 << 8) | 0x14).to_le_bytes()); + bytes + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(mode << 16).to_le_bytes()); + let original = bytes.clone(); + let (_root, artifact) = source(bytes).await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + assert!(matches!( + store + .store_zip_code("special-file", "$LATEST", &Bytes::from(original)) + .await, + Err(LambdaServiceError::InvalidZipFile { .. }) + )); +} + +#[test] +fn test_should_bound_retained_buffer_capacity_and_reject_without_partial_append() { + let mut writer = BoundedWriter { + bytes: Vec::new(), + deadline: deadline(), + limit: 16, + }; + writer.write_all(b"123456789").unwrap(); + assert!(writer.bytes.capacity() <= 16); + writer.write_all(b"abcdefg").unwrap(); + assert!(writer.bytes.capacity() <= 16); + assert_eq!( + writer.write(b"x").unwrap_err().kind(), + std::io::ErrorKind::FileTooLarge + ); + assert_eq!(writer.bytes, b"123456789abcdefg"); + writer.deadline = Instant::now(); + assert_eq!( + writer.write(b"").unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); +} + +#[tokio::test] +async fn test_should_bind_exactly_two_global_transfer_reservations() { + let bootstrap = elf(Architecture::Arm64); + let (_root, source) = source(zip(&[("bootstrap", &bootstrap, 0o755)])).await; + let first = DockerCodeArchive::prepare(source.clone(), Architecture::Arm64, deadline()) + .await + .unwrap(); + let second = DockerCodeArchive::prepare(source.clone(), Architecture::Arm64, deadline()) + .await + .unwrap(); + let error = DockerCodeArchive::prepare( + source, + Architecture::Arm64, + Instant::now() + Duration::from_millis(50), + ) + .await + .unwrap_err(); + assert!(matches!(error, DockerError::Deadline)); + drop(first); + drop(second); +} + +#[tokio::test] +async fn test_should_hold_buffer_admission_until_the_last_archive_clone_drops() { + let bootstrap = elf(Architecture::Arm64); + let (_root, artifact) = source(zip(&[("bootstrap", &bootstrap, 0o755)])).await; + let admission = Arc::new(Semaphore::new(1)); + let first = with_reservation(admission.clone(), deadline(), move |permit| { + convert(artifact, Architecture::Arm64, permit, deadline()) + }) + .await + .unwrap(); + let clone = first.clone(); + drop(first); + assert_eq!(admission.available_permits(), 0); + let error = with_reservation( + admission.clone(), + Instant::now() + Duration::from_millis(10), + Ok, + ) + .await + .unwrap_err(); + assert!(matches!(error, DockerError::Deadline)); + drop(clone); + assert_eq!(admission.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_retain_canceled_blocking_work_and_classify_panics() { + let admission = Arc::new(Semaphore::new(1)); + let (entered, observed) = oneshot::channel(); + let (release, held) = blocking::channel(); + let task = tokio::spawn(with_reservation( + admission.clone(), + deadline(), + move |permit| { + entered.send(()).unwrap(); + held.recv().unwrap(); + Ok(permit) + }, + )); + observed.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(admission.available_permits(), 0); + release.send(()).unwrap(); + timeout(Duration::from_secs(3), async { + while admission.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let result = with_reservation::<()>(admission.clone(), deadline(), |_| { + panic!("controlled code worker panic") + }) + .await; + assert!(matches!(result, Err(DockerError::Worker))); + assert_eq!(admission.available_permits(), 1); +} + +#[test] +fn test_should_bound_actual_reader_progress_and_expiry() { + let mut reader = CheckedReader { + inner: Cursor::new(vec![0; 8193]), + actual: 0, + remaining: 8193, + deadline: deadline(), + }; + let mut first = vec![0; 16_384]; + assert_eq!(reader.read(&mut first).unwrap(), 8192); + assert_eq!(reader.read(&mut first).unwrap(), 1); + assert!(reader.read(&mut first).is_err()); + reader.deadline = Instant::now(); + assert_eq!( + reader.read(&mut first).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); +} + +#[tokio::test] +async fn test_should_read_empty_directory_streams_before_accepting_their_metadata() { + let bootstrap = elf(Architecture::Arm64); + let mut bytes = zip(&[ + ("bootstrap", &bootstrap, 0o755), + ("directory/", b"\x07", 0o755), + ]); + let local = bytes + .windows(4) + .rposition(|part| part == b"PK\x03\x04") + .unwrap(); + let central = bytes + .windows(4) + .rposition(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(local + 8..local + 10) + .unwrap() + .copy_from_slice(&8u16.to_le_bytes()); + bytes.get_mut(local + 14..local + 18).unwrap().fill(0); + bytes.get_mut(local + 22..local + 26).unwrap().fill(0); + bytes + .get_mut(central + 10..central + 12) + .unwrap() + .copy_from_slice(&8u16.to_le_bytes()); + bytes.get_mut(central + 16..central + 20).unwrap().fill(0); + bytes.get_mut(central + 24..central + 28).unwrap().fill(0); + bytes + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(0o040_755u32 << 16).to_le_bytes()); + let original = bytes.clone(); + let (_root, artifact) = source(bytes).await; + assert!(matches!( + DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()).await, + Err(DockerError::InvalidCode) + )); + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + assert!(matches!( + store + .store_zip_code("directory", "$LATEST", &Bytes::from(original)) + .await, + Err(LambdaServiceError::InvalidZipFile { .. }) + )); +} + +#[rstest] +#[case::crc(false)] +#[case::declared_expansion(true)] +#[tokio::test] +async fn test_should_reject_regular_crc_or_declared_expansion_before_exposing_code( + #[case] oversized: bool, +) { + let bootstrap = elf(Architecture::Arm64); + let mut bytes = zip(&[("bootstrap", &bootstrap, 0o755)]); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + if oversized { + let size = 250u32 * 1024 * 1024 + 1; + bytes + .get_mut(22..26) + .unwrap() + .copy_from_slice(&size.to_le_bytes()); + bytes + .get_mut(central + 24..central + 28) + .unwrap() + .copy_from_slice(&size.to_le_bytes()); + } else { + // Leave the complete checked ELF identity intact and corrupt only CRC. + bytes.get_mut(14..18).unwrap().fill(0); + bytes.get_mut(central + 16..central + 20).unwrap().fill(0); + } + let (_root, artifact) = source(bytes).await; + let error = DockerCodeArchive::prepare(artifact, Architecture::Arm64, deadline()) + .await + .unwrap_err(); + assert!(matches!( + (oversized, error), + (true, DockerError::TooLarge) | (false, DockerError::InvalidCode) + )); +} + +#[tokio::test] +async fn test_should_consume_valid_compressed_empty_directory_and_emit_each_parent_once() { + let bootstrap = elf(Architecture::Arm64); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (name, data) in [ + ("bootstrap", bootstrap.as_slice()), + ("parent/file", b"data".as_slice()), + ("parent/", b"".as_slice()), + ] { + writer + .start_file( + name, + SimpleFileOptions::default() + .unix_permissions(0o755) + .compression_method(CompressionMethod::Deflated), + ) + .unwrap(); + writer.write_all(data).unwrap(); + } + let mut bytes = writer.finish().unwrap().into_inner(); + let central = bytes + .windows(4) + .rposition(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(0o040_755u32 << 16).to_le_bytes()); + let (_root, source) = source(bytes).await; + let code = prepare(source).await; + let mut archive = Archive::new(Cursor::new(code.bytes())); + let names: Vec<_> = archive + .entries() + .unwrap() + .map(|entry| entry.unwrap().path().unwrap().into_owned()) + .collect(); + assert_eq!( + names, + vec![ + Path::new("bootstrap"), + Path::new("parent"), + Path::new("parent/file") + ] + ); +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume.rs b/crates/rustack-lambda-core/src/docker/code_volume.rs new file mode 100644 index 00000000..0b5c2a73 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume.rs @@ -0,0 +1,17 @@ +//! Retained code-volume staging and physical retirement, separate from readiness. + +mod base; +mod entry; +mod owner; +mod stage; +mod types; + +pub(in crate::docker) use base::validate_base; +pub use owner::{CodeVolumeManager, CodeVolumePort}; +pub use types::{ + CodePreparation, CodeRetirement, CodeVolumeConfig, CodeVolumeIdentity, CodeVolumeStatus, + PinnedCode, +}; + +#[cfg(test)] +pub(in crate::docker) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/code_volume/base.rs b/crates/rustack-lambda-core/src/docker/code_volume/base.rs new file mode 100644 index 00000000..3382f6f6 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/base.rs @@ -0,0 +1,48 @@ +//! Shared immutable base policy for startup and retained staging. + +use tokio::time::Instant; + +use crate::{ + artifact::{Architecture, PinnedImage}, + docker::{DockerError, DockerPort}, +}; + +pub(in crate::docker) async fn validate_base( + base: &PinnedImage, + docker: &DockerPort, + deadline: Instant, +) -> Result<(), DockerError> { + let metadata = base.resolved().metadata(); + let reference = metadata.image_id.as_str().parse()?; + let inspected = docker.inspect_image(&reference, deadline).await?; + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let labels = config.labels.as_ref(); + if inspected.id.as_deref() != Some(metadata.image_id.as_str()) + || inspected.os.as_deref() != Some("linux") + || inspected.architecture.as_deref() + != Some(match metadata.architecture { + Architecture::Arm64 => "arm64", + Architecture::X86_64 => "amd64", + }) + || config + .volumes + .as_ref() + .is_some_and(|volumes| !volumes.is_empty()) + || labels.is_some_and(|labels| { + labels.len() > 56 + || labels.iter().any(|(key, value)| { + key.is_empty() + || key.len() > 128 + || value.len() > 256 + || key.starts_with("rustack.lambda.") + || !key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) + || value.chars().any(char::is_control) + }) + }) + { + return Err(DockerError::Quarantined); + } + Ok(()) +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume/entry.rs b/crates/rustack-lambda-core/src/docker/code_volume/entry.rs new file mode 100644 index 00000000..e0b1fb82 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/entry.rs @@ -0,0 +1,265 @@ +//! Above-worker guards, durable-state reconciliation and independent reaping. + +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, watch}, + time::{Instant, sleep}, +}; +use typed_builder::TypedBuilder; + +use super::{CodeVolumeConfig, CodeVolumeIdentity, types::Acceptance}; +use crate::{ + artifact::PinnedImage, + docker::{ + DirectoryLease, DockerCodeArchive, DockerError, DockerPort, + mutation::MutationReceipt, + ownership::{ + CodeProducerGuard, + codes::{CodeEffect, CodeRecord, LoaderState, VolumeState}, + }, + recovery::code::retire_effects, + }, + invocation::CleanupTask, +}; + +#[derive(Debug, Default)] +pub(super) struct Health { + pub(super) physical: AtomicBool, + pub(super) failed: AtomicBool, + pub(super) quarantined: AtomicUsize, + #[cfg(test)] + pub(super) ready: AtomicUsize, + #[cfg(test)] + pub(super) fault: std::sync::atomic::AtomicU8, +} +impl Health { + pub(super) fn check(&self) -> Result<(), DockerError> { + if self.failed.load(Ordering::Acquire) { + Err(DockerError::Worker) + } else if self.quarantined.load(Ordering::Acquire) > 0 { + Err(DockerError::Quarantined) + } else { + Ok(()) + } + } +} +#[derive(Debug, TypedBuilder)] +pub(super) struct Context { + pub(super) docker: DockerPort, + pub(super) directory: DirectoryLease, + pub(super) config: CodeVolumeConfig, + pub(super) stop: watch::Sender, + pub(super) health: Arc, + pub(super) runtime: Handle, + pub(super) cleanup: Semaphore, + pub(super) producer: CodeProducerGuard, +} +impl Context { + #[cfg(test)] + pub(super) fn fault(&self, point: u8) { + if self + .health + .fault + .compare_exchange(point, 0, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + panic!("deliberate code-volume owner fault"); + } + } + pub(super) fn fail(&self) { + self.health.failed.store(true, Ordering::Release); + self.stop.send_replace(true); + } +} +#[derive(Debug, Clone, Default)] +pub(super) struct Receipts { + pub(super) volume: Option, + pub(super) loader: Option, +} +impl Receipts { + fn unissued(&self) -> bool { + self.volume + .as_ref() + .is_none_or(MutationReceipt::definitely_unissued) + && self + .loader + .as_ref() + .is_none_or(MutationReceipt::definitely_unissued) + } +} +#[derive(Debug)] +pub(super) struct Progress { + pub(super) receipts: watch::Sender, + pub(super) active: Arc, + pub(super) clean: watch::Sender, + pub(super) retired: AtomicBool, + pub(super) running: AtomicBool, +} +impl Progress { + pub(super) fn new(clean: watch::Sender) -> Self { + let (receipts, _) = watch::channel(Receipts::default()); + Self { + receipts, + active: Arc::new(AtomicBool::new(false)), + clean, + retired: AtomicBool::new(false), + running: AtomicBool::new(false), + } + } +} +#[derive(Debug, TypedBuilder)] +pub(super) struct Plan { + pub(super) identity: Arc, + pub(super) base: PinnedImage, + pub(super) archive: watch::Sender>, + pub(super) record: CodeRecord, + pub(super) deadline: Instant, + pub(super) cancel: watch::Receiver<()>, +} +#[derive(Debug, TypedBuilder)] +pub(super) struct Entry { + pub(super) plan: Plan, + pub(super) progress: Progress, + pub(super) stage: CleanupTask, + pub(super) outcome: watch::Sender>>, + pub(super) handoff: Acceptance, + pub(super) released: watch::Receiver<()>, + pub(super) _volume: OwnedSemaphorePermit, +} +impl Entry { + pub(super) fn check(&self, context: &Context) -> Result<(), DockerError> { + if Instant::now() >= self.plan.deadline { + return Err(DockerError::Deadline); + } + if *context.stop.borrow() || !self.plan.base.liveness().is_live() { + return Err(DockerError::Stopped); + } + if self.plan.cancel.has_changed().is_err() { + return Err(DockerError::Cancelled); + } + Ok(()) + } + pub(super) async fn retire(&self, context: &Context) { + self.progress.active.store(false, Ordering::Release); + if self.stage.wait().await.is_err() { + context.fail(); + } + self.plan.archive.send_replace(None); + let mut dirty = None; + let mut delay = Duration::from_secs(1); + loop { + if self.cleanup_step(context).await { + self.progress.clean.send_replace(true); + self.handoff.send_modify(|handoff| { + handoff.slot.take(); + }); + return; + } + if dirty.is_none() { + dirty = Some(Dirty::new(context.health.clone())); + } + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + async fn cleanup_step(&self, context: &Context) -> bool { + let Ok(_permit) = context.cleanup.acquire().await else { + return false; + }; + let deadline = context.config.deadline(); + let Ok(records) = context.directory.code_records(deadline).await else { + return false; + }; + let receipts = self.progress.receipts.borrow().clone(); + let Some(mut record) = records + .into_iter() + .find(|r| r.environment == self.plan.record.environment) + else { + return self.progress.retired.load(Ordering::Acquire) || receipts.unissued(); + }; + if !record.same_identity(&self.plan.record) { + return receipts.unissued(); + } + let proof = match (record.volume, record.loader) { + (VolumeState::Uncertain, LoaderState::Planned) => { + receipts.volume.as_ref().map(|r| (CodeEffect::Volume, r)) + } + (VolumeState::Known, LoaderState::Uncertain) => { + receipts.loader.as_ref().map(|r| (CodeEffect::Loader, r)) + } + _ => None, + }; + if let Some((effect, receipt)) = proof.filter(|(_, r)| r.definitely_unissued()) { + if context + .directory + .abandon_code(record.clone(), effect, receipt, deadline) + .await + .is_err() + { + return false; + } + match effect { + CodeEffect::Volume => { + record.volume = VolumeState::Retired; + record.loader = LoaderState::Retired; + } + CodeEffect::Loader => { + record.loader = LoaderState::Retired; + } + } + } + if !matches!( + Box::pin(retire_effects( + &context.directory, + &context.docker, + &mut record, + deadline + )) + .await, + Ok(true) + ) { + return false; + } + // This witness outlives a worker and is published before erase can commit. + self.progress.retired.store(true, Ordering::Release); + #[cfg(test)] + context.fault(6); + let erased = context + .directory + .settle_code(record, deadline) + .await + .is_ok(); + #[cfg(test)] + if erased + && context + .health + .fault + .compare_exchange(7, 0, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return false; + } + erased + } +} +#[derive(Debug)] +struct Dirty(Arc); +impl Dirty { + fn new(health: Arc) -> Self { + health.quarantined.fetch_add(1, Ordering::AcqRel); + Self(health) + } +} +impl Drop for Dirty { + fn drop(&mut self) { + self.0.quarantined.fetch_sub(1, Ordering::AcqRel); + } +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume/owner.rs b/crates/rustack-lambda-core/src/docker/code_volume/owner.rs new file mode 100644 index 00000000..59daa8ee --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/owner.rs @@ -0,0 +1,770 @@ +//! Bounded producer actor and a transferred, non-aborting cleanup ledger. + +use std::{ + collections::HashMap, + fmt, mem, + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinError, JoinHandle, JoinSet}, + time::{Instant, sleep_until, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + CodePreparation, CodeRetirement, CodeVolumeConfig, CodeVolumeIdentity, CodeVolumeStatus, + PinnedCode, + entry::{Context, Entry, Health, Plan, Progress}, + stage, + types::{CodeLease, Consumption, Handoff}, +}; +use crate::{ + artifact::PinnedImage, + docker::{ + DirectoryLease, DockerCodeArchive, DockerError, DockerPort, check_deadline, + connection::stopped, + ownership::codes::{CodeRecord, LoaderState, VolumeState}, + }, + invocation::CleanupTask, +}; + +#[derive(Debug)] +struct NoEffectRetirement(Option>); +impl Drop for NoEffectRetirement { + fn drop(&mut self) { + if let Some(retired) = &self.0 { + retired.send_replace(true); + } + } +} + +#[derive(Debug, TypedBuilder)] +struct Envelope { + retirement: NoEffectRetirement, + identity: CodeVolumeIdentity, + archive: DockerCodeArchive, + base: PinnedImage, + deadline: Instant, + cancel: watch::Receiver<()>, + reply: oneshot::Sender>, + volume: OwnedSemaphorePermit, + handoff: watch::Sender, +} +/// Cloneable bounded code preparation port; retain the manager through cleanup. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct CodeVolumePort { + sender: mpsc::Sender, + slots: Arc, + volumes: Arc, + stop: watch::Sender, + health: Arc, + directory: DirectoryLease, + config: CodeVolumeConfig, +} +impl CodeVolumePort { + #[cfg(test)] + pub(super) fn set_fault(&self, point: u8) { + self.health.fault.store(point, Ordering::Release); + } + #[cfg(test)] + pub(super) fn ready_handoffs(&self) -> usize { + self.health.ready.load(Ordering::Acquire) + } + + /// Observe retained reservations without Docker IO or mailbox admission. + /// + /// ```no_run + /// # fn inspect(port:&rustack_lambda_core::docker::CodeVolumePort) {let status=port.status();} + /// ``` + #[must_use] + pub fn status(&self) -> CodeVolumeStatus { + let worker_failed = self.health.failed.load(Ordering::Acquire); + let quarantined = self.health.quarantined.load(Ordering::Acquire); + CodeVolumeStatus { + volumes: usize::from(self.config.volumes) + .saturating_sub(self.volumes.available_permits()), + preparing: usize::from(self.config.jobs).saturating_sub(self.slots.available_permits()), + quarantined, + worker_failed, + accepting: !worker_failed && quarantined == 0 && !*self.stop.borrow(), + } + } + /// Prepare typed code under the exact source-pin acquisition and deadline. + /// + /// Dropping or expiring the observer leaves accepted work and cleanup owned. + /// The returned lease must outlive every known/uncertain mount consumer. + /// + /// # Errors + /// Returns invalid identity/source/budgets, bounded admission, IO or staging + /// failures. An error does not establish physical retirement. + /// + /// ```no_run + /// # async fn prepare(port:&rustack_lambda_core::docker::CodeVolumePort,code:rustack_lambda_core::docker::DockerCodeArchive,base:rustack_lambda_core::artifact::PinnedImage)->Result<(),rustack_lambda_core::docker::DockerError> { + /// let id=rustack_lambda_core::docker::CodeVolumeIdentity::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; + /// let lease=port.prepare(id,code,base,tokio::time::Instant::now()+std::time::Duration::from_secs(120)).await?; + /// assert_eq!(lease.mount()?.read_only,Some(true)); + /// # Ok(()) } + /// ``` + #[tracing::instrument(skip_all)] + pub async fn prepare( + &self, + identity: CodeVolumeIdentity, + archive: DockerCodeArchive, + base: PinnedImage, + deadline: Instant, + ) -> Result { + self.start_preparation(identity, archive, base, deadline)? + .wait() + .await + } + /// Accept preparation and expose retirement before its fallible result. + /// + /// Parent environments retain their own admission until the retirement + /// receipt succeeds, including when result observation fails or expires. + /// + /// # Errors + /// Returns source, budget, stopped or bounded admission failures before any + /// code effect can be issued; a returned handle owns accepted observation. + /// + /// ```no_run + /// # async fn start(port:&rustack_lambda_core::docker::CodeVolumePort,id:rustack_lambda_core::docker::CodeVolumeIdentity,code:rustack_lambda_core::docker::DockerCodeArchive,base:rustack_lambda_core::artifact::PinnedImage)->Result<(),rustack_lambda_core::docker::DockerError> { + /// let preparation=port.start_preparation(id,code,base,tokio::time::Instant::now()+std::time::Duration::from_secs(120))?; + /// let mut retired=preparation.retirement(); + /// let result=preparation.wait().await; + /// drop(result); + /// retired.wait(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await?; + /// # Ok(()) } + /// ``` + pub fn start_preparation( + &self, + identity: CodeVolumeIdentity, + archive: DockerCodeArchive, + base: PinnedImage, + deadline: Instant, + ) -> Result { + self.start_with_receipt(identity, archive, base, deadline, |_| {}) + } + pub(in crate::docker) fn same_scope(&self, directory: &DirectoryLease) -> bool { + self.directory.same_instance(directory) + } + pub(in crate::docker) fn live(&self) -> bool { + !*self.stop.borrow() + && !self.health.failed.load(Ordering::Acquire) + && !self.sender.is_closed() + } + pub(in crate::docker) fn start_with_receipt( + &self, + identity: CodeVolumeIdentity, + archive: DockerCodeArchive, + base: PinnedImage, + deadline: Instant, + publish: F, + ) -> Result + where + F: FnOnce(CodeRetirement), + { + self.health.check()?; + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|v| !v.is_zero()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(3600) + || !base.belongs_to(&self.directory) + || !base.is_live() + || !base.liveness().is_live() + || archive.architecture() != base.resolved().metadata().architecture + { + return Err(DockerError::Configuration); + } + let slot = self + .slots + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + let volume = self + .volumes + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + let (handoff, _) = watch::channel(Handoff { + state: Consumption::Pending, + slot: Some(slot), + }); + let (cancel, cancelled) = watch::channel(()); + let (reply, response) = oneshot::channel(); + let (retirement, settled) = watch::channel(false); + let retired = CodeRetirement::new(settled); + let request = Envelope::builder() + .retirement(NoEffectRetirement(Some(retirement))) + .identity(identity) + .archive(archive) + .base(base) + .deadline(deadline) + .cancel(cancelled) + .reply(reply) + .volume(volume) + .handoff(handoff) + .build(); + // Publication cannot race accepted work: the envelope is still local. + publish(retired.clone()); + self.sender.try_send(request).map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Stopped, + })?; + Ok(CodePreparation { + result: response, + _cancel: cancel, + retired, + deadline, + }) + } +} +/// Retained producer ownership; stop after all code mount consumers have joined. +#[non_exhaustive] +pub struct CodeVolumeManager { + port: CodeVolumePort, + task: Option>>, + primary: Option>, + emergency: Arc, + settlement: Option>, +} +impl fmt::Debug for CodeVolumeManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CodeVolumeManager") + .field("status", &self.port.status()) + .finish_non_exhaustive() + } +} +impl CodeVolumeManager { + pub(in crate::docker) fn physically_settled(&self) -> bool { + self.settlement.is_some() && self.port.health.physical.load(Ordering::Acquire) + } + + /// Start one producing owner for this acquired code-generation directory. + /// + /// Historical namespace recovery must precede production. No readiness or + /// ordinary ZIP capability is granted by this library constructor. + /// + /// # Errors + /// Rejects missing runtime, foreign/unbound ports, wrong provenance or a + /// previous producer still retaining this acquisition. + /// + /// ```no_run + /// # fn start(port:rustack_lambda_core::docker::DockerPort,directory:rustack_lambda_core::docker::DirectoryLease)->Result<(),rustack_lambda_core::docker::DockerError> { + /// let manager=rustack_lambda_core::docker::CodeVolumeManager::start(port,directory,Default::default())?; + /// # Ok(()) } + /// ``` + pub fn start( + docker: DockerPort, + directory: DirectoryLease, + config: CodeVolumeConfig, + ) -> Result { + docker.require_directory(&directory)?; + let runtime = Handle::try_current().map_err(|_| DockerError::Worker)?; + let producer = directory.code_producer()?; + let (sender, inbox) = mpsc::channel(32); + let (stop, _) = watch::channel(false); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let port = CodeVolumePort::builder() + .sender(sender) + .slots(Arc::new(Semaphore::new(usize::from(config.jobs)))) + .volumes(Arc::new(Semaphore::new(usize::from(config.volumes)))) + .stop(stop.clone()) + .health(health.clone()) + .directory(directory.clone()) + .config(config) + .build(); + let context = Arc::new( + Context::builder() + .docker(docker.owned()) + .directory(directory) + .config(config) + .stop(stop) + .health(health) + .runtime(runtime.clone()) + .cleanup(Semaphore::new(2)) + .producer(producer) + .build(), + ); + let actor = Actor { + ledger: Ledger::new(context), + inbox, + emergency: emergency.clone(), + settled: false, + }; + let task = runtime.spawn(actor.run()); + Ok(Self { + port, + task: Some(task), + primary: None, + emergency, + settlement: None, + }) + } + #[cfg(test)] + pub(super) fn abort_root(&self) { + if let Some(task) = &self.task { + task.abort(); + } + } + + /// Clone a preparation port without transferring manager shutdown ownership. + /// + /// ```no_run + /// # fn inspect(manager:&rustack_lambda_core::docker::CodeVolumeManager) {let port=manager.port();} + /// ``` + #[must_use] + pub fn port(&self) -> CodeVolumePort { + self.port.clone() + } + /// Revoke admission/leases and join original stages and all physical cleanup. + /// + /// First join every known/uncertain function and its IO, then stop this + /// manager, then image ownership and the Docker wire. An expired observer + /// retains the same join task and all outstanding resource reservations. + /// + /// # Errors + /// Returns deadline while retaining work, or cached supervision failure. + /// + /// ```no_run + /// # async fn stop(manager:&mut rustack_lambda_core::docker::CodeVolumeManager) {let result=manager.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await;} + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), DockerError> { + self.port.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + check_deadline(deadline)?; + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Err(DockerError::Worker), + }; + self.primary = Some(result); + self.task.take(); + } + let emergency = timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let result = self + .primary + .clone() + .ok_or(DockerError::Worker)? + .and(emergency); + self.settlement = Some(result.clone()); + result + } +} +impl Drop for CodeVolumeManager { + fn drop(&mut self) { + self.port.stop.send_replace(true); + } +} + +type Request = ( + watch::Sender<()>, + oneshot::Sender>, +); + +type Job = (Uuid, Result<(), DockerError>); +type Joined = Result<(Id, Job), JoinError>; +#[derive(Debug)] +struct Ledger { + context: Arc, + entries: HashMap>, + jobs: JoinSet, + index: HashMap, +} +impl Ledger { + fn new(context: Arc) -> Self { + Self { + context, + entries: HashMap::new(), + jobs: JoinSet::new(), + index: HashMap::new(), + } + } + fn spawn(&mut self, entry: Arc, request: Option) { + let id = entry.plan.identity.environment; + let context = self.context.clone(); + let guard = WorkerGuard::new(entry.clone(), context.clone()); + let job = self.jobs.spawn(async move { + let mut guard = guard; + if let Some((release, reply)) = request { + run(entry, context, release, reply).await; + } else { + tokio::time::sleep(Duration::from_secs(1)).await; + entry.retire(&context).await; + } + guard.complete = true; + (id, Ok(())) + }); + self.index.insert(job.id(), id); + } + fn finished(&mut self, result: &Joined) { + let task = match &result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let indexed = self.index.remove(&task); + let returned = match &result { + Ok((_, (id, _))) => Some(*id), + Err(_) => None, + }; + let Some(id) = indexed.or(returned) else { + self.context.fail(); + return; + }; + let Some(entry) = self.entries.get(&id).cloned() else { + self.context.fail(); + return; + }; + if matches!(result,Ok((_,(returned,Ok(())))) if *returned==id) + && *entry.progress.clean.borrow() + { + self.entries.remove(&id); + } else { + self.context.fail(); + self.spawn(entry, None); + } + } + fn schedule_missing(&mut self) { + let missing = self + .entries + .iter() + .filter(|(id, entry)| { + !entry.progress.running.load(Ordering::Acquire) + && !self.index.values().any(|mapped| mapped == *id) + }) + .map(|(_, entry)| entry.clone()) + .collect::>(); + for entry in missing { + self.spawn(entry, None); + } + } + async fn drain(&mut self) { + loop { + if self.jobs.is_empty() { + self.index.clear(); + } + self.schedule_missing(); + if self.jobs.is_empty() { + self.index.clear(); + if self.entries.is_empty() { + self.context.producer.settle(); + self.context.health.physical.store(true, Ordering::Release); + return; + } + for entry in self.entries.values().cloned().collect::>() { + self.spawn(entry, None); + } + } + if let Some(result) = self.jobs.join_next_with_id().await { + self.finished(&result); + } + } + } +} +#[derive(Debug)] +struct Actor { + ledger: Ledger, + inbox: mpsc::Receiver, + emergency: Arc, + settled: bool, +} +impl Actor { + async fn run(mut self) -> Result<(), DockerError> { + let mut stop = self.ledger.context.stop.subscribe(); + let mut wire_stop = self.ledger.context.docker.raw.stop.subscribe(); + loop { + if *stop.borrow() { + self.inbox.close(); + while let Ok(request) = self.inbox.try_recv() { + let _ = request.reply.send(Err(DockerError::Stopped)); + } + self.ledger.drain().await; + self.settled = true; + return if self.ledger.context.health.failed.load(Ordering::Acquire) { + Err(DockerError::Worker) + } else { + Ok(()) + }; + } + tokio::select! {biased; + ()=stopped(&mut wire_stop)=>self.ledger.context.fail(), + ()=stopped(&mut stop)=>{}, + result=self.ledger.jobs.join_next_with_id(),if !self.ledger.jobs.is_empty()=>if let Some(result)=result {self.ledger.finished(&result);}, + request=self.inbox.recv()=>match request {Some(request)=>self.start(request),None=>{self.ledger.context.stop.send_replace(true);}}, + } + } + } + fn start(&mut self, mut request: Envelope) { + let id = request.identity.environment; + if self.ledger.context.health.check().is_err() + || *self.ledger.context.stop.borrow() + || request.reply.is_closed() + || request.cancel.has_changed().is_err() + || Instant::now() >= request.deadline + || self.ledger.entries.contains_key(&id) + { + let _ = request.reply.send(Err(DockerError::Stopped)); + return; + } + let record = match record(&request, &self.ledger.context.directory) { + Ok(record) => record, + Err(error) => { + let _ = request.reply.send(Err(error)); + return; + } + }; + let Some(retirement) = request.retirement.0.take() else { + self.ledger.context.fail(); + let _ = request.reply.send(Err(DockerError::Worker)); + return; + }; + let (archive, _) = watch::channel(Some(request.archive)); + let (outcome, _) = watch::channel(None); + let (release, released) = watch::channel(()); + let entry = Arc::new( + Entry::builder() + .plan( + Plan::builder() + .identity(Arc::new(request.identity)) + .base(request.base) + .archive(archive) + .record(record) + .deadline(request.deadline) + .cancel(request.cancel) + .build(), + ) + .progress(Progress::new(retirement)) + .stage(CleanupTask::default()) + .outcome(outcome) + .handoff(request.handoff) + .released(released) + ._volume(request.volume) + .build(), + ); + self.ledger.entries.insert(id, entry.clone()); + self.ledger.spawn(entry, Some((release, request.reply))); + #[cfg(test)] + self.ledger.context.fault(5); + } +} +impl Drop for Actor { + fn drop(&mut self) { + if self.settled { + return; + } + self.ledger.context.fail(); + if self.ledger.entries.is_empty() && self.ledger.jobs.is_empty() { + self.ledger.context.producer.settle(); + self.ledger + .context + .health + .physical + .store(true, Ordering::Release); + return; + } + // Move every original worker; never abort IO in order to replace its observer. + let context = self.ledger.context.clone(); + let mut ledger = mem::replace(&mut self.ledger, Ledger::new(context)); + self.emergency + .start_on(&ledger.context.runtime.clone(), move || async move { + ledger.drain().await; + Ok(()) + }); + } +} +#[derive(Debug)] +struct WorkerGuard { + entry: Arc, + context: Arc, + complete: bool, +} +impl WorkerGuard { + fn new(entry: Arc, context: Arc) -> Self { + entry.progress.running.store(true, Ordering::Release); + Self { + entry, + context, + complete: false, + } + } +} +impl Drop for WorkerGuard { + fn drop(&mut self) { + self.entry.progress.running.store(false, Ordering::Release); + if !self.complete { + self.context.fail(); + } + } +} + +fn record(request: &Envelope, directory: &DirectoryLease) -> Result { + let digest = |text: &str| -> Result<[u8; 32], DockerError> { + let mut result = [0; 32]; + hex::decode_to_slice( + text.strip_prefix("sha256:") + .ok_or(DockerError::Configuration)?, + &mut result, + ) + .map_err(|_| DockerError::Configuration)?; + Ok(result) + }; + let zip = STANDARD + .decode(request.archive.code_sha256()) + .map_err(|_| DockerError::InvalidCode)? + .try_into() + .map_err(|_| DockerError::InvalidCode)?; + Ok(CodeRecord::builder() + .run(directory.run_id()) + .environment(request.identity.environment) + .resource(request.identity.resource) + .revision(request.identity.revision) + .zip(zip) + .base(digest( + request.base.resolved().metadata().image_id.as_str(), + )?) + .fingerprint(digest( + request.base.resolved().metadata().inspection_hash.as_str(), + )?) + .volume(VolumeState::Planned) + .loader(LoaderState::Planned) + .build()) +} +async fn run( + entry: Arc, + context: Arc, + release: watch::Sender<()>, + reply: oneshot::Sender>, +) { + #[cfg(test)] + context.fault(1); + stage::start(&entry, context.clone()); + let joined = entry.stage.wait().await; + let staged = if joined.is_err() { + context.fail(); + Err(DockerError::Worker) + } else { + entry + .outcome + .borrow() + .clone() + .unwrap_or(Err(DockerError::Worker)) + }; + #[cfg(test)] + context.fault(2); + let staged = staged.and_then(|()| entry.check(&context)); + match staged { + Ok(()) => { + entry.progress.active.store(true, Ordering::Release); + let ready = CodeLease::builder() + .identity(entry.plan.identity.clone()) + .volume(match entry.plan.record.volume_key() { + Ok(key) => key, + Err(error) => { + let _ = reply.send(Err(error)); + entry.retire(&context).await; + return; + } + }) + .image(entry.plan.base.resolved().metadata().image_id.clone()) + .source(entry.plan.base.liveness()) + .release(release) + .active(entry.progress.active.clone()) + .stop(context.stop.subscribe()) + .handoff(entry.handoff.clone()) + .deadline(entry.plan.deadline) + .retired(entry.progress.clean.subscribe()) + .build(); + if reply.send(Ok(PinnedCode::from_lease(ready))).is_ok() + && consumed(&entry, &context).await + { + let mut released = entry.released.clone(); + let mut stop = context.stop.subscribe(); + let source = entry.plan.base.liveness(); + tokio::select! {biased;()=stopped(&mut stop)=>{},()=source.revoked()=>{},_ = released.changed()=>{}} + } + } + Err(error) => { + let _ = reply.send(Err(error)); + } + } + entry.retire(&context).await; +} +#[cfg(test)] +#[derive(Debug)] +struct ReadyProbe(Arc); +#[cfg(test)] +impl Drop for ReadyProbe { + fn drop(&mut self) { + self.0.ready.fetch_sub(1, Ordering::AcqRel); + } +} + +async fn consumed(entry: &Entry, context: &Context) -> bool { + #[cfg(test)] + let _probe = { + context.health.ready.fetch_add(1, Ordering::AcqRel); + ReadyProbe(context.health.clone()) + }; + let mut changed = entry.handoff.subscribe(); + let mut released = entry.released.clone(); + let mut cancelled = entry.plan.cancel.clone(); + let mut stop = context.stop.subscribe(); + let source = entry.plan.base.liveness(); + let mut abandoned = false; + loop { + let mut decision = None; + entry + .handoff + .send_if_modified(|handoff| match handoff.state { + Consumption::Consumed(at) if at < entry.plan.deadline => { + decision = Some(true); + false + } + Consumption::Expired => { + decision = Some(false); + false + } + _ if abandoned + || Instant::now() >= entry.plan.deadline + || *stop.borrow() + || !source.is_live() + || cancelled.has_changed().is_err() => + { + handoff.state = Consumption::Expired; + decision = Some(false); + true + } + _ => false, + }); + if let Some(decision) = decision { + return decision; + } + tokio::select! {biased; + ()=stopped(&mut stop)=>{},()=source.revoked()=>{},_ = cancelled.changed()=>{}, + _ = released.changed()=>{abandoned=true;},_ = changed.changed()=>{},()=sleep_until(entry.plan.deadline)=>{}, + } + } +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume/stage.rs b/crates/rustack-lambda-core/src/docker/code_volume/stage.rs new file mode 100644 index 00000000..b13e0747 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/stage.rs @@ -0,0 +1,203 @@ +//! One independently retained stage; no observer can abort accepted Docker IO. + +use std::{collections::HashMap, sync::Arc}; + +use tokio::time::Instant; + +use super::entry::{Context, Entry}; +use crate::docker::{ + DockerError, + models::{ContainerCreateBody, HostConfig, Mount, MountType, MountVolumeOptions}, + mutation::MutationGuard, + ownership::codes::{CodeRecord, LoaderState, VolumeState}, + recovery::code::retire_loader, +}; + +pub(super) fn start(entry: &Arc, context: Arc) { + let retained = entry.clone(); + let runtime = context.runtime.clone(); + entry.stage.start_on(&runtime, move || async move { + let result = prepare(&retained, &context).await; + retained.outcome.send_replace(Some(result)); + Ok(()) + }); +} +fn io(entry: &Entry, context: &Context) -> Instant { + entry.plan.deadline.min(context.config.deadline()) +} + +#[tracing::instrument(skip_all)] +async fn prepare(entry: &Entry, context: &Context) -> Result<(), DockerError> { + entry.check(context)?; + super::validate_base(&entry.plan.base, &context.docker, io(entry, context)).await?; + entry.check(context)?; + let mut record = prepare_volume(entry, context).await?; + let mutation = MutationGuard::new(); + entry + .progress + .receipts + .send_modify(|receipts| receipts.loader = Some(mutation.receipt())); + let mut next = record.clone(); + next.loader = LoaderState::Uncertain; + save(entry, context, &mut record, next).await?; + entry.check(context)?; + #[cfg(test)] + context.fault(9); + let body = loader_body(entry, &record, context)?; + let created = context + .docker + .create_container_inner( + &record.loader_key()?, + &body, + Some(entry.plan.base.resolved().metadata().architecture), + io(entry, context), + Some(mutation), + ) + .await?; + let inspected = context + .docker + .inspect_container(&created.id.parse()?, io(entry, context)) + .await?; + let id = record.owned_loader(context.directory.owner_id(), &inspected)?; + let mut next = record.clone(); + next.loader = LoaderState::Known(id); + save(entry, context, &mut record, next).await?; + entry.check(context)?; + #[cfg(test)] + context.fault(4); + let archive = entry + .plan + .archive + .borrow() + .clone() + .ok_or(DockerError::Worker)?; + let uploaded = context + .docker + .upload_code( + &record.loader_key()?, + &archive, + entry + .plan + .deadline + .min(Instant::now() + context.config.upload), + ) + .await; + drop(archive); + // The stage awaits the original call through its wire/driver join. + entry.plan.archive.send_replace(None); + uploaded?; + entry.check(context)?; + if !Box::pin(retire_loader( + &context.directory, + &context.docker, + &mut record, + io(entry, context), + )) + .await? + { + return Err(DockerError::Quarantined); + } + entry.check(context)?; + tracing::debug!(environment_id=%record.environment,"code volume prepared; loader retired"); + Ok(()) +} +async fn prepare_volume(entry: &Entry, context: &Context) -> Result { + let mut record = entry.plan.record.clone(); + context + .directory + .prepare_code(record.clone(), io(entry, context)) + .await?; + entry.check(context)?; + let mutation = MutationGuard::new(); + entry + .progress + .receipts + .send_modify(|receipts| receipts.volume = Some(mutation.receipt())); + let mut next = record.clone(); + next.volume = VolumeState::Uncertain; + save(entry, context, &mut record, next).await?; + entry.check(context)?; + #[cfg(test)] + context.fault(8); + let volume = context + .docker + .create_volume_tracked( + &record.volume_key()?, + &record.labels(context.directory.owner_id(), false), + io(entry, context), + Some(mutation), + ) + .await?; + record.owned_volume(context.directory.owner_id(), &volume)?; + let mut next = record.clone(); + next.volume = VolumeState::Known; + save(entry, context, &mut record, next).await?; + entry.check(context)?; + #[cfg(test)] + context.fault(3); + Ok(record) +} +async fn save( + entry: &Entry, + context: &Context, + record: &mut CodeRecord, + next: CodeRecord, +) -> Result<(), DockerError> { + context + .directory + .advance_code(record.clone(), next.clone(), io(entry, context)) + .await?; + *record = next; + Ok(()) +} + +fn loader_body( + entry: &Entry, + record: &CodeRecord, + context: &Context, +) -> Result { + Ok(ContainerCreateBody { + image: Some( + entry + .plan + .base + .resolved() + .metadata() + .image_id + .as_str() + .into(), + ), + user: Some("65534:65534".into()), + entrypoint: Some(vec!["/never-start-this-loader".into()]), + cmd: Some(Vec::new()), + working_dir: Some("/var/task".into()), + labels: Some( + record + .labels(context.directory.owner_id(), true) + .into_iter() + .collect::>(), + ), + host_config: Some(HostConfig { + network_mode: Some("none".into()), + readonly_rootfs: Some(true), + memory: Some(128 * 1024 * 1024), + nano_cpus: Some(1_000_000_000), + pids_limit: Some(64), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + mounts: Some(vec![Mount { + typ: Some(MountType::VOLUME), + source: Some(record.volume_key()?.as_str().into()), + target: Some("/var/task".into()), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..MountVolumeOptions::default() + }), + ..Mount::default() + }]), + ..HostConfig::default() + }), + ..ContainerCreateBody::default() + }) +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume/tests.rs b/crates/rustack-lambda-core/src/docker/code_volume/tests.rs new file mode 100644 index 00000000..334d5f4c --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/tests.rs @@ -0,0 +1,1168 @@ +use std::{ + sync::{Arc, atomic::AtomicBool}, + time::Duration, +}; + +use rstest::rstest; +use serde_json::{Value, from_slice, json, to_vec}; +use tempfile::TempDir; +use tokio::{ + sync::watch, + time::{Instant, timeout}, +}; +use uuid::Uuid; + +use super::*; +use crate::{ + artifact::{Architecture, ImageLeaseContext, PinnedImage, ResolvedImage}, + docker::{ + DirectoryLease, DockerError, DockerWire, + code::tests::{elf, prepare, source, zip}, + tests::{Daemon, IncomingRequest}, + }, +}; + +pub(in crate::docker) fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn config() -> CodeVolumeConfig { + CodeVolumeConfig::new(2, 2, Duration::from_secs(1), Duration::from_secs(2)).unwrap() +} +pub(in crate::docker) async fn fixture() -> ( + TempDir, + DirectoryLease, + Daemon, + DockerWire, + crate::docker::DockerPort, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + (root, directory, daemon, wire, port) +} +pub(in crate::docker) fn base( + directory: &DirectoryLease, +) -> (PinnedImage, watch::Sender, watch::Receiver<()>) { + let (release, released) = watch::channel(()); + let (stop, stopping) = watch::channel(false); + let base = PinnedImage::with_context( + ResolvedImage::fixture(Architecture::Arm64), + 1024.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopping, + ImageLeaseContext::Owned { + directory: directory.clone(), + preparation: None, + }, + ); + (base, stop, released) +} +pub(in crate::docker) async fn code() -> (TempDir, crate::docker::DockerCodeArchive) { + let (root, artifact) = source(zip(&[("bootstrap", &elf(Architecture::Arm64), 0o755)])).await; + (root, prepare(artifact).await) +} +pub(in crate::docker) fn target(request: &IncomingRequest) -> String { + request + .head + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .into() +} +pub(in crate::docker) async fn reply(request: IncomingRequest, value: &Value) { + request.reply(200, &to_vec(value).unwrap()).await; +} +fn image() -> Value { + json!({"Id":format!("sha256:{}","a".repeat(64)),"Os":"linux","Architecture":"arm64","Size":1024,"Config":{"Labels":{"com.amazonaws.lambda.platform.kernel":"k510ga"}}}) +} +pub(in crate::docker) async fn begin_volume( + daemon: &mut Daemon, + directory: &DirectoryLease, +) -> (IncomingRequest, Value) { + let request = daemon.next().await; + assert!(target(&request).contains("/images/sha256%3A")); + reply(request, &image()).await; + let request = daemon.next().await; + assert!(target(&request).ends_with("/volumes/create")); + let body: Value = from_slice(&request.body).unwrap(); + let saved = directory + .code_records(deadline()) + .await + .unwrap() + .into_iter() + .find(|row| row.volume_key().unwrap().as_str() == body["Name"].as_str().unwrap()) + .unwrap(); + assert_eq!( + saved.volume, + crate::docker::ownership::codes::VolumeState::Uncertain + ); + let volume = json!({"Name":body["Name"],"Labels":body["Labels"],"Driver":"local","Scope":"local","Options":{},"Mountpoint":"/not-a-host-path"}); + (request, volume) +} +async fn begin_loader( + daemon: &mut Daemon, + directory: &DirectoryLease, + volume: &Value, +) -> (IncomingRequest, Value) { + let request = daemon.next().await; + assert!(target(&request).contains("/containers/create?")); + let mut body: Value = from_slice(&request.body).unwrap(); + assert_eq!(body["HostConfig"]["NetworkMode"], "none"); + assert_eq!(body["HostConfig"]["ReadonlyRootfs"], true); + assert_eq!( + body["HostConfig"]["Mounts"][0]["VolumeOptions"]["NoCopy"], + true + ); + let name = form_urlencoded::parse(target(&request).split_once('?').unwrap().1.as_bytes()) + .find(|(k, _)| k == "name") + .unwrap() + .1 + .into_owned(); + let saved = directory + .code_records(deadline()) + .await + .unwrap() + .into_iter() + .find(|row| row.volume_key().unwrap().as_str() == volume["Name"].as_str().unwrap()) + .unwrap(); + assert_eq!( + saved.loader, + crate::docker::ownership::codes::LoaderState::Uncertain + ); + body["Labels"]["com.amazonaws.lambda.platform.kernel"] = json!("k510ga"); + let inspected = json!({"Id":"c".repeat(64),"Name":format!("/{name}"),"Image":body["Image"],"Config":{"Image":body["Image"],"Labels":body["Labels"]},"State":{"Status":"created","Running":false,"StartedAt":"0001-01-01T00:00:00Z"},"Mounts":[{"Type":"volume","Name":volume["Name"],"Driver":"local","Destination":"/var/task","RW":true}]}); + (request, inspected) +} +pub(in crate::docker) async fn allocated( + daemon: &mut Daemon, + directory: &DirectoryLease, +) -> (Value, Value) { + let (request, volume) = begin_volume(daemon, directory).await; + request.reply(201, &to_vec(&volume).unwrap()).await; + let (request, inspected) = begin_loader(daemon, directory, &volume).await; + request + .reply( + 201, + &to_vec(&json!({"Id":"c".repeat(64),"Warnings":[]})).unwrap(), + ) + .await; + reply(daemon.next().await, &inspected).await; + (volume, inspected) +} +pub(in crate::docker) async fn finish_loader(daemon: &mut Daemon, inspected: &Value) { + reply(daemon.next().await, inspected).await; + reply(daemon.next().await, inspected).await; + let request = daemon.next().await; + assert!(request.head.starts_with("DELETE ")); + assert!(target(&request).ends_with("?force=false")); + request.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; +} +pub(in crate::docker) async fn stage(daemon: &mut Daemon, directory: &DirectoryLease) -> Value { + let (volume, inspected) = allocated(daemon, directory).await; + let request = daemon.next().await; + assert!(request.head.starts_with("PUT ")); + assert!( + request + .head + .to_ascii_lowercase() + .contains("content-type: application/x-tar") + ); + assert!(target(&request).contains("copyUIDGID=false")); + request.reply(200, b"").await; + finish_loader(daemon, &inspected).await; + volume +} +pub(in crate::docker) async fn retire(daemon: &mut Daemon, volume: &Value) { + let request = daemon.next().await; + assert!(target(&request).contains("/volumes/")); + reply(request, volume).await; + let request = daemon.next().await; + assert!(request.head.starts_with("DELETE ")); + assert!(target(&request).ends_with("?force=false")); + request.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; +} +pub(in crate::docker) async fn empty(port: &CodeVolumePort) { + timeout(Duration::from_secs(5), async { + while port.status().volumes != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[rstest] +#[case(0, 1, 1, 1)] +#[case(33, 1, 1, 1)] +#[case(1, 2, 1, 1)] +#[case(2, 0, 1, 1)] +#[case(3, 3, 1, 1)] +#[case(2, 2, 0, 1)] +#[case(2, 2, 61, 1)] +#[case(2, 2, 1, 0)] +#[case(2, 2, 1, 601)] +fn test_should_reject_invalid_admission_and_io_settings( + #[case] volumes: u8, + #[case] jobs: u8, + #[case] io: u64, + #[case] upload: u64, +) { + assert!( + CodeVolumeConfig::new( + volumes, + jobs, + Duration::from_secs(io), + Duration::from_secs(upload) + ) + .is_err() + ); +} +#[test] +fn test_should_require_nonnil_resource_revision_and_generate_fresh_environments() { + assert!(CodeVolumeIdentity::new(Uuid::nil(), Uuid::new_v4()).is_err()); + assert!(CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::nil()).is_err()); + let a = CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(); + let b = CodeVolumeIdentity::new(a.resource, a.revision).unwrap(); + assert_ne!(a.environment_id(), b.environment_id()); +} +#[tokio::test] +async fn test_should_exclude_replacement_producers_until_original_owner_joins() { + let (_root, directory, daemon, mut wire, port) = fixture().await; + let mut owner = CodeVolumeManager::start(port.clone(), directory.clone(), config()).unwrap(); + assert!(CodeVolumeManager::start(port.clone(), directory.clone(), config()).is_err()); + let stale = owner.port(); + owner.shutdown(deadline()).await.unwrap(); + assert!(!stale.status().accepting); + let mut next = CodeVolumeManager::start(port, directory, config()).unwrap(); + next.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_prepare_clone_and_retire_code_only_after_last_ready_release() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, mut released) = base(&directory); + let id = CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(); + let env = id.environment_id(); + let (ready, volume) = tokio::join!( + port.prepare(id, archive, base, deadline()), + stage(&mut daemon, &directory) + ); + let ready = ready.unwrap(); + assert_eq!(ready.identity().environment_id(), env); + assert_eq!(ready.mount().unwrap().read_only, Some(true)); + assert!(ready.is_live()); + assert_eq!(port.status().volumes, 1); + assert_eq!(port.status().preparing, 0); + assert_eq!(port.status().quarantined, 0); + let clone = ready.clone(); + drop(ready); + assert!(!released.has_changed().unwrap()); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + drop(clone); + retire(&mut daemon, &volume).await; + empty(&port).await; + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + owner.shutdown(deadline()).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +async fn queued(port: &CodeVolumePort) { + timeout(Duration::from_secs(3), async { + while port.ready_handoffs() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +fn poll_once(future: std::pin::Pin<&mut F>) -> std::task::Poll { + let waker = std::task::Waker::noop(); + let mut context = std::task::Context::from_waker(waker); + future.poll(&mut context) +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_expire_or_abandon_ready_without_repolling_its_live_observer( + #[case] abandon: bool, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, mut released) = base(&directory); + let at = Instant::now() + Duration::from_secs(30); + let mut waiting = Box::pin(port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + at, + )); + assert!(poll_once(waiting.as_mut()).is_pending()); + let volume = stage(&mut daemon, &directory).await; + queued(&port).await; + assert_eq!(port.status().preparing, 1); + assert!(!released.has_changed().unwrap()); + if abandon { + drop(waiting); + } else { + // Advance only after physical staging and Ready publication are observed. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(31)).await; + tokio::time::resume(); + // This future stays alive, with a queued Ready value, throughout physical cleanup. + let ((), ()) = tokio::join!(retire(&mut daemon, &volume), empty(&port)); + assert_eq!(port.status().preparing, 0); + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + assert!(matches!(waiting.await, Err(DockerError::Deadline))); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; + return; + } + retire(&mut daemon, &volume).await; + empty(&port).await; + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + assert_eq!(port.status().preparing, 0); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_honor_consumption_before_deadline_even_when_owner_processes_it_later() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let mut waiting = Box::pin(port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + )); + assert!(poll_once(waiting.as_mut()).is_pending()); + let volume = stage(&mut daemon, &directory).await; + queued(&port).await; + let std::task::Poll::Ready(Ok(ready)) = poll_once(waiting.as_mut()) else { + panic!("queued on-time handoff not consumed"); + }; + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(11)).await; + tokio::time::resume(); + timeout(Duration::from_secs(3), async { + while port.ready_handoffs() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(ready.is_live()); + assert_eq!(port.status().volumes, 1); + assert_eq!(port.status().preparing, 0); + drop(ready); + retire(&mut daemon, &volume).await; + empty(&port).await; + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_equal_uuid_foreign_acquisition_and_bad_deadlines_before_io() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let foreign = DirectoryLease::testing(directory.owner_id(), directory.run_id()); + let (foreign_base, _foreign_stop, _) = base(&foreign); + assert!(matches!( + port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive.clone(), + foreign_base, + deadline() + ) + .await, + Err(DockerError::Configuration) + )); + let (base, _stop, _) = base(&directory); + for at in [Instant::now(), Instant::now() + Duration::from_secs(3601)] { + assert!(matches!( + port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive.clone(), + base.clone(), + at + ) + .await, + Err(DockerError::Configuration | DockerError::Deadline) + )); + } + assert_eq!(port.status().volumes, 0); + assert_eq!(port.status().preparing, 0); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_revoke_ready_and_retire_when_the_source_owner_revokes_its_pin() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, stop, mut released) = base(&directory); + let (ready, volume) = tokio::join!( + port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline() + ), + stage(&mut daemon, &directory) + ); + let ready = ready.unwrap(); + stop.send_replace(true); + assert!(!ready.is_live()); + assert!(ready.mount().is_err()); + retire(&mut daemon, &volume).await; + empty(&port).await; + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + drop(ready); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_join_original_accepted_upload_before_cleanup_after_expired_shutdown_or_root_loss( + #[case] root_loss: bool, +) { + let (_root, directory, mut daemon, mut wire, mut docker) = fixture().await; + let (entered, mut observed) = watch::channel(false); + let (release, released) = watch::channel(false); + docker.raw.upload_completion = Some(crate::docker::CompletionGate { + entered, + release: released, + }); + let mut owner = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::new(2, 2, Duration::from_secs(1), Duration::from_secs(1)).unwrap(), + ) + .unwrap(); + let port = owner.port(); + let reservation = Arc::new(tokio::sync::Semaphore::new(2)); + let (_source, artifact) = source(zip(&[("bootstrap", &elf(Architecture::Arm64), 0o755)])).await; + let archive = crate::docker::code::tests::prepare_with(artifact, reservation.clone()).await; + let (base, _stop, mut source_released) = base(&directory); + let call = port.clone(); + let waiting = tokio::spawn(async move { + call.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .await + }); + let (volume, inspected) = allocated(&mut daemon, &directory).await; + let upload = daemon.next().await; + assert!(upload.head.starts_with("PUT ")); + timeout(Duration::from_secs(3), observed.wait_for(|v| *v)) + .await + .unwrap() + .unwrap(); + // The original envelope is retained after its IO deadline and driver join, + // before response delivery; a timed observer must not replace that ownership. + assert_eq!(reservation.available_permits(), 1); + assert_eq!(port.status().volumes, 1); + assert_eq!(port.status().preparing, 1); + assert!(!source_released.has_changed().unwrap()); + if root_loss { + owner.abort_root(); + } else { + waiting.abort(); + } + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(30)) + .await, + Err(DockerError::Deadline) + )); + assert!(CodeVolumeManager::start(docker, directory.clone(), config()).is_err()); + assert!( + timeout(Duration::from_millis(30), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!(reservation.available_permits(), 1); + assert_eq!(port.status().preparing, 1); + release.send_replace(true); + drop(upload); + finish_loader(&mut daemon, &inspected).await; + retire(&mut daemon, &volume).await; + empty(&port).await; + assert_eq!(reservation.available_permits(), 2); + assert_eq!(port.status().preparing, 0); + let _ = source_released.changed().await; + assert!(source_released.has_changed().is_err()); + if root_loss { + assert!(matches!( + waiting.await.unwrap(), + Err(DockerError::Deadline | DockerError::Stopped | DockerError::Worker) + )); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + } else { + assert!(waiting.await.unwrap_err().is_cancelled()); + owner.shutdown(deadline()).await.unwrap(); + } + assert!(owner.physically_settled()); + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_retain_issued_unknown_create_until_late_matching_observation( + #[case] loader: bool, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker.clone(), directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, mut source_released) = base(&directory); + let call = port.clone(); + let waiting = tokio::spawn(async move { + call.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .await + }); + let (volume_request, volume) = begin_volume(&mut daemon, &directory).await; + let (held, inspected) = if loader { + volume_request.reply(201, &to_vec(&volume).unwrap()).await; + let (held, inspected) = begin_loader(&mut daemon, &directory, &volume).await; + (held, Some(inspected)) + } else { + (volume_request, None) + }; + assert!(matches!(waiting.await.unwrap(), Err(DockerError::Deadline))); + drop(held); + daemon.next().await.reply(404, b"{}").await; + timeout(Duration::from_secs(3), async { + while port.status().quarantined == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(port.status().volumes, 1); + assert_eq!(port.status().preparing, 1); + assert!(!source_released.has_changed().unwrap()); + assert!(CodeVolumeManager::start(docker, directory.clone(), config()).is_err()); + if let Some(inspected) = inspected { + reply(daemon.next().await, &inspected).await; + // Unknown name is now persisted as a known full ID before the recheck/delete. + reply(daemon.next().await, &inspected).await; + let request = daemon.next().await; + assert!(request.head.starts_with("DELETE ")); + assert!(target(&request).contains(&"c".repeat(64))); + request.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + } else { + reply(daemon.next().await, &volume).await; + } + retire(&mut daemon, &volume).await; + empty(&port).await; + assert_eq!(port.status().preparing, 0); + assert_eq!(port.status().quarantined, 0); + assert!(!port.status().worker_failed); + let _ = source_released.changed().await; + assert!(source_released.has_changed().is_err()); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_create_owned_after_cancel_then_retire_positive_ack_without_loader_creation() + { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, mut source_released) = base(&directory); + let call = port.clone(); + let waiting = tokio::spawn(async move { + call.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .await + }); + let (request, volume) = begin_volume(&mut daemon, &directory).await; + waiting.abort(); + assert!(waiting.await.unwrap_err().is_cancelled()); + assert_eq!(port.status().preparing, 1); + assert!(!source_released.has_changed().unwrap()); + request.reply(201, &to_vec(&volume).unwrap()).await; + retire(&mut daemon, &volume).await; + empty(&port).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + let _ = source_released.changed().await; + assert!(source_released.has_changed().is_err()); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(1)] +#[case(5)] +#[case(8)] +#[tokio::test] +async fn test_should_reap_worker_root_and_unissued_stage_panics_without_creating_resources( + #[case] point: u8, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + port.set_fault(point); + let (_source, archive) = code().await; + let (base, _stop, mut source_released) = base(&directory); + let call = port.clone(); + let waiting = tokio::spawn(async move { + call.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .await + }); + if point == 8 { + reply(daemon.next().await, &image()).await; + } + assert!(matches!( + waiting.await.unwrap(), + Err(DockerError::Worker | DockerError::Stopped) + )); + empty(&port).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + assert_eq!(port.status().preparing, 0); + assert!(port.status().worker_failed); + let _ = source_released.changed().await; + assert!(source_released.has_changed().is_err()); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(3)] +#[case(9)] +#[tokio::test] +async fn test_should_reap_stage_panics_after_volume_commit_or_unissued_loader(#[case] point: u8) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + port.set_fault(point); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let call = port.clone(); + let waiting = tokio::spawn(async move { + call.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .await + }); + let (request, volume) = begin_volume(&mut daemon, &directory).await; + request.reply(201, &to_vec(&volume).unwrap()).await; + assert!(matches!(waiting.await.unwrap(), Err(DockerError::Worker))); + retire(&mut daemon, &volume).await; + empty(&port).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(6)] +#[case(7)] +#[tokio::test] +async fn test_should_retain_retirement_witness_across_cleanup_panic_or_lost_erase_ack( + #[case] point: u8, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, mut source_released) = base(&directory); + let (ready, volume) = tokio::join!( + port.prepare( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline() + ), + stage(&mut daemon, &directory) + ); + let ready = ready.unwrap(); + port.set_fault(point); + drop(ready); + retire(&mut daemon, &volume).await; + // The reaper must retain quota/source after physical absence until the + // original erased/retired journal state is positively reconciled. + timeout(Duration::from_secs(3), async { + while if point == 6 { + !port.status().worker_failed + } else { + port.status().quarantined == 0 + } { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(port.status().volumes, 1); + assert!(!source_released.has_changed().unwrap()); + empty(&port).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + let _ = source_released.changed().await; + assert!(source_released.has_changed().is_err()); + let result = owner.shutdown(deadline()).await; + if point == 6 { + assert!(matches!(result, Err(DockerError::Worker))); + } else { + result.unwrap(); + } + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(true)] +#[case(false)] +#[tokio::test(start_paused = true)] +async fn test_should_reject_retirement_completion_observed_after_deadline_and_allow_reobserve( + #[case] positive: bool, +) { + let (settled, received) = watch::channel(false); + let mut receipt = CodeRetirement::new(received); + let mut waiting = Box::pin(receipt.wait(Instant::now() + Duration::from_secs(1))); + assert!(poll_once(waiting.as_mut()).is_pending()); + if positive { + settled.send_replace(true); + } + drop(settled); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!( + poll_once(waiting.as_mut()), + std::task::Poll::Ready(Err(DockerError::Deadline)) + )); + drop(waiting); + let result = receipt.wait(deadline()).await; + if positive { + result.unwrap(); + } else { + assert!(matches!(result, Err(DockerError::Worker))); + } +} + +#[tokio::test] +async fn test_should_publish_no_effect_retirement_when_stopping_queued_preparation() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let preparation = port + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .unwrap(); + let mut retired = preparation.retirement(); + // No await between enqueue and stop: biased stop discards this unissued envelope. + owner.shutdown(deadline()).await.unwrap(); + retired.wait(deadline()).await.unwrap(); + assert!(preparation.wait().await.is_err()); + assert_eq!(port.status().volumes, 0); + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_observe_same_physical_retirement_after_failed_prepare_and_expired_receipt() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let preparation = port + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .unwrap(); + let mut retired = preparation.retirement(); + let (request, volume) = begin_volume(&mut daemon, &directory).await; + drop(preparation); + request.reply(201, &to_vec(&volume).unwrap()).await; + let inspect = daemon.next().await; + assert!(matches!( + retired + .wait(Instant::now() + Duration::from_millis(10)) + .await, + Err(DockerError::Deadline) + )); + assert_eq!(port.status().volumes, 1); + reply(inspect, &volume).await; + let delete = daemon.next().await; + assert!(delete.head.starts_with("DELETE ")); + delete.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + retired.wait(deadline()).await.unwrap(); + empty(&port).await; + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(2)] +#[case(4)] +#[tokio::test] +async fn test_should_reap_panics_with_known_loader_without_starting_it(#[case] point: u8) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + port.set_fault(point); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let preparation = port + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .unwrap(); + let mut retired = preparation.retirement(); + let waiting = tokio::spawn(preparation.wait()); + let volume = if point == 2 { + stage(&mut daemon, &directory).await + } else { + let (volume, inspected) = allocated(&mut daemon, &directory).await; + finish_loader(&mut daemon, &inspected).await; + volume + }; + assert!(matches!(waiting.await.unwrap(), Err(DockerError::Worker))); + retire(&mut daemon, &volume).await; + retired.wait(deadline()).await.unwrap(); + empty(&port).await; + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(0)] +#[case(1)] +#[case(2)] +#[case(3)] +#[case(4)] +#[case(5)] +#[case(6)] +#[case(7)] +#[case(8)] +#[tokio::test] +async fn test_should_reject_unsafe_or_changed_base_before_durable_or_daemon_effects( + #[case] invalid: u8, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let preparation = port + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .unwrap(); + let mut retired = preparation.retirement(); + let waiting = tokio::spawn(preparation.wait()); + let mut value = image(); + match invalid { + 0 => value["Id"] = json!(format!("sha256:{}", "b".repeat(64))), + 1 => value["Os"] = json!("windows"), + 2 => value["Architecture"] = json!("amd64"), + 3 => value["Config"]["Volumes"] = json!({"/inherited":{}}), + 4 => value["Config"]["Labels"] = json!({"rustack.lambda.ownerId":"foreign"}), + 5 => value["Config"]["Labels"] = json!({"bad/key":"value"}), + 6 => value["Config"]["Labels"] = json!({"key":"x".repeat(257)}), + 7 => value["Config"]["Labels"] = json!({"key":"line\nbreak"}), + _ => { + value["Config"]["Labels"] = + Value::Object((0..57).map(|i| (format!("label{i}"), json!("v"))).collect()); + } + } + reply(daemon.next().await, &value).await; + assert!(matches!( + waiting.await.unwrap(), + Err(DockerError::Quarantined) + )); + retired.wait(deadline()).await.unwrap(); + empty(&port).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_queued_admission_and_reject_architecture_before_io() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let port = owner.port(); + let (_source, archive) = code().await; + let (base, _stop, _released) = base(&directory); + let (_other, artifact) = source(zip(&[("bootstrap", &elf(Architecture::X86_64), 0o755)])).await; + let foreign = + crate::docker::DockerCodeArchive::prepare(artifact, Architecture::X86_64, deadline()) + .await + .unwrap(); + let identity = || CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(); + assert!(matches!( + port.start_preparation(identity(), foreign, base.clone(), deadline()), + Err(DockerError::Configuration) + )); + let first = port + .start_preparation(identity(), archive.clone(), base.clone(), deadline()) + .unwrap(); + let second = port + .start_preparation(identity(), archive.clone(), base.clone(), deadline()) + .unwrap(); + assert!(matches!( + port.start_preparation(identity(), archive, base, deadline()), + Err(DockerError::Busy) + )); + assert_eq!(port.status().volumes, 2); + assert_eq!(port.status().preparing, 2); + let mut one = first.retirement(); + let mut two = second.retirement(); + owner.shutdown(deadline()).await.unwrap(); + one.wait(deadline()).await.unwrap(); + two.wait(deadline()).await.unwrap(); + assert_eq!(port.status().volumes, 0); + assert_eq!(port.status().preparing, 0); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_poison_same_acquisition_after_runtime_loss_with_an_issued_uncertain_create() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let ( + root, + directory, + old_daemon, + mut old_wire, + mut owner, + preparation, + held, + source, + stop, + saved, + ) = runtime.block_on(async { + let (root, directory, mut daemon, wire, docker) = fixture().await; + let owner = CodeVolumeManager::start(docker, directory.clone(), config()).unwrap(); + let (source, archive) = code().await; + let (base, stop, _released) = base(&directory); + let preparation = owner + .port() + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + base, + deadline(), + ) + .unwrap(); + let (held, _) = begin_volume(&mut daemon, &directory).await; + let saved = directory.code_records(deadline()).await.unwrap(); + assert_eq!(saved.len(), 1); + assert_eq!( + saved.first().unwrap().volume, + crate::docker::ownership::codes::VolumeState::Uncertain + ); + ( + root, + directory, + daemon, + wire, + owner, + preparation, + held, + source, + stop, + saved, + ) + }); + let journal = runtime + .block_on(tokio::fs::read( + root.path().join(".rustack-lambda-docker").join("code.v1"), + )) + .unwrap(); + drop(runtime); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + assert!(matches!( + directory.code_producer(), + Err(crate::docker::DirectoryLeaseError::InvalidState) + )); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let docker = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + assert!(CodeVolumeManager::start(docker, directory.clone(), config()).is_err()); + assert_eq!(directory.code_records(deadline()).await.unwrap(), saved); + assert_eq!( + tokio::fs::read(root.path().join(".rustack-lambda-docker").join("code.v1")) + .await + .unwrap(), + journal + ); + let mut receipt = preparation.retirement(); + assert!(matches!(preparation.wait().await, Err(DockerError::Worker))); + assert!(matches!( + receipt.wait(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert!(!owner.physically_settled()); + assert!(old_wire.shutdown().await.is_err()); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; + }); + drop((held, old_daemon, source, stop)); +} diff --git a/crates/rustack-lambda-core/src/docker/code_volume/types.rs b/crates/rustack-lambda-core/src/docker/code_volume/types.rs new file mode 100644 index 00000000..340029b6 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/code_volume/types.rs @@ -0,0 +1,349 @@ +//! Validated code ownership inputs and revocable metadata-only Ready leases. + +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use tokio::{ + sync::{OwnedSemaphorePermit, oneshot, watch}, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::{ + artifact::{ImageLiveness, Sha256Digest}, + docker::{ + DockerError, DockerKey, check_deadline, + models::{Mount, MountType, MountVolumeOptions}, + }, +}; + +/// Immutable identities shared by the code volume and its eventual environment. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct CodeVolumeIdentity { + pub(super) environment: Uuid, + pub(super) resource: Uuid, + pub(super) revision: Uuid, +} +impl CodeVolumeIdentity { + /// Validate resource/revision UUIDs and allocate a fresh environment identity. + /// + /// # Errors + /// Returns Configuration for a nil identity. + /// + /// ``` + /// let id = rustack_lambda_core::docker::CodeVolumeIdentity::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(resource: Uuid, revision: Uuid) -> Result { + if [resource, revision].iter().any(Uuid::is_nil) { + return Err(DockerError::Configuration); + } + Ok(Self { + environment: Uuid::new_v4(), + resource, + revision, + }) + } + /// Environment UUID used in volume/loader names. + /// + /// ```no_run + /// # fn inspect(id:rustack_lambda_core::docker::CodeVolumeIdentity) { assert!(!id.environment_id().is_nil()); } + /// ``` + #[must_use] + pub fn environment_id(&self) -> Uuid { + self.environment + } +} + +/// Independent bounded resource, preparation and wire budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct CodeVolumeConfig { + pub(super) volumes: u8, + pub(super) jobs: u8, + pub(super) io: Duration, + pub(super) upload: Duration, +} +impl Default for CodeVolumeConfig { + fn default() -> Self { + Self { + volumes: 32, + jobs: 2, + io: Duration::from_secs(5), + upload: Duration::from_secs(60), + } + } +} +impl CodeVolumeConfig { + /// Validate volume1..32/job1..2 admission and IO1..60s/upload1..600s. + /// + /// # Errors + /// Returns Configuration for out-of-range or jobs-exceed-volumes settings. + /// + /// ``` + /// let config=rustack_lambda_core::docker::CodeVolumeConfig::new(32,2,std::time::Duration::from_secs(5),std::time::Duration::from_secs(60))?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(volumes: u8, jobs: u8, io: Duration, upload: Duration) -> Result { + if !(1..=32).contains(&volumes) + || !(1..=2).contains(&jobs) + || jobs > volumes + || !(Duration::from_secs(1)..=Duration::from_secs(60)).contains(&io) + || !(Duration::from_secs(1)..=Duration::from_secs(600)).contains(&upload) + { + return Err(DockerError::Configuration); + } + Ok(Self { + volumes, + jobs, + io, + upload, + }) + } + pub(super) fn deadline(self) -> Instant { + Instant::now() + self.io + } +} + +/// Retained ownership counts, independent of Docker observation/admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct CodeVolumeStatus { + /// Queued, preparing, live or quarantined volume reservations. + pub volumes: usize, + /// Preparation reservations retained until consumption or physical cleanup. + pub preparing: usize, + /// Reapers retaining unresolved physical/durable effects. + pub quarantined: usize, + /// Whether the owner permits new preparations (capacity may still be full). + pub accepting: bool, + /// A permanent supervision failure closed this owner. + pub worker_failed: bool, +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum Consumption { + Pending, + Consumed(Instant), + Expired, +} +#[derive(Debug)] +pub(super) struct Handoff { + pub(super) state: Consumption, + pub(super) slot: Option, +} +pub(super) type Acceptance = watch::Sender; + +/// Metadata-only physical/journal retirement observation; it owns no resources. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct CodeRetirement { + settled: watch::Receiver, +} +impl CodeRetirement { + pub(super) fn new(settled: watch::Receiver) -> Self { + Self { settled } + } + /// Observe the same cleanup without cancelling or restarting it. + /// + /// # Errors + /// Returns Deadline on observer expiry, or Worker if ownership vanished + /// without a positive retirement proof. + /// + /// ```no_run + /// # async fn observe(mut receipt:rustack_lambda_core::docker::CodeRetirement)->Result<(),rustack_lambda_core::docker::DockerError> {receipt.wait(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await?;Ok(())} + /// ``` + pub async fn wait(&mut self, deadline: Instant) -> Result<(), DockerError> { + check_deadline(deadline)?; + let result = timeout_at(deadline, async { + loop { + if *self.settled.borrow() { + return Ok(()); + } + if self.settled.changed().await.is_err() { + return if *self.settled.borrow() { + Ok(()) + } else { + Err(DockerError::Worker) + }; + } + } + }) + .await + .map_err(|_| DockerError::Deadline)?; + check_deadline(deadline)?; + result + } +} + +/// Accepted result/cancellation observation with a separate physical receipt. +#[derive(Debug)] +#[non_exhaustive] +pub struct CodePreparation { + pub(super) result: oneshot::Receiver>, + pub(super) _cancel: watch::Sender<()>, + pub(super) retired: CodeRetirement, + pub(super) deadline: Instant, +} +impl CodePreparation { + /// Retain cleanup evidence before observing a result which may fail/expire. + /// + /// ```no_run + /// # fn inspect(preparation:&rustack_lambda_core::docker::CodePreparation) {let retired=preparation.retirement();} + /// ``` + #[must_use] + pub fn retirement(&self) -> CodeRetirement { + self.retired.clone() + } + /// Consume an on-time Ready result; dropping this observer requests cancel. + /// + /// # Errors + /// Returns the preparation failure, expiry, cancellation or supervisor error. + /// None proves physical retirement; observe the separate receipt for that. + /// + /// ```no_run + /// # async fn ready(preparation:rustack_lambda_core::docker::CodePreparation)->Result<(),rustack_lambda_core::docker::DockerError> {let code=preparation.wait().await?;assert!(code.is_live());Ok(())} + /// ``` + pub async fn wait(mut self) -> Result { + let ready = timeout_at(self.deadline, &mut self.result) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)??; + ready.accept()?; + Ok(ready) + } +} + +/// Revocable code metadata; source and physical admission live in the actor. +#[derive(Clone)] +#[non_exhaustive] +pub struct PinnedCode(Arc); +#[derive(Debug, TypedBuilder)] +pub(super) struct CodeLease { + identity: Arc, + volume: DockerKey, + image: Sha256Digest, + source: ImageLiveness, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, + handoff: Acceptance, + deadline: Instant, + retired: watch::Receiver, +} +impl Drop for CodeLease { + fn drop(&mut self) { + self.release.send_replace(()); + } +} +impl fmt::Debug for PinnedCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PinnedCode") + .field("live", &self.is_live()) + .finish_non_exhaustive() + } +} +impl PinnedCode { + pub(super) fn from_lease(lease: CodeLease) -> Self { + Self(Arc::new(lease)) + } + pub(super) fn accept(&self) -> Result<(), DockerError> { + let mut accepted = false; + self.0.handoff.send_modify(|handoff| { + let at = Instant::now(); + if matches!(handoff.state, Consumption::Pending) + && at < self.0.deadline + && self.is_live() + { + handoff.state = Consumption::Consumed(at); + handoff.slot.take(); + accepted = true; + } + }); + if accepted { + Ok(()) + } else if Instant::now() >= self.0.deadline { + Err(DockerError::Deadline) + } else { + Err(DockerError::Stopped) + } + } + /// Whether this code and its source/manager still permit consumption. + /// + /// ```no_run + /// # fn inspect(code:&rustack_lambda_core::docker::PinnedCode) { let live=code.is_live(); } + /// ``` + #[must_use] + pub fn is_live(&self) -> bool { + self.0.active.load(Ordering::Acquire) && !*self.0.stop.borrow() && self.0.source.is_live() + } + /// Exact immutable environment/resource/revision identity. + /// + /// ```no_run + /// # fn inspect(code:&rustack_lambda_core::docker::PinnedCode) { let id=code.identity(); } + /// ``` + #[must_use] + pub fn identity(&self) -> &CodeVolumeIdentity { + &self.0.identity + } + /// Immutable parent image ID whose pin the actor retains through retirement. + /// + /// ```no_run + /// # fn inspect(code:&rustack_lambda_core::docker::PinnedCode) { let id=code.base_image_id(); } + /// ``` + #[must_use] + pub fn base_image_id(&self) -> &Sha256Digest { + &self.0.image + } + /// Named volume for diagnostics; observation does not confer ownership. + /// + /// ```no_run + /// # fn inspect(code:&rustack_lambda_core::docker::PinnedCode) { let name=code.volume_name(); } + /// ``` + #[must_use] + pub fn volume_name(&self) -> &DockerKey { + &self.0.volume + } + /// Observe retirement after releasing all Ready clones and their consumers. + /// + /// ```no_run + /// # async fn release(code:rustack_lambda_core::docker::PinnedCode)->Result<(),rustack_lambda_core::docker::DockerError> {let mut retired=code.retirement();drop(code);retired.wait(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await?;Ok(())} + /// ``` + #[must_use] + pub fn retirement(&self) -> CodeRetirement { + CodeRetirement::new(self.0.retired.clone()) + } + + /// Return only the checked RO code mount, retaining this lease in its consumer. + /// + /// # Errors + /// Returns Stopped if this lease, manager or source has been revoked. + /// + /// ```no_run + /// # fn inspect(code:&rustack_lambda_core::docker::PinnedCode)->Result<(),rustack_lambda_core::docker::DockerError> { let mount=code.mount()?;assert_eq!(mount.read_only,Some(true));Ok(()) } + /// ``` + pub fn mount(&self) -> Result { + if !self.is_live() { + return Err(DockerError::Stopped); + } + Ok(Mount { + typ: Some(MountType::VOLUME), + source: Some(self.0.volume.as_str().into()), + target: Some("/var/task".into()), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..MountVolumeOptions::default() + }), + ..Mount::default() + }) + } +} diff --git a/crates/rustack-lambda-core/src/docker/config.rs b/crates/rustack-lambda-core/src/docker/config.rs new file mode 100644 index 00000000..433a1bdd --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/config.rs @@ -0,0 +1,420 @@ +//! Pure operator settings; private credential IO has separate retained ownership. + +mod auth; + +use std::{ + fmt, + path::{Component, Path, PathBuf}, + str::FromStr, + time::Duration, +}; + +pub use auth::RegistryAuthError; +use rustack_core::settings::RuntimeBudgets; +use rustack_lambda_runtime::probe::GatewayEndpoint; +use tokio::time::Instant; +use typed_builder::TypedBuilder; + +use super::{ + ContainerPolicy, DockerError, DockerSocket, ImagePullPolicy, NetworkSelection, + RegistryCredentials, RegistryHost, RelayBudgets, sdk::LocalSdkCredentials, +}; +use crate::artifact::ImageReference; + +/// Operator-selected function endpoint, independent of request Host and readiness. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct DockerGatewayEndpoint(GatewayEndpoint); +impl fmt::Debug for DockerGatewayEndpoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DockerGatewayEndpoint()") + } +} +impl FromStr for DockerGatewayEndpoint { + type Err = DockerError; + fn from_str(raw: &str) -> Result { + raw.parse() + .map(Self) + .map_err(|_| DockerError::Configuration) + } +} +impl DockerGatewayEndpoint { + /// Canonical operator endpoint; this does not establish network reachability. + /// ``` + /// let endpoint:rustack_lambda_core::docker::DockerGatewayEndpoint="http://localhost:4566".parse()?; + /// assert_eq!(endpoint.as_str(),"http://localhost:4566/"); + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl From for GatewayEndpoint { + fn from(endpoint: DockerGatewayEndpoint) -> Self { + endpoint.0 + } +} + +/// Explicit absolute private-file path; never searched or expanded. +/// +/// Requires a controlled local filesystem and ACLs granting no untrusted +/// credential read/write or ancestor replacement; POSIX checks do not prove ACL privacy. +#[derive(Clone)] +#[non_exhaustive] +pub struct RegistryAuthFile(PathBuf); +impl fmt::Debug for RegistryAuthFile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryAuthFile()") + } +} +impl FromStr for RegistryAuthFile { + type Err = DockerError; + fn from_str(raw: &str) -> Result { + let path = Path::new(raw); + if raw.is_empty() + || raw.len() > 4096 + || raw.ends_with('/') + || raw.chars().any(char::is_control) + || !path.is_absolute() + || path + .components() + .any(|part| matches!(part, Component::ParentDir)) + || path.file_name().is_none() + { + return Err(DockerError::Configuration); + } + Ok(Self(path.into())) + } +} +impl RegistryAuthFile { + /// Read an explicitly approved private credential document under a total budget. + /// # Errors + /// Rejects expiry, unsafe files, IO/worker failure or invalid bounded YAML/policy. + /// ```no_run + /// # async fn example()->Result<(),rustack_lambda_core::docker::RegistryAuthError>{ + /// let path:rustack_lambda_core::docker::RegistryAuthFile="/private/registry.yaml".parse().map_err(|_|rustack_lambda_core::docker::RegistryAuthError::Configuration)?; + /// let credentials=path.load(vec!["registry.example".parse().map_err(|_|rustack_lambda_core::docker::RegistryAuthError::Configuration)?],tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn load( + &self, + approved: Vec, + deadline: Instant, + ) -> Result { + auth::load(self.0.clone(), approved, deadline).await + } +} + +#[derive(Clone)] +struct PullSettings { + hosts: Vec, + file: Option, +} +impl fmt::Debug for PullSettings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PullSettings()") + } +} +/// Immutable operator configuration; constructing it performs no IO or activation. +/// +/// Only the validated constructor is public: +/// ```compile_fail +/// rustack_lambda_core::docker::DockerRuntimeSettings::builder(); +/// ``` +#[derive(Clone, TypedBuilder)] +#[builder(builder_method(vis = ""))] +#[non_exhaustive] +pub struct DockerRuntimeSettings { + socket: DockerSocket, + relay: ImageReference, + zip: Option, + endpoint: Option, + sdk_credentials: Option, + policy: ContainerPolicy, + budgets: RuntimeBudgets, + log_ring_bytes: usize, + cross_architecture: bool, + pull: Option, +} +impl fmt::Debug for DockerRuntimeSettings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DockerRuntimeSettings()") + } +} +impl DockerRuntimeSettings { + pub(super) fn sdk_credentials(&self) -> Option<&LocalSdkCredentials> { + self.sdk_credentials.as_ref() + } + /// Validate explicit settings and shared budgets without reading files or Docker. + /// # Errors + /// Rejects missing required settings, malformed domain values or inconsistent budgets. + /// ``` + /// let settings=rustack_lambda_core::docker::DockerRuntimeSettings::from_lookup(|key|match key{ + /// "LAMBDA_DOCKER_RELAY_IMAGE"=>Some("relay:v1".into()),"LAMBDA_DOCKER_NETWORK"=>Some("none".into()),_=>None},&Default::default(),std::time::Duration::from_secs(600))?; + /// assert!(!settings.cross_architecture()); + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn from_lookup( + lookup: impl Fn(&str) -> Option, + budgets: &RuntimeBudgets, + idle: Duration, + ) -> Result { + budgets.validate().map_err(|_| DockerError::Configuration)?; + let socket = lookup("LAMBDA_DOCKER_SOCKET") + .unwrap_or_else(|| "unix:///var/run/docker.sock".into()) + .parse()?; + let relay = lookup("LAMBDA_DOCKER_RELAY_IMAGE") + .ok_or(DockerError::Configuration)? + .parse()?; + let network: NetworkSelection = lookup("LAMBDA_DOCKER_NETWORK") + .ok_or(DockerError::Configuration)? + .parse()?; + let io = Duration::from_secs(budgets.lambda_docker_io_seconds); + let cleanup = Duration::from_secs(budgets.lambda_docker_cleanup_seconds); + let policy = ContainerPolicy::new( + network, + number(&lookup, "UID", 1000, 1, 2_147_483_647)? + .try_into() + .map_err(|_| DockerError::Configuration)?, + number(&lookup, "GID", 1000, 1, 2_147_483_647)? + .try_into() + .map_err(|_| DockerError::Configuration)?, + cpu(&lookup("LAMBDA_DOCKER_CPU").unwrap_or_else(|| "1".into()))?, + RelayBudgets::new(io, idle, cleanup)?, + )?; + let cross_architecture = match lookup("LAMBDA_DOCKER_CROSS_ARCHITECTURE").as_deref() { + None | Some("false") => false, + Some("true") => true, + _ => return Err(DockerError::Configuration), + }; + let endpoint = lookup("LAMBDA_DOCKER_GATEWAY_ENDPOINT") + .map(|raw| raw.parse::()) + .transpose()?; + let sdk_credentials = LocalSdkCredentials::parse( + lookup("LAMBDA_DOCKER_SDK_ACCESS_KEY_ID"), + lookup("LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY"), + )?; + if sdk_credentials.is_some() && (endpoint.is_none() || policy.network.value() == "none") { + return Err(DockerError::Configuration); + } + Ok(Self::builder() + .socket(socket) + .relay(relay) + .zip( + lookup("LAMBDA_DOCKER_ZIP_IMAGE") + .map(|raw| { + let reference: ImageReference = raw.parse()?; + if reference.digest().is_none() { + return Err(DockerError::Configuration); + } + Ok(reference) + }) + .transpose()?, + ) + .endpoint(endpoint) + .sdk_credentials(sdk_credentials) + .policy(policy) + .budgets(budgets.clone()) + .log_ring_bytes( + number(&lookup, "LOG_RING_BYTES", 262_144, 4096, 1_048_576)? + .try_into() + .map_err(|_| DockerError::Configuration)?, + ) + .cross_architecture(cross_architecture) + .pull(parse_pull(&lookup)?) + .build()) + } + /// Load only explicit credentials and construct the existing pull policy. + /// # Errors + /// Returns bounded file/schema/worker/deadline or policy errors; never falls back. + /// ```no_run + /// # async fn load(settings:&rustack_lambda_core::docker::DockerRuntimeSettings)->Result<(),rustack_lambda_core::docker::RegistryAuthError>{ + /// let policy=settings.load_pull_policy(tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn load_pull_policy( + &self, + deadline: Instant, + ) -> Result { + auth::valid_deadline(deadline)?; + let Some(pull) = &self.pull else { + return Ok(ImagePullPolicy::default()); + }; + let credentials = match &pull.file { + Some(file) => file.load(pull.hosts.clone(), deadline).await?, + None => RegistryCredentials::default(), + }; + let policy = ImagePullPolicy::if_missing( + pull.hosts.clone(), + credentials, + Duration::from_secs(self.budgets.lambda_docker_pull_seconds), + ) + .map_err(|_| RegistryAuthError::Configuration)?; + auth::valid_deadline(deadline)?; + Ok(policy) + } + /// Validated local socket. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let socket=s.socket(); } + /// ``` + #[must_use] + pub fn socket(&self) -> &DockerSocket { + &self.socket + } + /// Relay source selected by the operator; not yet resolved or pinned. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let source=s.relay_image(); } + /// ``` + #[must_use] + pub fn relay_image(&self) -> &ImageReference { + &self.relay + } + /// Optional ZIP runtime source; no implicit catalog. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let source=s.zip_image(); } + /// ``` + #[must_use] + pub fn zip_image(&self) -> Option<&ImageReference> { + self.zip.as_ref() + } + /// Optional endpoint; parsing is not reachability evidence. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let endpoint=s.gateway_endpoint(); } + /// ``` + #[must_use] + pub fn gateway_endpoint(&self) -> Option<&DockerGatewayEndpoint> { + self.endpoint.as_ref() + } + /// Validated immutable container policy. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let policy=s.container_policy(); } + /// ``` + #[must_use] + pub fn container_policy(&self) -> &ContainerPolicy { + &self.policy + } + /// Validated shared runtime budgets captured for startup. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let limits=s.budgets(); } + /// ``` + #[must_use] + pub fn budgets(&self) -> &RuntimeBudgets { + &self.budgets + } + /// Per-channel bounded function log storage. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let bytes=s.log_ring_bytes(); } + /// ``` + #[must_use] + pub fn log_ring_bytes(&self) -> usize { + self.log_ring_bytes + } + /// Explicit permission to qualify an emulated architecture path. + /// ```no_run + /// # fn inspect(s:&rustack_lambda_core::docker::DockerRuntimeSettings){ let emulation=s.cross_architecture(); } + /// ``` + #[must_use] + pub fn cross_architecture(&self) -> bool { + self.cross_architecture + } +} +fn parse_pull( + lookup: &impl Fn(&str) -> Option, +) -> Result, DockerError> { + let hosts = lookup("LAMBDA_DOCKER_REGISTRIES"); + let file = lookup("LAMBDA_DOCKER_REGISTRY_AUTH_FILE"); + match lookup("LAMBDA_DOCKER_PULL_POLICY").as_deref() { + None | Some("never") => { + if hosts.is_some() || file.is_some() { + return Err(DockerError::Configuration); + } + Ok(None) + } + Some("ifMissing") => { + let raw = hosts.ok_or(DockerError::Configuration)?; + if raw.len() > 8127 || raw.split(',').count() > 32 { + return Err(DockerError::Configuration); + } + let hosts: Vec = + raw.split(',').map(str::parse).collect::>()?; + ImagePullPolicy::if_missing( + hosts.clone(), + RegistryCredentials::default(), + Duration::from_secs(300), + )?; + Ok(Some(PullSettings { + hosts, + file: file.map(|raw| raw.parse()).transpose()?, + })) + } + _ => Err(DockerError::Configuration), + } +} +fn number( + lookup: &impl Fn(&str) -> Option, + suffix: &str, + default: u64, + min: u64, + max: u64, +) -> Result { + let value = match lookup(&format!("LAMBDA_DOCKER_{suffix}")) { + None => default, + Some(raw) => { + if raw.len() > 20 || !raw.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(DockerError::Configuration); + } + raw.parse().map_err(|_| DockerError::Configuration)? + } + }; + if !(min..=max).contains(&value) { + return Err(DockerError::Configuration); + } + Ok(value) +} +fn cpu(raw: &str) -> Result { + if raw.is_empty() || raw.len() > 12 { + return Err(DockerError::Configuration); + } + let (whole, fraction) = raw + .split_once('.') + .map_or((raw, ""), |(whole, fraction)| (whole, fraction)); + if whole.is_empty() + || !whole.bytes().all(|byte| byte.is_ascii_digit()) + || fraction.len() > 9 + || !fraction.bytes().all(|byte| byte.is_ascii_digit()) + || (raw.contains('.') && fraction.is_empty()) + { + return Err(DockerError::Configuration); + } + let whole: u64 = whole.parse().map_err(|_| DockerError::Configuration)?; + let fraction_value = if fraction.is_empty() { + 0 + } else { + fraction + .parse::() + .map_err(|_| DockerError::Configuration)? + }; + let scale = 10_u64 + .checked_pow( + u32::try_from(9_usize.saturating_sub(fraction.len())) + .map_err(|_| DockerError::Configuration)?, + ) + .ok_or(DockerError::Configuration)?; + let nano = whole + .checked_mul(1_000_000_000) + .and_then(|value| { + fraction_value + .checked_mul(scale) + .and_then(|fraction| value.checked_add(fraction)) + }) + .ok_or(DockerError::Configuration)?; + if !(100_000_000..=16_000_000_000).contains(&nano) { + return Err(DockerError::Configuration); + } + Ok(nano) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/config/auth.rs b/crates/rustack-lambda-core/src/docker/config/auth.rs new file mode 100644 index 00000000..84d2340a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/config/auth.rs @@ -0,0 +1,441 @@ +//! Bounded retained read-only workers; no ambient credentials or parser trees. +//! +//! A timed-out/dropped observer cannot stop a local filesystem syscall. Its +//! read-only blocking job retains the permit, descriptor and zeroizing buffer +//! until it exits. No late result can publish runtime readiness. POSIX checks +//! require operator-controlled ACLs; they do not prove extended-ACL privacy. + +use std::{ + collections::BTreeSet, + fmt, io, + os::fd::OwnedFd, + path::{Component, Path, PathBuf}, + str::{Chars, from_utf8}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use nix::{ + errno::Errno, + fcntl::{OFlag, open, openat}, + sys::stat::{FileStat, Mode, SFlag, fstat}, + unistd::{geteuid, read}, +}; +use secrecy::{ + ExposeSecret, SecretString, + zeroize::{Zeroize, Zeroizing}, +}; +use tokio::{ + runtime::Handle, + sync::{Semaphore, SemaphorePermit}, + task::{JoinHandle, spawn_blocking}, + time::{Instant, timeout_at}, +}; +use yaml_rust2::{ + parser::{Event, Parser}, + scanner::TScalarStyle, +}; + +use crate::docker::{RegistryCredential, RegistryCredentials, RegistryHost}; + +static WORKERS: Semaphore = Semaphore::const_new(4); +const MAX_BYTES: usize = 65_536; + +/// Fixed credential-source failure classes; paths, YAML and secrets are redacted. +#[derive(thiserror::Error)] +#[non_exhaustive] +pub enum RegistryAuthError { + /// Invalid path, approved-host set or budget. + #[error("invalid registry credential configuration")] + Configuration, + /// Unsafe file kind, owner, links, modes or changed observed metadata. + #[error("unsafe registry credential file")] + UnsafeFile, + /// Malformed or unsupported bounded credential YAML/schema. + #[error("invalid registry credential document")] + Document, + /// Source or parsed collection exceeds its hard ceiling. + #[error("registry credential byte budget exceeded")] + TooLarge, + /// Absolute queue/read/parse observation deadline expired. + #[error("registry credential deadline exceeded")] + Deadline, + /// The requesting observer was abandoned. + #[error("registry credential read cancelled")] + Cancelled, + /// No runtime or retained blocking worker failed. + #[error("registry credential worker failed")] + Worker, + /// Local descriptor IO failed without displaying source/path text. + #[error("registry credential IO failed")] + Io(#[source] io::Error), +} +impl fmt::Debug for RegistryAuthError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl From for RegistryAuthError { + fn from(error: Errno) -> Self { + match error { + Errno::ELOOP | Errno::ENOTDIR => Self::UnsafeFile, + _ => Self::Io(error.into()), + } + } +} +struct Control { + deadline: Instant, + cancelled: Arc, +} +impl fmt::Debug for Control { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryReadControl") + } +} +impl Control { + fn check(&self) -> Result<(), RegistryAuthError> { + if Instant::now() >= self.deadline { + return Err(RegistryAuthError::Deadline); + } + if self.cancelled.load(Ordering::Acquire) { + return Err(RegistryAuthError::Cancelled); + } + Ok(()) + } +} +struct Cancel(Arc); +impl Drop for Cancel { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} +pub(super) fn valid_deadline(deadline: Instant) -> Result<(), RegistryAuthError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(RegistryAuthError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(RegistryAuthError::Configuration); + } + Ok(()) +} +pub(super) async fn load( + path: PathBuf, + approved: Vec, + deadline: Instant, +) -> Result { + read_owned(path, approved, deadline, &WORKERS, |_| Ok(())).await +} +async fn read_owned( + path: PathBuf, + approved: Vec, + deadline: Instant, + pool: &'static Semaphore, + before: impl FnOnce(&Control) -> Result<(), RegistryAuthError> + Send + 'static, +) -> Result { + valid_deadline(deadline)?; + Handle::try_current().map_err(|_| RegistryAuthError::Worker)?; + let count = approved.len(); + if count == 0 || count > 32 { + return Err(RegistryAuthError::Configuration); + } + let approved: BTreeSet<_> = approved.into_iter().collect(); + if approved.len() != count { + return Err(RegistryAuthError::Configuration); + } + let permit = timeout_at(deadline, pool.acquire()) + .await + .map_err(|_| RegistryAuthError::Deadline)? + .map_err(|_| RegistryAuthError::Worker)?; + let cancelled = Arc::new(AtomicBool::new(false)); + let _cancel = Cancel(cancelled.clone()); + let task = spawn_read( + path, + approved, + Control { + deadline, + cancelled, + }, + permit, + before, + ); + let (credentials, _permit) = timeout_at(deadline, task) + .await + .map_err(|_| RegistryAuthError::Deadline)? + .map_err(|_| RegistryAuthError::Worker)??; + if Instant::now() >= deadline { + return Err(RegistryAuthError::Deadline); + } + Ok(credentials) +} +fn spawn_read( + path: PathBuf, + approved: BTreeSet, + control: Control, + permit: SemaphorePermit<'static>, + before: impl FnOnce(&Control) -> Result<(), RegistryAuthError> + Send + 'static, +) -> JoinHandle), RegistryAuthError>> { + spawn_blocking(move || { + control.check()?; + before(&control)?; + control.check()?; + let value = read_document(&path, &approved, &control)?; + control.check()?; + // Ready results remain charged until observed or discarded. + Ok::<(RegistryCredentials, SemaphorePermit<'static>), RegistryAuthError>((value, permit)) + }) +} + +fn flags() -> OFlag { + OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NONBLOCK | OFlag::O_NOCTTY +} +fn trusted_directory(file: &OwnedFd) -> Result<(), RegistryAuthError> { + let stat = fstat(file)?; + let owner = stat.st_uid; + let mode = stat.st_mode; + if SFlag::from_bits_truncate(mode) != SFlag::S_IFDIR + || (owner != 0 && owner != geteuid().as_raw()) + || (mode & 0o022 != 0 && !(owner == 0 && mode & 0o1000 != 0)) + { + return Err(RegistryAuthError::UnsafeFile); + } + Ok(()) +} +fn private_file(file: &OwnedFd) -> Result { + let stat = fstat(file)?; + if SFlag::from_bits_truncate(stat.st_mode) != SFlag::S_IFREG + || stat.st_uid != geteuid().as_raw() + || stat.st_nlink != 1 + || stat.st_mode & 0o7177 != 0 + { + return Err(RegistryAuthError::UnsafeFile); + } + if stat.st_size <= 0 + || stat.st_size > i64::try_from(MAX_BYTES).map_err(|_| RegistryAuthError::Configuration)? + { + return Err(RegistryAuthError::TooLarge); + } + Ok(stat) +} +fn open_private(path: &Path, control: &Control) -> Result { + control.check()?; + let mut directory = open(Path::new("/"), flags() | OFlag::O_DIRECTORY, Mode::empty())?; + trusted_directory(&directory)?; + let mut parts = path.components().peekable(); + while let Some(part) = parts.next() { + control.check()?; + match part { + Component::RootDir | Component::CurDir => {} + Component::Normal(name) => { + if parts.peek().is_none() { + return Ok(openat(&directory, name, flags(), Mode::empty())?); + } + let child = openat( + &directory, + name, + flags() | OFlag::O_DIRECTORY, + Mode::empty(), + )?; + trusted_directory(&child)?; + directory = child; + } + _ => return Err(RegistryAuthError::Configuration), + } + } + Err(RegistryAuthError::Configuration) +} +fn same(before: &FileStat, after: &FileStat) -> bool { + before.st_dev == after.st_dev + && before.st_ino == after.st_ino + && before.st_uid == after.st_uid + && before.st_mode == after.st_mode + && before.st_nlink == after.st_nlink + && before.st_size == after.st_size + && before.st_mtime == after.st_mtime + && before.st_mtime_nsec == after.st_mtime_nsec + && before.st_ctime == after.st_ctime + && before.st_ctime_nsec == after.st_ctime_nsec +} +fn read_document( + path: &Path, + approved: &BTreeSet, + control: &Control, +) -> Result { + let file = open_private(path, control)?; + let initial = private_file(&file)?; + let mut bytes = Zeroizing::new(Vec::new()); + bytes + .try_reserve_exact(MAX_BYTES + 1) + .map_err(|_| RegistryAuthError::TooLarge)?; + bytes.resize(MAX_BYTES + 1, 0); + let mut used = 0_usize; + loop { + control.check()?; + let end = used.saturating_add(1024).min(MAX_BYTES + 1); + let buffer = bytes + .get_mut(used..end) + .ok_or(RegistryAuthError::TooLarge)?; + if buffer.is_empty() { + return Err(RegistryAuthError::TooLarge); + } + let count = read(&file, buffer)?; + if count == 0 { + break; + } + used = used.checked_add(count).ok_or(RegistryAuthError::TooLarge)?; + if used > MAX_BYTES { + return Err(RegistryAuthError::TooLarge); + } + } + control.check()?; + validate_observation(&file, &initial, used)?; + let text = from_utf8(bytes.get(..used).ok_or(RegistryAuthError::TooLarge)?) + .map_err(|_| RegistryAuthError::Document)?; + parse(text, approved, control) +} + +fn validate_observation( + file: &OwnedFd, + initial: &FileStat, + used: usize, +) -> Result<(), RegistryAuthError> { + let after = private_file(file)?; + if !same(initial, &after) || usize::try_from(initial.st_size).ok() != Some(used) { + return Err(RegistryAuthError::UnsafeFile); + } + Ok(()) +} + +struct Events<'a> { + parser: Parser>, + count: usize, + control: &'a Control, +} +impl fmt::Debug for Events<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryYamlEvents()") + } +} +impl Events<'_> { + fn next(&mut self) -> Result { + self.control.check()?; + self.count = self + .count + .checked_add(1) + .ok_or(RegistryAuthError::TooLarge)?; + if self.count > 512 { + return Err(RegistryAuthError::TooLarge); + } + self.parser + .next_token() + .map(|(event, _)| event) + .map_err(|_| RegistryAuthError::Document) + } + fn expect(&mut self, event: &Event) -> Result<(), RegistryAuthError> { + let actual = self.next()?; + if &actual != event { + if let Event::Scalar(mut value, _, _, _) = actual { + value.zeroize(); + } + return Err(RegistryAuthError::Document); + } + Ok(()) + } +} +fn scalar(event: Event, quoted: bool, max: usize) -> Result { + match event { + Event::Scalar(value, style, anchor, tag) => { + let value: SecretString = value.into(); + if anchor != 0 + || tag.is_some() + || (quoted + && !matches!( + style, + TScalarStyle::SingleQuoted | TScalarStyle::DoubleQuoted + )) + || value.expose_secret().is_empty() + || value.expose_secret().len() > max + { + return Err(RegistryAuthError::Document); + } + Ok(value) + } + _ => Err(RegistryAuthError::Document), + } +} +fn credential(events: &mut Events<'_>) -> Result { + events.expect(&Event::MappingStart(0, None))?; + let mut user = None; + let mut password = None; + let mut token = None; + loop { + let event = events.next()?; + if event == Event::MappingEnd { + break; + } + let key = scalar(event, false, 32)?; + let target = match key.expose_secret() { + "username" => (&mut user, 256), + "password" => (&mut password, 4096), + "identityToken" => (&mut token, 8192), + _ => return Err(RegistryAuthError::Document), + }; + if target.0.is_some() { + return Err(RegistryAuthError::Document); + } + *target.0 = Some(scalar(events.next()?, true, target.1)?); + } + match (user, password, token) { + (Some(user), Some(password), None) => RegistryCredential::basic(user, password), + (None, None, Some(token)) => RegistryCredential::identity_token(token), + _ => return Err(RegistryAuthError::Document), + } + .map_err(|_| RegistryAuthError::Document) +} +fn parse( + text: &str, + approved: &BTreeSet, + control: &Control, +) -> Result { + if text.len() > MAX_BYTES { + return Err(RegistryAuthError::TooLarge); + } + let mut events = Events { + parser: Parser::new_from_str(text), + count: 0, + control, + }; + events.expect(&Event::StreamStart)?; + events.expect(&Event::DocumentStart)?; + events.expect(&Event::MappingStart(0, None))?; + let mut records = Vec::new(); + let mut seen = BTreeSet::new(); + loop { + let event = events.next()?; + if event == Event::MappingEnd { + break; + } + if records.len() >= 32 { + return Err(RegistryAuthError::TooLarge); + } + let raw = scalar(event, false, 253)?; + let host: RegistryHost = raw + .expose_secret() + .parse() + .map_err(|_| RegistryAuthError::Document)?; + if !approved.contains(&host) || !seen.insert(host.clone()) { + return Err(RegistryAuthError::Document); + } + records.push((host, credential(&mut events)?)); + } + events.expect(&Event::DocumentEnd)?; + events.expect(&Event::StreamEnd)?; + control.check()?; + RegistryCredentials::try_from(records).map_err(|_| RegistryAuthError::Document) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/config/auth/tests.rs b/crates/rustack-lambda-core/src/docker/config/auth/tests.rs new file mode 100644 index 00000000..009ed6ab --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/config/auth/tests.rs @@ -0,0 +1,560 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Write, + os::unix::fs::PermissionsExt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc, + }, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use nix::{sys::stat::Mode, unistd::mkfifo}; +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::settings::RuntimeBudgets; +use serde_json::{Value, from_slice}; +use tempfile::{TempDir, tempdir}; +use tokio::{ + fs, spawn, + sync::{Semaphore, oneshot}, + task::yield_now, + time::{Instant, sleep, timeout}, +}; + +use super::{ + Control, RegistryAuthError, open_private, parse, private_file, read_owned, spawn_read, + validate_observation, +}; +use crate::{ + artifact::Architecture, + docker::{ + DirectoryLease, DockerRuntimeSettings, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, + RegistryAuthFile, RegistryHost, tests::Daemon, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn approved() -> BTreeSet { + BTreeSet::from(["registry.example".parse().unwrap()]) +} +fn control() -> Control { + Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + } +} +async fn fixture(text: &str) -> (TempDir, RegistryAuthFile) { + let root = tempdir().unwrap(); + let canonical = fs::canonicalize(root.path()).await.unwrap(); + let path = canonical.join("credentials.yaml"); + fs::write(&path, text).await.unwrap(); + fs::set_permissions(&path, PermissionsExt::from_mode(0o600)) + .await + .unwrap(); + (root, path.to_str().unwrap().parse().unwrap()) +} +#[rstest] +#[case("{}")] +#[case("registry.example: {username: 'user', password: \"password\"}")] +#[case("---\nregistry.example:\n identityToken: \"token\\u2603\"\n...")] +#[case("'registry.example':\n username: 'user'\n password: 'pas''sword'")] +fn test_should_parse_exact_shallow_quoted_credential_schema(#[case] text: &str) { + assert!(parse(text, &approved(), &control()).is_ok()); +} +#[rstest] +#[case("")] +#[case("[]")] +#[case("secret")] +#[case("registry.example: {}")] +#[case("registry.example: {username: 'user'}")] +#[case("registry.example: {password: 'p'}")] +#[case("registry.example: {identityToken: ''}")] +#[case("registry.example: {identityToken: token}")] +#[case("registry.example: {identityToken: true}")] +#[case("registry.example: {identityToken: 123}")] +#[case("registry.example: {identityToken: null}")] +#[case("registry.example: {identityToken: 'a', identityToken: 'b'}")] +#[case("registry.example: {username: 'u', password: 'p', identityToken: 't'}")] +#[case("registry.example: {username: 'u', password: 'p', extra: 'x'}")] +#[case("registry.example: {identityToken: \"a\\nb\"}")] +#[case("registry.example: {identityToken: !secret 'token'}")] +#[case("registry.example: &secret {identityToken: 'token'}")] +#[case("registry.example: {identityToken: &secret 'token'}")] +#[case("registry.example: {identityToken: *secret}")] +#[case("registry.example: {identityToken: {nested: 'token'}}")] +#[case("registry.example: {identityToken: ['token']}")] +#[case("registry.example: {<<: {identityToken: 'token'}}")] +#[case("registry.example: {identityToken: 'a'}\nregistry.example: {identityToken: 'b'}")] +#[case("REGISTRY.EXAMPLE: {identityToken: 'a'}\nregistry.example: {identityToken: 'b'}")] +#[case("other.example: {identityToken: 'token'}")] +#[case("{}\n---\n{}")] +#[case("{}\ntrailing: [")] +fn test_should_reject_unsafe_ambiguous_or_typed_yaml(#[case] text: &str) { + let error = parse(text, &approved(), &control()).unwrap_err(); + assert!(!format!("{error:?}: {error}").contains("token")); +} +#[test] +fn test_should_bound_decoded_secrets_collections_and_input() { + for (field, max) in [ + ("username", 256), + ("password", 4096), + ("identityToken", 8192), + ] { + let text = if field == "identityToken" { + format!( + "registry.example: {{identityToken: '{}' }}", + "t".repeat(max) + ) + } else { + format!( + "registry.example: {{username: '{}', password: '{}'}}", + if field == "username" { + "u".repeat(max) + } else { + "u".into() + }, + if field == "password" { + "p".repeat(max) + } else { + "p".into() + } + ) + }; + assert!(parse(&text, &approved(), &control()).is_ok()); + let too_big = text.replacen(&"t".repeat(max), &"t".repeat(max + 1), 1); + if field == "identityToken" { + assert!(parse(&too_big, &approved(), &control()).is_err()); + } else { + let big = format!( + "registry.example: {{username: '{}', password: '{}'}}", + if field == "username" { + "u".repeat(max + 1) + } else { + "u".into() + }, + if field == "password" { + "p".repeat(max + 1) + } else { + "p".into() + } + ); + assert!(parse(&big, &approved(), &control()).is_err()); + } + } + let text = format!("{{}}\n#{}", "x".repeat(65_532)); + assert_eq!(text.len(), 65_536); + assert!(parse(&text, &approved(), &control()).is_ok()); + assert!(parse(&(text + "x"), &approved(), &control()).is_err()); + let hosts: BTreeSet<_> = (0..33) + .map(|i| format!("r{i}.example").parse().unwrap()) + .collect(); + let mut raw = String::new(); + for i in 0..33 { + writeln!(&mut raw, "r{i}.example: {{identityToken: 't'}}").unwrap(); + } + assert!(parse(&raw, &hosts, &control()).is_err()); + let hosts: BTreeSet<_> = (0..8) + .map(|i| format!("r{i}.example").parse().unwrap()) + .collect(); + let mut raw = String::new(); + for i in 0..8 { + writeln!( + &mut raw, + "r{i}.example: {{identityToken: \"{}xx\"}}", + "\\L".repeat(2730) + ) + .unwrap(); + } + assert!(raw.len() < 65_536); + assert!(parse(&raw, &hosts, &control()).is_err()); +} +proptest! { + #![proptest_config(ProptestConfig::with_cases(96))] + #[test] + fn test_should_never_panic_or_expose_arbitrary_yaml(text in ".{0,2048}"){ + if let Err(error)=parse(&text,&approved(),&control()){let shown=format!("{error}");prop_assert!(shown.len()<128);} + } +} +#[tokio::test] +async fn test_should_load_real_private_file_and_reject_modes_links_and_replacements() { + let (root, file) = fixture("registry.example: {identityToken: 'PRIVATE-TOKEN'}").await; + let credentials = file + .load(approved().into_iter().collect(), deadline()) + .await + .unwrap(); + assert!(!format!("{credentials:?}").contains("PRIVATE")); + for mode in [0o644, 0o640, 0o660, 0o700, 0o1600, 0o4600] { + fs::set_permissions(&file.0, PermissionsExt::from_mode(mode)) + .await + .unwrap(); + assert!(matches!( + file.load(approved().into_iter().collect(), deadline()) + .await, + Err(RegistryAuthError::UnsafeFile) + )); + } + fs::set_permissions(&file.0, PermissionsExt::from_mode(0o600)) + .await + .unwrap(); + let link = root.path().join("hardlink"); + fs::hard_link(&file.0, &link).await.unwrap(); + assert!(matches!( + file.load(approved().into_iter().collect(), deadline()) + .await, + Err(RegistryAuthError::UnsafeFile) + )); + fs::remove_file(link).await.unwrap(); + let descriptor = open_private(&file.0, &control()).unwrap(); + let original = private_file(&descriptor).unwrap(); + fs::write(&file.0, "{}\n#changed").await.unwrap(); + assert!(matches!( + validate_observation( + &descriptor, + &original, + usize::try_from(original.st_size).unwrap() + ), + Err(RegistryAuthError::UnsafeFile) + )); + fs::remove_file(&file.0).await.unwrap(); + fs::write(&file.0, "{}").await.unwrap(); + fs::set_permissions(&file.0, PermissionsExt::from_mode(0o600)) + .await + .unwrap(); + assert!(matches!( + private_file(&descriptor), + Err(RegistryAuthError::UnsafeFile) + )); + file.load(approved().into_iter().collect(), deadline()) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_reject_leaf_ancestor_symlinks_and_nonregular_files_without_hanging() { + let (root, file) = fixture("{}").await; + let canonical = fs::canonicalize(root.path()).await.unwrap(); + let symlink = canonical.join("link"); + fs::symlink(&file.0, &symlink).await.unwrap(); + let source: RegistryAuthFile = symlink.to_str().unwrap().parse().unwrap(); + assert!( + source + .load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + let dir = canonical.join("dir"); + fs::create_dir(&dir).await.unwrap(); + let source: RegistryAuthFile = dir.to_str().unwrap().parse().unwrap(); + assert!( + source + .load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + let ancestor = canonical.join("ancestor"); + fs::symlink(&canonical, &ancestor).await.unwrap(); + let source: RegistryAuthFile = ancestor + .join("credentials.yaml") + .to_str() + .unwrap() + .parse() + .unwrap(); + assert!( + source + .load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + let fifo = canonical.join("fifo"); + mkfifo(&fifo, Mode::from_bits_truncate(0o600)).unwrap(); + let source: RegistryAuthFile = fifo.to_str().unwrap().parse().unwrap(); + assert!( + timeout( + Duration::from_secs(1), + source.load(approved().into_iter().collect(), deadline()) + ) + .await + .unwrap() + .is_err() + ); + fs::set_permissions(&canonical, PermissionsExt::from_mode(0o777)) + .await + .unwrap(); + assert!( + file.load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + fs::set_permissions(&canonical, PermissionsExt::from_mode(0o700)) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_enforce_real_file_size_utf8_and_approved_set_without_fallback() { + for text in [ + String::new(), + "x".repeat(65_537), + "registry.example: {identityToken: token}".into(), + ] { + let (_root, file) = fixture(&text).await; + assert!( + file.load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + } + let (_root, file) = fixture(&format!("{{}}\n#{}", "x".repeat(65_532))).await; + file.load(approved().into_iter().collect(), deadline()) + .await + .unwrap(); + fs::write(&file.0, [0xff]).await.unwrap(); + assert!( + file.load(approved().into_iter().collect(), deadline()) + .await + .is_err() + ); + assert!(file.load(Vec::new(), deadline()).await.is_err()); + assert!( + file.load(vec!["registry.example".parse().unwrap(); 33], deadline()) + .await + .is_err() + ); + assert!( + file.load(approved().into_iter().collect(), Instant::now()) + .await + .is_err() + ); + assert!( + file.load( + approved().into_iter().collect(), + Instant::now() + Duration::from_secs(61) + ) + .await + .is_err() + ); + fs::remove_file(&file.0).await.unwrap(); + let error = file + .load(approved().into_iter().collect(), deadline()) + .await + .unwrap_err(); + assert!(!format!("{error:?}:{error}").contains("credentials.yaml")); +} +#[tokio::test] +async fn test_should_retain_all_four_worker_permits_after_observers_are_abandoned() { + static POOL: Semaphore = Semaphore::const_new(4); + let (_root, file) = fixture("{}").await; + let mut releases = Vec::new(); + for _ in 0..4 { + let (release, held) = mpsc::sync_channel(1); + let (entered, saw) = oneshot::channel(); + let path = file.0.clone(); + let task = spawn(read_owned( + path, + approved().into_iter().collect(), + deadline(), + &POOL, + move |_| { + let _ = entered.send(()); + held.recv().map_err(|_| RegistryAuthError::Worker) + }, + )); + saw.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + releases.push(release); + } + assert_eq!(POOL.available_permits(), 0); + let started = Arc::new(AtomicBool::new(false)); + let observed = started.clone(); + assert!(matches!( + read_owned( + file.0.clone(), + approved().into_iter().collect(), + Instant::now() + Duration::from_millis(25), + &POOL, + move |_| { + started.store(true, Ordering::Release); + Ok(()) + } + ) + .await, + Err(RegistryAuthError::Deadline) + )); + assert!(!observed.load(Ordering::Acquire)); + assert_eq!(POOL.available_permits(), 0); + for release in releases { + release.send(()).unwrap(); + } + timeout(Duration::from_secs(2), async { + while POOL.available_permits() != 4 { + yield_now().await; + } + }) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_charge_ready_results_and_release_failed_workers() { + static POOL: Semaphore = Semaphore::const_new(1); + let (_root, file) = fixture("{}").await; + let task = spawn_read( + file.0.clone(), + approved(), + control(), + POOL.acquire().await.unwrap(), + |_| Ok(()), + ); + timeout(Duration::from_secs(2), async { + while !task.is_finished() { + yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(POOL.available_permits(), 0); + let (_credentials, permit) = task.await.unwrap().unwrap(); + assert_eq!(POOL.available_permits(), 0); + drop(permit); + assert_eq!(POOL.available_permits(), 1); + assert!(matches!( + read_owned( + file.0.clone(), + approved().into_iter().collect(), + deadline(), + &POOL, + |_| panic!("controlled credential worker failure") + ) + .await, + Err(RegistryAuthError::Worker) + )); + assert_eq!(POOL.available_permits(), 1); +} +#[tokio::test] +async fn test_should_keep_timed_out_local_read_charged_until_worker_observes_cancellation() { + static POOL: Semaphore = Semaphore::const_new(1); + let (_root, file) = fixture("{}").await; + let (release, held) = mpsc::sync_channel(1); + let (entered, saw) = oneshot::channel(); + let path = file.0.clone(); + let task = spawn(read_owned( + path, + approved().into_iter().collect(), + Instant::now() + Duration::from_secs(1), + &POOL, + move |_| { + let _ = entered.send(()); + held.recv().map_err(|_| RegistryAuthError::Worker) + }, + )); + saw.await.unwrap(); + let result = task.await.unwrap(); + assert!(matches!(result, Err(RegistryAuthError::Deadline))); + assert_eq!(POOL.available_permits(), 0); + release.send(()).unwrap(); + timeout(Duration::from_secs(2), async { + while POOL.available_permits() != 1 { + sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_consume_file_loaded_policy_through_bound_image_preparation() { + let (root, file) = + fixture("registry.example: {username: 'PRIVATE-USER', password: 'PRIVATE-PASSWORD'}").await; + let values = BTreeMap::from([ + ("LAMBDA_DOCKER_RELAY_IMAGE", "relay:v1".to_owned()), + ("LAMBDA_DOCKER_NETWORK", "none".into()), + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing".into()), + ("LAMBDA_DOCKER_REGISTRIES", "registry.example".into()), + ( + "LAMBDA_DOCKER_REGISTRY_AUTH_FILE", + file.0.to_str().unwrap().to_owned(), + ), + ]); + let settings = DockerRuntimeSettings::from_lookup( + |key| values.get(key).cloned(), + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap(); + let policy = settings.load_pull_policy(deadline()).await.unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut images = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy, + ) + .unwrap(); + let port = images.port(); + let reference = "registry.example/fn:v1".parse().unwrap(); + let (result, ()) = tokio::join!( + port.prepare(&reference, Architecture::Arm64, None, deadline()), + async { + daemon.next().await.reply(404, b"{}").await; + let request = daemon.next().await; + assert!(request.head.starts_with("POST /v1.53/images/create?")); + let header = request + .head + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("X-Registry-Auth")) + .map(|(_, value)| value.trim()) + }) + .unwrap(); + let auth: Value = from_slice(&URL_SAFE.decode(header).unwrap()).unwrap(); + assert_eq!(auth["username"], "PRIVATE-USER"); + assert_eq!(auth["password"], "PRIVATE-PASSWORD"); + assert_eq!(auth["serveraddress"], "registry.example"); + request.reply(200, b"{\"status\":\"done\"}\n").await; + daemon + .next() + .await + .reply(200, b"{\"Id\":\"invalid\"}") + .await; + } + ); + assert!(result.is_err()); + images.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_oversized_approved_hosts_before_worker_admission_or_io() { + static POOL: Semaphore = Semaphore::const_new(0); + let approved = (0..33) + .map(|i| format!("r{i}.example").parse().unwrap()) + .collect(); + let result = timeout( + Duration::from_millis(250), + read_owned( + "/missing/unreadable.yaml".into(), + approved, + deadline(), + &POOL, + |_| panic!("must reject before worker"), + ), + ) + .await + .unwrap(); + assert!(matches!(result, Err(RegistryAuthError::Configuration))); +} diff --git a/crates/rustack-lambda-core/src/docker/config/tests.rs b/crates/rustack-lambda-core/src/docker/config/tests.rs new file mode 100644 index 00000000..1d235515 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/config/tests.rs @@ -0,0 +1,285 @@ +use std::{collections::BTreeMap, time::Duration}; + +use rstest::rstest; +use rustack_core::settings::{RuntimeBudgets, ValidatedSettings}; + +use super::{DockerGatewayEndpoint, DockerRuntimeSettings, RegistryAuthFile, cpu}; + +fn settings(extra: &[(&str, &str)]) -> Result { + let mut values = BTreeMap::from([ + ("LAMBDA_DOCKER_RELAY_IMAGE", "private-relay:v1"), + ("LAMBDA_DOCKER_NETWORK", "none"), + ]); + values.extend(extra.iter().copied()); + DockerRuntimeSettings::from_lookup( + |key| values.get(key).map(|value| (*value).into()), + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) +} +#[rstest] +#[case("0.1", 100_000_000)] +#[case("1", 1_000_000_000)] +#[case("1.234567891", 1_234_567_891)] +#[case("16.000000000", 16_000_000_000)] +fn test_should_convert_cpu_decimal_without_rounding(#[case] raw: &str, #[case] expected: u64) { + assert_eq!(cpu(raw).unwrap(), expected); +} +#[rstest] +#[case("")] +#[case("0")] +#[case("0.099999999")] +#[case("16.000000001")] +#[case("1.1234567891")] +#[case("1e0")] +#[case("NaN")] +#[case("+1")] +#[case("-1")] +#[case("1.")] +#[case(".5")] +#[case(" 1")] +#[case("1..1")] +#[case("999999999999")] +fn test_should_reject_invalid_or_out_of_range_cpu(#[case] raw: &str) { + assert!(cpu(raw).is_err()); +} +#[rstest] +#[case("http://localhost:4566")] +#[case("https://gateway.example")] +#[case("http://[::1]:4566/")] +fn test_should_validate_operator_endpoint_without_network_io(#[case] raw: &str) { + let endpoint: DockerGatewayEndpoint = raw.parse().unwrap(); + assert!(!format!("{endpoint:?}").contains("localhost")); +} +#[rstest] +#[case("http://user:password@localhost:4566")] +#[case("http://@localhost:4566")] +#[case("http://:password@localhost:4566")] +#[case("http://localhost:0")] +#[case("http://localhost/path")] +#[case("http://localhost/?")] +#[case("http://localhost/#")] +#[case("ftp://localhost/")] +#[case("http:localhost")] +#[case("http://localhost\\")] +#[case("http://localhost\n")] +#[case(" http://localhost")] +#[case("http://localhost ")] +#[case("http://localhost/a/..")] +#[case("http://localhost/%2e")] +fn test_should_reject_ambiguous_userinfo_or_nonroot_endpoints(#[case] raw: &str) { + assert!(raw.parse::().is_err()); +} +#[test] +fn test_should_bound_auth_paths_and_endpoint_bytes() { + for raw in [ + "relative.yaml", + "~/auth.yaml", + "/private/../auth.yaml", + "/", + "/private/auth.yaml/", + "/private/secret\0.yaml", + ] { + assert!(raw.parse::().is_err()); + } + assert!( + format!("/{}", "a".repeat(4096)) + .parse::() + .is_err() + ); + assert!( + format!("http://{}.example/", "a".repeat(2048)) + .parse::() + .is_err() + ); + let path: RegistryAuthFile = "/private/auth.yaml".parse().unwrap(); + assert!(!format!("{path:?}").contains("auth.yaml")); +} +#[test] +fn test_should_require_explicit_runtime_sources_and_validate_every_operator_limit() { + assert!( + DockerRuntimeSettings::from_lookup( + |_| None, + &RuntimeBudgets::default(), + Duration::from_secs(600) + ) + .is_err() + ); + for (key, value) in [ + ("LAMBDA_DOCKER_SOCKET", "tcp://localhost:2375"), + ("LAMBDA_DOCKER_RELAY_IMAGE", "bad image"), + ("LAMBDA_DOCKER_NETWORK", "host"), + ("LAMBDA_DOCKER_NETWORK", "bridge"), + ("LAMBDA_DOCKER_UID", "0"), + ("LAMBDA_DOCKER_GID", "2147483648"), + ("LAMBDA_DOCKER_UID", "+1"), + ("LAMBDA_DOCKER_CPU", "17"), + ("LAMBDA_DOCKER_LOG_RING_BYTES", "4095"), + ("LAMBDA_DOCKER_LOG_RING_BYTES", "1048577"), + ("LAMBDA_DOCKER_CROSS_ARCHITECTURE", "yes"), + ("LAMBDA_DOCKER_ZIP_IMAGE", "bad image"), + ("LAMBDA_DOCKER_GATEWAY_ENDPOINT", "http://@localhost"), + ] { + assert!(settings(&[(key, value)]).is_err(), "{key}"); + } +} +#[test] +fn test_should_capture_settings_from_existing_shared_yaml_and_redact_runtime_values() { + let shared = ValidatedSettings::parse( + "environment:\n LAMBDA_DOCKER_RELAY_IMAGE: private-relay:v1\n LAMBDA_DOCKER_NETWORK: \ + none\n LAMBDA_DOCKER_CPU: '0.125'\n LAMBDA_DOCKER_CROSS_ARCHITECTURE: yes\nbudgets:\n \ + lambdaDockerPullSeconds: 600", + &BTreeMap::new(), + ) + .unwrap(); + let runtime = DockerRuntimeSettings::from_lookup( + |key| shared.get(key).map(str::to_owned), + shared.budgets(), + Duration::from_secs(600), + ) + .unwrap(); + assert!(runtime.cross_architecture()); + assert_eq!(runtime.budgets().lambda_docker_pull_seconds, 600); + assert_eq!(runtime.log_ring_bytes(), 262_144); + assert_eq!(runtime.relay_image().as_str(), "private-relay:v1"); + assert!(runtime.zip_image().is_none()); + assert!(runtime.gateway_endpoint().is_none()); + assert!(!format!("{runtime:?}").contains("private-relay")); +} +#[test] +fn test_should_require_exact_if_missing_allowlist_without_ignored_settings() { + for values in [ + vec![("LAMBDA_DOCKER_REGISTRIES", "registry.example")], + vec![("LAMBDA_DOCKER_REGISTRY_AUTH_FILE", "/private/auth.yaml")], + vec![("LAMBDA_DOCKER_PULL_POLICY", "always")], + vec![("LAMBDA_DOCKER_PULL_POLICY", "ifMissing")], + vec![ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ("LAMBDA_DOCKER_REGISTRIES", ""), + ], + vec![ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ( + "LAMBDA_DOCKER_REGISTRIES", + "registry.example,registry.example", + ), + ], + vec![ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ( + "LAMBDA_DOCKER_REGISTRIES", + "registry.example, other.example", + ), + ], + ] { + assert!(settings(&values).is_err()); + } + assert!( + settings(&[ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ( + "LAMBDA_DOCKER_REGISTRIES", + "registry.example,public.ecr.aws" + ) + ]) + .is_ok() + ); +} + +#[test] +fn test_should_bound_registry_csv_before_collecting_and_accept_full_documented_capacity() { + let hosts = (0..32) + .map(|i| { + format!( + "{i:02}{}.{}.{}.{}", + "a".repeat(61), + "b".repeat(63), + "c".repeat(63), + "d".repeat(61) + ) + }) + .collect::>(); + assert!(hosts.iter().all(|host| host.len() == 253)); + let csv = hosts.join(","); + assert_eq!(csv.len(), 8127); + let shared = ValidatedSettings::parse( + "", + &BTreeMap::from([("LAMBDA_DOCKER_REGISTRIES".into(), csv.clone())]), + ) + .unwrap(); + assert_eq!(shared.get("LAMBDA_DOCKER_REGISTRIES"), Some(csv.as_str())); + assert!( + settings(&[ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ("LAMBDA_DOCKER_REGISTRIES", &csv) + ]) + .is_ok() + ); + let csv = (0..33) + .map(|i| format!("r{i}.example")) + .collect::>() + .join(","); + assert!( + settings(&[ + ("LAMBDA_DOCKER_PULL_POLICY", "ifMissing"), + ("LAMBDA_DOCKER_REGISTRIES", &csv) + ]) + .is_err() + ); +} + +#[test] +fn test_should_require_explicit_paired_sdk_settings_and_a_connected_endpoint() { + let pair = [ + ("LAMBDA_DOCKER_SDK_ACCESS_KEY_ID", "private-sdk-key"), + ("LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY", "private-sdk-secret"), + ]; + assert!(settings(&pair).is_err()); + assert!(settings(&[("LAMBDA_DOCKER_SDK_ACCESS_KEY_ID", "private-sdk-key")]).is_err()); + assert!(settings(&[("LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY", "private-sdk-secret")]).is_err()); + let mut connected = pair.to_vec(); + connected.push(("LAMBDA_DOCKER_NETWORK", "sdk-bridge")); + assert!(settings(&connected).is_err()); + connected.push(("LAMBDA_DOCKER_GATEWAY_ENDPOINT", "http://127.0.0.1:4566")); + let configured = settings(&connected).unwrap(); + assert!(configured.sdk_credentials().is_some()); + assert!(!format!("{configured:?}").contains("private-sdk")); + let disabled = + settings(&[("LAMBDA_DOCKER_GATEWAY_ENDPOINT", "http://127.0.0.1:4566")]).unwrap(); + assert!(disabled.sdk_credentials().is_none()); +} +#[test] +fn test_should_never_query_ambient_or_server_credentials_for_handler_sdk_settings() { + let configured = DockerRuntimeSettings::from_lookup( + |key| { + assert!(!matches!( + key, + "AWS_ACCESS_KEY_ID" | "AWS_SECRET_ACCESS_KEY" | "ACCESS_KEY" | "SECRET_KEY" + )); + match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + _ => None, + } + }, + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap(); + assert!(configured.sdk_credentials().is_none()); +} +#[test] +fn test_should_accept_explicit_sdk_pair_through_the_existing_yaml_schema() { + let shared = ValidatedSettings::parse( + "environment:\n LAMBDA_DOCKER_RELAY_IMAGE: relay:v1\n LAMBDA_DOCKER_NETWORK: sdk-bridge\n LAMBDA_DOCKER_GATEWAY_ENDPOINT: http://127.0.0.1:4566\n LAMBDA_DOCKER_SDK_ACCESS_KEY_ID: private-sdk-key\n LAMBDA_DOCKER_SDK_SECRET_ACCESS_KEY: private-sdk-secret\n", + &BTreeMap::new(), + ).unwrap(); + let configured = DockerRuntimeSettings::from_lookup( + |key| shared.get(key).map(str::to_owned), + shared.budgets(), + Duration::from_secs(600), + ) + .unwrap(); + assert!(configured.sdk_credentials().is_some()); + assert!(!format!("{shared:?} {configured:?}").contains("private-sdk")); +} diff --git a/crates/rustack-lambda-core/src/docker/connection.rs b/crates/rustack-lambda-core/src/docker/connection.rs new file mode 100644 index 00000000..dfb913b8 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/connection.rs @@ -0,0 +1,394 @@ +//! One owned Unix HTTP exchange; the driver is joined on every outcome. + +use std::{future::pending, path::Path, sync::Arc}; + +use bytes::Bytes; +use http::{ + HeaderValue, Method, Request, Response, StatusCode, + header::{ + CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, HOST, TRANSFER_ENCODING, + UPGRADE, + }, +}; +use http_body_util::{BodyExt, Full}; +use hyper::{ + body::Incoming, + client::conn::http1::{Builder, SendRequest}, + upgrade::on, +}; +use hyper_util::rt::TokioIo; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use secrecy::ExposeSecret; +use tokio::{ + net::UnixStream, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use super::{ + DockerError, MAX_BODY_BYTES, Operation, RawReply, RawRequest, + binding::{DaemonBinding, DaemonIdentity}, + check_deadline, + pull::progress::{MAX_TOTAL, Progress}, +}; + +pub(super) async fn exchange( + path: &Path, + mut request: RawRequest, + deadline: Instant, + mut stop: watch::Receiver, +) -> Result { + // Keep the typed buffer reservation through HTTP-driver settlement, even + // after prepare_request transfers its byte view to Hyper. + let _archive = request.archive.take(); + // The exchange retains directory exclusion through every driver-join outcome. + let binding = request.binding.take(); + let mut pull_stop = request.pull.as_ref().map(|transfer| transfer.stop.clone()); + let mut pull_cancel = request + .pull + .as_ref() + .map(|transfer| transfer.cancel.clone()); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let connected = async { + let stream = UnixStream::connect(path).await?; + let pair = Builder::new() + .max_headers(64) + .max_buf_size(16 * 1024) + .handshake::<_, Full>(TokioIo::new(stream)) + .await?; + Ok::<_, DockerError>(pair) + }; + let (mut sender, connection) = tokio::select! { + biased; + () = stopped(&mut stop) => return Err(DockerError::Stopped), + error = pull_cancelled(&mut pull_stop,&mut pull_cancel) => return Err(error), + result = timeout_at(deadline, connected) => result.map_err(|_| DockerError::Deadline)??, + }; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if *stop.borrow() { + return Err(DockerError::Stopped); + } + let expected = request.expected; + let pulling = request.pull.is_some(); + let upgraded = request.expected == StatusCode::SWITCHING_PROTOCOLS; + let mut driver = tokio::spawn( + async move { + if upgraded { + connection.with_upgrades().await + } else { + // This fresh connection is never reused. Recover/drop its owned IO + // instead of half-closing an already-disconnected Unix socket. + connection.without_shutdown().await.map(drop) + } + } + .with_subscriber(runtime_dispatch()), + ); + let operation_stop = stop.clone(); + let operation = async { + if let Some(binding) = &binding { + check_identity(&mut sender, binding, deadline) + .await + .map_err(|source| DockerError::IdentityCheck(Arc::new(source)))?; + sender.ready().await?; + } + check_deadline(deadline)?; + if *operation_stop.borrow() { + return Err(DockerError::Stopped); + } + let mut request = request; + let mut mutation = request.mutation.take(); + let request = prepare_request(request, upgraded)?; + if let Some(mutation) = &mut mutation { + mutation.issued(); + } + let response = sender.send_request(request).await?; + if pulling { + read_pull(response, deadline).await + } else { + read_response(response, expected, deadline).await + } + }; + let outcome = tokio::select! { + biased; + () = stopped(&mut stop) => Err(DockerError::Stopped), + error = pull_cancelled(&mut pull_stop,&mut pull_cancel) => Err(error), + result = timeout_at(deadline, operation) => result.map_err(|_| DockerError::Deadline).and_then(|result| result), + }; + drop(sender); + let successful_upgrade = matches!(&outcome, Ok(RawReply::Upgrade(_))); + let joined = join_driver(&mut driver, deadline, successful_upgrade, &mut stop).await; + if matches!(&joined, Err(DockerError::Worker)) { + drop(outcome); + return Err(DockerError::Worker); + } + if Instant::now() >= deadline { + drop(outcome); + return Err(DockerError::Deadline); + } + if *stop.borrow() { + drop(outcome); + return Err(DockerError::Stopped); + } + match (outcome, joined) { + (Ok(reply), Ok(())) => Ok(reply), + (Err(error), _) | (_, Err(error)) => Err(error), + } +} + +fn prepare_request( + request: RawRequest, + upgraded: bool, +) -> Result>, DockerError> { + let mut builder = Request::builder() + .method(request.method) + .uri(request.target) + .header(HOST, "docker") + .header(CONTENT_LENGTH, request.body.len()) + .header(CONNECTION, if upgraded { "Upgrade" } else { "close" }); + if upgraded { + builder = builder.header(UPGRADE, "tcp"); + } + if !request.body.is_empty() { + builder = builder.header( + CONTENT_TYPE, + if matches!(request.operation, Operation::UploadCode) { + "application/x-tar" + } else { + "application/json" + }, + ); + } + if let Some(auth) = request + .pull + .as_ref() + .and_then(|transfer| transfer.auth.as_ref()) + { + let mut header = + HeaderValue::from_str(auth.expose_secret()).map_err(|_| DockerError::Configuration)?; + header.set_sensitive(true); + builder = builder.header("X-Registry-Auth", header); + } + builder + .body(Full::new(request.body)) + .map_err(|_| DockerError::Protocol) +} + +async fn pull_cancelled( + stop: &mut Option>, + cancel: &mut Option>, +) -> DockerError { + tokio::select! { biased; + () = async { match stop {Some(stop)=>stopped(stop).await,None=>pending().await} } => DockerError::Stopped, + () = async { match cancel {Some(cancel)=>{let _=cancel.changed().await;},None=>pending().await} } => DockerError::Cancelled, + } +} +async fn read_pull( + response: Response, + deadline: Instant, +) -> Result { + if response.status() != StatusCode::OK { + return read_response(response, StatusCode::OK, deadline).await; + } + check_headers_limit(&response, MAX_TOTAL)?; + let mut body = response.into_body(); + let mut progress = Progress::default(); + let mut quantum = 0_u8; + while let Some(frame) = body.frame().await { + check_deadline(deadline)?; + let data = frame?.into_data().map_err(|_| DockerError::Protocol)?; + progress.push(&data)?; + quantum = quantum.saturating_add(1); + if quantum == 64 { + tokio::task::yield_now().await; + quantum = 0; + } + } + progress.finish()?; + Ok(RawReply::Pulled) +} + +async fn check_identity( + sender: &mut SendRequest>, + binding: &DaemonBinding, + deadline: Instant, +) -> Result<(), DockerError> { + let request = Request::builder() + .method(Method::GET) + .uri("/info") + .header(HOST, "docker") + .header(CONTENT_LENGTH, 0) + .header(CONNECTION, "keep-alive") + .body(Full::new(Bytes::new())) + .map_err(|_| DockerError::Protocol)?; + let response = sender.send_request(request).await?; + let reply = read_response(response, StatusCode::OK, deadline).await?; + if DaemonIdentity::decode(reply, deadline)? != binding.identity { + return Err(DockerError::DaemonBinding); + } + Ok(()) +} + +async fn read_response( + response: Response, + expected: StatusCode, + deadline: Instant, +) -> Result { + let upgraded = expected == StatusCode::SWITCHING_PROTOCOLS; + check_headers(&response)?; + let status = response.status(); + if upgraded && status == StatusCode::SWITCHING_PROTOCOLS { + let headers = response.headers(); + if !headers + .get(UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"tcp")) + || !headers + .get(CONNECTION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) + }) + || headers.contains_key(TRANSFER_ENCODING) + || headers + .get(CONTENT_LENGTH) + .is_some_and(|value| value.as_bytes() != b"0") + { + return Err(DockerError::Protocol); + } + return Ok(RawReply::Upgrade(TokioIo::new(on(response).await?))); + } + if status == StatusCode::SWITCHING_PROTOCOLS { + return Err(DockerError::Protocol); + } + let body = collect(response.into_body(), deadline).await?; + if status != expected { + return Err(DockerError::Status(status)); + } + Ok(RawReply::Body { + status, + bytes: body, + }) +} + +async fn join_driver( + driver: &mut JoinHandle>, + deadline: Instant, + upgraded: bool, + stop: &mut watch::Receiver, +) -> Result<(), DockerError> { + if upgraded { + let joined = tokio::select! { + biased; + () = stopped(stop) => None, + result = timeout_at(deadline, &mut *driver) => result.ok(), + }; + if let Some(result) = joined { + return result + .map_err(|_| DockerError::Worker)? + .map_err(DockerError::from); + } + } + // Hyper connection IO is cancel-safe. Abort is followed by an actual join; + // it never replaces ownership of a container or an external backend job. + driver.abort(); + match driver.await { + Ok(result) => result.map_err(DockerError::from), + Err(error) if error.is_cancelled() => Ok(()), + Err(_) => Err(DockerError::Worker), + } +} + +pub(super) async fn stopped(stop: &mut watch::Receiver) { + let _ = stop.wait_for(|stop| *stop).await; +} + +fn check_headers(response: &Response) -> Result<(), DockerError> { + check_headers_limit(response, MAX_BODY_BYTES) +} +fn check_headers_limit(response: &Response, max_bytes: usize) -> Result<(), DockerError> { + let headers = response.headers(); + if headers.len() > 64 { + return Err(DockerError::TooLarge); + } + let mut total = 0_usize; + for (name, value) in headers { + total = total + .checked_add(name.as_str().len()) + .and_then(|length| length.checked_add(value.as_bytes().len())) + .ok_or(DockerError::TooLarge)?; + if name.as_str().len() > 64 || value.as_bytes().len() > 4096 || total > 16 * 1024 { + return Err(DockerError::TooLarge); + } + if !value + .as_bytes() + .iter() + .all(|byte| *byte == b'\t' || (32..=126).contains(byte)) + { + return Err(DockerError::Protocol); + } + } + if headers.get_all(CONTENT_LENGTH).iter().count() > 1 { + return Err(DockerError::Protocol); + } + if let Some(value) = headers.get(CONTENT_LENGTH) { + let value = value.to_str().map_err(|_| DockerError::Protocol)?; + if value.is_empty() || value.len() > 20 || !value.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(DockerError::Protocol); + } + if value.parse::().map_err(|_| DockerError::Protocol)? + > u64::try_from(max_bytes).map_err(|_| DockerError::TooLarge)? + { + return Err(DockerError::TooLarge); + } + } + if headers + .get(CONTENT_ENCODING) + .is_some_and(|value| !value.as_bytes().eq_ignore_ascii_case(b"identity")) + { + return Err(DockerError::Protocol); + } + if headers + .get(TRANSFER_ENCODING) + .is_some_and(|value| !value.as_bytes().eq_ignore_ascii_case(b"chunked")) + { + return Err(DockerError::Protocol); + } + Ok(()) +} + +async fn collect(mut body: Incoming, deadline: Instant) -> Result { + let mut output = Vec::new(); + let mut quantum = 0_u8; + while let Some(frame) = body.frame().await { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let frame = frame?; + let data = frame.into_data().map_err(|_| DockerError::Protocol)?; + let length = output + .len() + .checked_add(data.len()) + .ok_or(DockerError::TooLarge)?; + if length > MAX_BODY_BYTES { + return Err(DockerError::TooLarge); + } + output + .try_reserve(data.len()) + .map_err(|_| DockerError::Allocation)?; + output.extend_from_slice(&data); + quantum = quantum.saturating_add(1); + if quantum == 64 { + tokio::task::yield_now().await; + quantum = 0; + } + } + Ok(Bytes::from(output)) +} diff --git a/crates/rustack-lambda-core/src/docker/environment/actor.rs b/crates/rustack-lambda-core/src/docker/environment/actor.rs new file mode 100644 index 00000000..47f0010a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/actor.rs @@ -0,0 +1,479 @@ +//! Root registry owns reservations above workers and their recovery jobs. + +use std::{ + collections::HashMap, + mem, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use rustack_lambda_runtime::{diagnostics::runtime_dispatch, protocol::EnvironmentId}; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, watch}, + task::{Id, JoinError, JoinSet}, + time::{Instant, sleep}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; + +use super::{ + Call, EnvironmentConfig, EnvironmentError, EnvironmentPhase, EnvironmentStatus, + cleanup::{Cleaner, Progress}, + worker::Worker, +}; +use crate::{ + docker::{DockerPort, EnvironmentPlan, zip::AllocationClaim}, + executor::ExecutorError, + invocation::CleanupTask, +}; + +pub(super) struct Health { + control: watch::Sender, + pub(super) failed: AtomicBool, + pub(super) quarantined: AtomicUsize, +} +impl Default for Health { + fn default() -> Self { + Self::new(watch::channel(false).0) + } +} +impl Health { + pub(super) fn new(control: watch::Sender) -> Self { + Self { + control, + failed: AtomicBool::new(false), + quarantined: AtomicUsize::new(0), + } + } + pub(super) fn fail(&self) { + self.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } +} +pub(super) struct Budget { + capacity: Arc, + maximum: usize, + pub(super) memory: AtomicU64, + limit: u64, +} +impl Budget { + pub(super) fn new(config: EnvironmentConfig) -> Self { + let maximum = usize::from(config.0.environments); + Self { + capacity: Arc::new(Semaphore::new(maximum)), + maximum, + memory: AtomicU64::new(0), + limit: config.0.memory_bytes, + } + } + pub(super) fn count(&self) -> usize { + self.maximum + .saturating_sub(self.capacity.available_permits()) + } + pub(super) fn check_memory(&self, memory: u64) -> Result<(), EnvironmentError> { + if memory > self.limit { + Err(EnvironmentError::MemoryLimit) + } else { + Ok(()) + } + } + pub(super) fn reserve(self: &Arc, memory: u64) -> Result { + self.check_memory(memory)?; + let permit = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| EnvironmentError::Busy)?; + self.memory + .try_update(Ordering::AcqRel, Ordering::Acquire, |current| { + current + .checked_add(memory) + .filter(|total| *total <= self.limit) + }) + .map_err(|_| EnvironmentError::Busy)?; + Ok(Reservation { + _permit: permit, + memory, + budget: self.clone(), + }) + } +} +pub(super) struct Reservation { + _permit: OwnedSemaphorePermit, + memory: u64, + budget: Arc, +} +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.memory.fetch_sub(self.memory, Ordering::AcqRel); + } +} +#[derive(Clone)] +pub(super) struct Signals { + pub(super) status: watch::Sender, + pub(super) stop: watch::Sender, +} +impl Signals { + pub(super) fn new() -> Self { + Self { + status: watch::channel(EnvironmentStatus { + phase: EnvironmentPhase::Allocating, + error: None, + }) + .0, + stop: watch::channel(false).0, + } + } + pub(super) fn phase(&self, phase: EnvironmentPhase) { + self.status.send_modify(|status| status.phase = phase); + } + pub(super) fn error(&self, error: EnvironmentError) { + self.status.send_modify(|status| { + if status.error.is_none() { + status.error = Some(error); + } + }); + } + fn worker_failed(&self) { + self.stop.send_replace(true); + self.status.send_modify(|status| { + status.phase = EnvironmentPhase::Retiring; + status.error = Some(EnvironmentError::Worker); + }); + } +} +#[derive(TypedBuilder)] +pub(super) struct Envelope { + #[builder(default)] + pub(super) code_claim: Option, + pub(super) plan: Arc, + pub(super) mailbox: mpsc::Receiver, + pub(super) signals: Signals, + pub(super) reservation: Reservation, + pub(super) done: watch::Sender>>, + pub(super) deadline: Instant, +} +pub(super) struct Record { + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) io: Arc, + pub(super) signals: Signals, + pub(super) done: watch::Sender>>, + _reservation: Reservation, +} +impl Record { + fn recovery(&self) -> Recovery { + Recovery { + plan: self.plan.clone(), + progress: self.progress.clone(), + io: self.io.clone(), + signals: self.signals.clone(), + done: self.done.clone(), + } + } +} +pub(super) struct Recovery { + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) io: Arc, + pub(super) signals: Signals, + pub(super) done: watch::Sender>>, +} +impl Recovery { + async fn run( + self, + docker: DockerPort, + config: EnvironmentConfig, + health: Arc, + cleanup: Arc, + ) { + let cleaner = Cleaner::builder() + .docker(docker) + .config(config) + .plan(self.plan) + .progress(self.progress) + .health(health) + .cleanup(cleanup) + .signals(self.signals.clone()) + .build(); + cleaner.retire(&self.io, None).await; + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Err(EnvironmentError::Worker))); + } +} + +type Job = (EnvironmentId, Result<(), EnvironmentError>); +type Joined = Result<(Id, Job), JoinError>; +#[derive(TypedBuilder)] +pub(super) struct Actor { + docker: DockerPort, + config: EnvironmentConfig, + inbox: mpsc::Receiver, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default = Arc::new(Semaphore::new(2)))] + cleanup: Arc, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + records: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + pub(super) async fn run(mut self) -> Result<(), EnvironmentError> { + let mut stopped = self.control.subscribe(); + let mut wire = self.docker.raw.stop.subscribe(); + loop { + if *stopped.borrow() { + self.inbox.close(); + while let Ok(envelope) = self.inbox.try_recv() { + let failed = self.health.failed.load(Ordering::Acquire); + envelope.signals.error(if failed { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + }); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope.done.send_replace(Some(if failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + for record in self.records.values() { + if self.health.failed.load(Ordering::Acquire) { + record.signals.worker_failed(); + } else { + record.signals.stop.send_replace(true); + } + } + if self.jobs.is_empty() { + break; + } + if let Some(result) = self.jobs.join_next_with_id().await { + self.finished(&result); + } + continue; + } + tokio::select! {biased; + _=wire.wait_for(|stop|*stop)=>{self.fail();}, + _=stopped.wait_for(|stop|*stop)=>{}, + result=self.jobs.join_next_with_id(),if !self.jobs.is_empty()=>if let Some(result)=result{self.finished(&result);}, + envelope=self.inbox.recv()=>match envelope{Some(envelope)=>self.start(envelope),None=>{self.control.send_replace(true);}}, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(EnvironmentError::Worker) + } else { + Ok(()) + } + } + fn start(&mut self, mut envelope: Envelope) { + if *self.control.borrow() + || *envelope.signals.stop.borrow() + || Instant::now() >= envelope.deadline + { + envelope.signals.error(EnvironmentError::Stopped); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope.done.send_replace(Some(Ok(()))); + return; + } + let id = envelope.plan.environment_id(); + if self.records.contains_key(&id) { + self.fail(); + envelope.signals.error(EnvironmentError::Worker); + envelope + .done + .send_replace(Some(Err(EnvironmentError::Worker))); + return; + } + if let Err(error) = envelope.plan.commit_code_claim(&mut envelope.code_claim) { + envelope.signals.error(error.into()); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope.done.send_replace(Some(Ok(()))); + return; + } + let progress = Arc::new(Progress::new()); + let io = Arc::new(CleanupTask::default()); + let worker = Worker::builder() + .docker(self.docker.clone()) + .config(self.config) + .plan(envelope.plan.clone()) + .progress(progress.clone()) + .cleanup(self.cleanup.clone()) + .health(self.health.clone()) + .signals(envelope.signals.clone()) + .global(self.control.clone()) + .io(io.clone()) + .runtime(self.runtime.clone()) + .build(); + self.records.insert( + id, + Record { + plan: envelope.plan, + progress, + io, + signals: envelope.signals, + done: envelope.done, + _reservation: envelope.reservation, + }, + ); + let job = self.jobs.spawn( + async move { (id, worker.run(envelope.mailbox, envelope.deadline).await) } + .with_subscriber(runtime_dispatch()), + ); + self.index.insert(job.id(), (id, false)); + } + fn finished(&mut self, result: &Joined) { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some((id, recovering)) = self.index.remove(&task) else { + self.fail(); + return; + }; + let clean = self + .records + .get(&id) + .is_some_and(|record| record.progress.clean() && record.plan.code_clean()); + if matches!(&result,Ok((_,(returned,Ok(())))) if *returned==id) && clean { + if let Some(record) = self.records.remove(&id) { + let failed = recovering || self.health.failed.load(Ordering::Acquire); + if failed { + record.signals.worker_failed(); + } + record.signals.phase(EnvironmentPhase::Removed); + record.done.send_replace(Some(if failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + return; + } + self.fail(); + let Some(record) = self.records.get(&id) else { + return; + }; + record.signals.worker_failed(); + let recovery = record.recovery(); + let docker = self.docker.clone(); + let config = self.config; + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + let job = self.jobs.spawn(async move { + if recovering { + sleep(Duration::from_secs(1)).await; + } + recovery.run(docker, config, health, cleanup).await; + (id, Ok(())) + }); + self.index.insert(job.id(), (id, true)); + } + fn fail(&self) { + self.health.fail(); + self.control.send_replace(true); + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + while let Ok(envelope) = self.inbox.try_recv() { + envelope.signals.worker_failed(); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope + .done + .send_replace(Some(Err(EnvironmentError::Worker))); + } + if self.records.is_empty() { + return; + } + self.fail(); + for record in self.records.values() { + record.signals.worker_failed(); + } + let records = mem::take(&mut self.records); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + let docker = self.docker.clone(); + let config = self.config; + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover_all(records, docker, config, health, cleanup).await + }); + } +} +async fn recover_all( + mut records: HashMap, + docker: DockerPort, + config: EnvironmentConfig, + health: Arc, + cleanup: Arc, +) -> Result<(), ExecutorError> { + let mut jobs = JoinSet::new(); + let mut index = HashMap::new(); + for (id, record) in &records { + let id = *id; + let recovery = record.recovery(); + let docker = docker.clone(); + let health = health.clone(); + let cleanup = cleanup.clone(); + let job = jobs.spawn(async move { + recovery.run(docker, config, health, cleanup).await; + id + }); + index.insert(job.id(), id); + } + while let Some(result) = jobs.join_next_with_id().await { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some(id) = index.remove(&task) else { + continue; + }; + if matches!(result,Ok((_,returned)) if returned==id) + && records + .get(&id) + .is_some_and(|record| record.progress.clean() && record.plan.code_clean()) + { + records.remove(&id); + continue; + } + let Some(record) = records.get(&id) else { + continue; + }; + let recovery = record.recovery(); + let docker = docker.clone(); + let health = health.clone(); + let cleanup = cleanup.clone(); + let job = jobs.spawn(async move { + sleep(Duration::from_secs(1)).await; + recovery.run(docker, config, health, cleanup).await; + id + }); + index.insert(job.id(), id); + } + if records.is_empty() { + Ok(()) + } else { + Err(ExecutorError::CleanupFailed( + "Environment recovery lost ownership".into(), + )) + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/cleanup.rs b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs new file mode 100644 index 00000000..f4b6e714 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs @@ -0,0 +1,266 @@ +//! Phase-aware ID-only cleanup; uncertain create absence never grants settlement. + +use std::{ + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use http::StatusCode; +use tokio::{ + sync::{Semaphore, watch}, + time::sleep, +}; +use typed_builder::TypedBuilder; + +use super::{ + EnvironmentConfig, EnvironmentError, EnvironmentPhase, + actor::{Health, Signals}, + io::IoGuard, +}; +use crate::{ + docker::{ + DockerError, DockerKey, DockerPort, EnvironmentPlan, models::ContainerInspectResponse, + mutation::MutationReceipt, ownership::ContainerRecord, plan::ContainerRole, + }, + invocation::CleanupTask, +}; + +#[derive(Debug, Clone)] +pub(super) enum Phase { + Unissued, + Uncertain(MutationReceipt), + Known(DockerKey), + Clean, +} +#[derive(Debug)] +pub(super) struct Progress { + pub(super) relay: watch::Sender, + pub(super) function: watch::Sender, +} +impl Progress { + pub(super) fn new() -> Self { + Self { + relay: watch::channel(Phase::Unissued).0, + function: watch::channel(Phase::Unissued).0, + } + } + pub(super) fn phase(&self, role: ContainerRole) -> &watch::Sender { + match role { + ContainerRole::Relay => &self.relay, + ContainerRole::Function => &self.function, + } + } + pub(super) fn clean(&self) -> bool { + [&self.relay, &self.function] + .iter() + .all(|phase| matches!(*phase.borrow(), Phase::Unissued | Phase::Clean)) + } +} + +#[derive(TypedBuilder)] +pub(super) struct Cleaner { + pub(super) docker: DockerPort, + pub(super) config: EnvironmentConfig, + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) health: Arc, + pub(super) cleanup: Arc, + pub(super) signals: Signals, +} +impl Cleaner { + async fn settled(&self, role: ContainerRole) -> bool { + let Ok(record) = ContainerRecord::from_plan(&self.plan, role) else { + return false; + }; + if self + .plan + .directory + .settle_container(record, self.config.deadline()) + .await + .is_err() + { + return false; + } + self.progress.phase(role).send_replace(Phase::Clean); + true + } + pub(super) async fn container(&self, role: ContainerRole) -> bool { + let phase = self.progress.phase(role).borrow().clone(); + let id = match phase { + Phase::Unissued | Phase::Clean => return self.settled(role).await, + Phase::Known(id) => id, + Phase::Uncertain(receipt) => { + if receipt.definitely_unissued() { + return self.settled(role).await; + } + + let Ok(inspected) = self + .docker + .inspect_container(self.plan.key(role), self.config.deadline()) + .await + else { + return false; + }; + let Ok(id) = self.plan.owned(role, &inspected) else { + return false; + }; + let Ok(record) = ContainerRecord::from_plan(&self.plan, role) else { + return false; + }; + if self + .plan + .directory + .know_container(record, id.clone(), self.config.deadline()) + .await + .is_err() + { + return false; + } + self.progress + .phase(role) + .send_replace(Phase::Known(id.clone())); + id + } + }; + let Ok(_permit) = self.cleanup.acquire().await else { + return false; + }; + let inspected = match self + .docker + .inspect_container(&id, self.config.deadline()) + .await + { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + return self.settled(role).await; + } + Ok(inspected) => inspected, + Err(_) => return false, + }; + if self.plan.owned(role, &inspected).is_err() + || inspected.id.as_deref() != Some(id.as_str()) + { + return false; + } + let Some(state) = inspected.state else { + return false; + }; + match state.running { + Some(true) => { + let _ = self + .docker + .kill_container(&id, self.config.deadline()) + .await; + let waited = self + .docker + .wait_container(&id, self.config.deadline()) + .await; + if !matches!(waited,Ok(ref exit) if exit.error.is_none()) { + return false; + } + } + Some(false) => {} + None => return false, + } + let _ = self + .docker + .remove_container(&id, self.config.deadline()) + .await; + if matches!( + self.docker + .inspect_container(&id, self.config.deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ) { + self.settled(role).await + } else { + false + } + } + pub(super) async fn retire(&self, io: &CleanupTask, mut streams: Option<&mut IoGuard>) { + let mut dirty = None; + let mut delay = Duration::from_secs(1); + loop { + let function = self.container(ContainerRole::Function).await; + if function { + if let Some(streams) = streams.take() { + streams.begin_cleanup(); + } + } + let joined = if function { io.wait().await } else { Ok(()) }; + let code = if let Some(zip) = &self.plan.zip { + if function && joined.is_ok() { + match zip.retire(self.config.deadline()).await { + Ok(failed) => { + if failed { + self.health.fail(); + } + true + } + Err(_) => false, + } + } else { + false + } + } else { + true + }; + let relay = if function && code { + self.container(ContainerRole::Relay).await + } else { + false + }; + if function && relay && code && joined.is_ok() { + return; + } + if joined.is_err() { + self.health.fail(); + } + if dirty.is_none() { + dirty = Some(Dirty::new(self.health.clone())); + } + self.signals.phase(EnvironmentPhase::Quarantined); + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + pub(super) async fn inspect_health(&self) -> Result<(), EnvironmentError> { + self.plan.live()?; + let phase = self.progress.function.borrow().clone(); + let Phase::Known(id) = phase else { + return Err(EnvironmentError::Stopped); + }; + let inspected = self + .docker + .inspect_container(&id, self.config.deadline()) + .await?; + if self.plan.owned(ContainerRole::Function, &inspected)? != id { + return Err(DockerError::Quarantined.into()); + } + function_health(&inspected) + } +} +pub(super) fn function_health( + inspected: &ContainerInspectResponse, +) -> Result<(), EnvironmentError> { + let state = inspected.state.as_ref().ok_or(DockerError::Protocol)?; + match state.running { + Some(true) => Ok(()), + Some(false) => Err(EnvironmentError::Exited { + exit_code: state.exit_code.ok_or(DockerError::Protocol)?, + oom: state.oom_killed.ok_or(DockerError::Protocol)?, + }), + None => Err(DockerError::Protocol.into()), + } +} +struct Dirty(Arc); +impl Dirty { + fn new(health: Arc) -> Self { + health.quarantined.fetch_add(1, Ordering::AcqRel); + Self(health) + } +} +impl Drop for Dirty { + fn drop(&mut self) { + self.0.quarantined.fetch_sub(1, Ordering::AcqRel); + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/io.rs b/crates/rustack-lambda-core/src/docker/environment/io.rs new file mode 100644 index 00000000..0704e531 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/io.rs @@ -0,0 +1,234 @@ +//! Worker-owned streams transfer to retained cleanup before their guard drops. + +use std::{collections::VecDeque, mem::take, sync::Arc, time::Duration}; + +use bytes::Bytes; +use rustack_lambda_runtime::{ + client::{ClientConfig, ControlClient}, + diagnostics::runtime_dispatch, + protocol::{Architecture, AttemptId, EnvironmentId}, + relay::RelayError, +}; +use tokio::{ + io::{AsyncWriteExt, duplex}, + runtime::Handle, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use super::{EnvironmentConfig, EnvironmentError, actor::Signals}; +use crate::{ + docker::{ + DockerAttachment, DockerError, FunctionLogs, LogSnapshot, LogWindow, OutputChannel, + connection::stopped as wait_for_stop, + }, + executor::ExecutorError, + invocation::CleanupTask, +}; + +struct Bridge { + stop: watch::Sender, + task: JoinHandle>, + status: watch::Receiver>>, +} +#[derive(Default)] +struct Streams { + client: Option, + logs: Option, + bridge: Option, +} +pub(super) struct IoGuard { + streams: Streams, + runtime: Handle, + cleanup: Arc, + config: EnvironmentConfig, + signals: Signals, + closing: bool, + logs_closed: bool, +} +impl IoGuard { + #[cfg(test)] + pub(super) fn test_streams(&mut self, client: ControlClient, logs: FunctionLogs) { + self.streams.client = Some(client); + self.streams.logs = Some(logs); + } + + pub(super) fn new( + runtime: Handle, + cleanup: Arc, + config: EnvironmentConfig, + signals: Signals, + ) -> Self { + Self { + streams: Streams::default(), + runtime, + cleanup, + config, + signals, + closing: false, + logs_closed: false, + } + } + pub(super) fn control( + &mut self, + attachment: DockerAttachment, + environment: EnvironmentId, + architecture: Architecture, + ) -> Result<(), EnvironmentError> { + let (mut output, input) = attachment.split(self.config.0.io)?; + let (mut bridge, reader) = duplex(64 * 1024); + self.streams.client = Some(ControlClient::start( + ClientConfig::new(environment, architecture, self.config.0.io)?, + reader, + input, + )?); + let (stop, mut stopped) = watch::channel(false); + let (status, observed) = watch::channel(None); + let budget = self.config.0.io; + let mut diagnostics = VecDeque::new(); + diagnostics + .try_reserve_exact(65536) + .map_err(|_| DockerError::Allocation)?; + let task=self.runtime.spawn(async move{ + let result=async{ + loop{ + let chunk=tokio::select!{biased;()=wait_for_stop(&mut stopped)=>return Ok(()),chunk=output.next(Instant::now()+Duration::from_secs(3600))=>chunk?,}; + match chunk{ + None=>return Ok(()), + Some(chunk) if chunk.channel==OutputChannel::Stdout=>{timeout_at(Instant::now()+budget,bridge.write_all(&chunk.bytes)).await.map_err(|_|DockerError::Deadline)??;}, + Some(chunk)=>{let remove=diagnostics.len().saturating_add(chunk.bytes.len()).saturating_sub(65536);diagnostics.drain(..remove.min(diagnostics.len()));diagnostics.extend(chunk.bytes.iter().copied());}, + } + } + }.await;status.send_replace(Some(result.clone()));result + }.with_subscriber(runtime_dispatch())); + self.streams.bridge = Some(Bridge { + stop, + task, + status: observed, + }); + Ok(()) + } + pub(super) fn client(&mut self) -> Result<&mut ControlClient, EnvironmentError> { + self.streams.client.as_mut().ok_or(EnvironmentError::Worker) + } + pub(super) fn logs(&mut self, attachment: DockerAttachment) -> Result<(), EnvironmentError> { + self.streams.logs = Some(FunctionLogs::start( + attachment, + self.config.0.io, + self.config.0.log_ring_bytes, + )?); + Ok(()) + } + pub(super) async fn tail(&mut self) -> Result { + let logs = self.streams.logs.as_ref().ok_or(EnvironmentError::Worker)?; + let snapshot = logs.snapshot(self.config.deadline()).await?; + self.observe(snapshot) + } + pub(super) async fn begin_logs( + &mut self, + attempt: AttemptId, + ) -> Result { + let logs = self.streams.logs.as_mut().ok_or(EnvironmentError::Worker)?; + Ok(logs.begin(attempt, self.config.deadline()).await?) + } + pub(super) async fn finish_logs( + &mut self, + window: LogWindow, + ) -> Result { + let logs = self.streams.logs.as_mut().ok_or(EnvironmentError::Worker)?; + let snapshot = logs.finish(window, self.config.deadline()).await?; + self.observe(snapshot) + } + fn observe(&mut self, snapshot: LogSnapshot) -> Result { + if let Some(error) = snapshot.failure { + return Err(error.into()); + } + self.logs_closed = snapshot.closed; + Ok(snapshot.tail) + } + pub(super) async fn healthy(&mut self) -> Result<(), EnvironmentError> { + if let Some(bridge) = &self.streams.bridge { + if let Some(result) = bridge.status.borrow().as_ref() { + return match result { + Ok(()) => Err(DockerError::Protocol.into()), + Err(error) => Err(error.clone().into()), + }; + } + if bridge.task.is_finished() && bridge.status.borrow().is_none() { + return Err(EnvironmentError::Worker); + } + } + self.tail().await?; + if self.logs_closed { + Err(DockerError::Protocol.into()) + } else { + Ok(()) + } + } + pub(super) fn reusable(&self) -> bool { + !self.logs_closed + && self + .streams + .bridge + .as_ref() + .is_none_or(|bridge| bridge.status.borrow().is_none()) + } + pub(super) fn begin_cleanup(&mut self) { + if self.closing { + return; + } + self.closing = true; + let streams = take(&mut self.streams); + let config = self.config; + let signals = self.signals.clone(); + self.cleanup.start_on(&self.runtime, move || async move { + close(streams, config, signals).await + }); + } +} +impl Drop for IoGuard { + fn drop(&mut self) { + self.begin_cleanup(); + } +} +async fn close( + mut streams: Streams, + config: EnvironmentConfig, + signals: Signals, +) -> Result<(), ExecutorError> { + let mut failed = false; + if let Some(client) = &mut streams.client { + if let Err(error) = client.shutdown(config.deadline()).await { + failed |= matches!(error, RelayError::Worker); + signals.error(error.into()); + } + } + if let Some(bridge) = &streams.bridge { + bridge.stop.send_replace(true); + } + if let Some(logs) = &mut streams.logs { + if let Err(error) = logs.shutdown().await { + failed |= matches!(error, DockerError::Worker); + signals.error(error.into()); + } + } + if let Some(bridge) = streams.bridge { + match bridge.task.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + signals.error(error.into()); + } + Err(_) => failed = true, + } + } + if failed { + signals.error(EnvironmentError::Worker); + Err(ExecutorError::CleanupFailed( + "Environment IO supervision failed".into(), + )) + } else { + Ok(()) + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs new file mode 100644 index 00000000..6f5d42a0 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -0,0 +1,677 @@ +//! Retained two-container environments; executor pooling and startup readiness are separate. + +mod actor; +mod cleanup; +mod io; +mod worker; + +use std::{ + fmt, + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use actor::{Actor, Budget, Envelope, Health, Signals}; +use bytes::Bytes; +use rustack_lambda_runtime::{ + protocol::{EnvironmentId, Payload}, + relay::RelayError, + session::{Invocation, RuntimeResult}, +}; +use serde::Serialize; +use tokio::{ + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{DirectoryLease, DockerError, DockerPort, EnvironmentPlan, check_deadline}; +use crate::invocation::CleanupTask; + +/// Raw operator budgets; convert to `EnvironmentConfig` before starting ownership. +#[derive(Debug, Clone, Copy, TypedBuilder)] +#[non_exhaustive] +pub struct EnvironmentSettings { + /// Count of allocated environments, including quarantine, in1..32. + #[builder(default = 32)] + pub environments: u8, + /// Aggregate function+relay memory,256MiB..1TiB. + #[builder(default = 8 * 1024 * 1024 * 1024)] + pub memory_bytes: u64, + /// Independent Docker/control IO budget,1..60s. + #[builder(default = Duration::from_secs(5))] + pub io: Duration, + /// Cold startup budget,1..60s, also bounded by caller admission deadline. + #[builder(default = Duration::from_secs(15))] + pub start: Duration, + /// Independent function `BootstrapReady` wait,1..900s, also bounded by total start. + #[builder(default = Duration::from_secs(60))] + pub init: Duration, + /// Idle lifetime,1..86400s. + #[builder(default = Duration::from_secs(600))] + pub idle: Duration, + /// Retained bytes per function-output channel,4KiB..1MiB. + #[builder(default = 262_144)] + pub log_ring_bytes: usize, +} +impl Default for EnvironmentSettings { + fn default() -> Self { + Self::builder().build() + } +} + +/// Closed, validated environment ownership budgets. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct EnvironmentConfig(EnvironmentSettings); +impl TryFrom for EnvironmentConfig { + type Error = EnvironmentError; + fn try_from(settings: EnvironmentSettings) -> Result { + if !(1..=32).contains(&settings.environments) + || !(256 * 1024 * 1024..=1024 * 1024 * 1024 * 1024).contains(&settings.memory_bytes) + || !(4096..=1024 * 1024).contains(&settings.log_ring_bytes) + { + return Err(EnvironmentError::Configuration); + } + for (duration, maximum) in [ + (settings.io, 60), + (settings.start, 60), + (settings.init, 900), + (settings.idle, 86400), + ] { + if duration.subsec_nanos() != 0 || !(1..=maximum).contains(&duration.as_secs()) { + return Err(EnvironmentError::Configuration); + } + } + Ok(Self(settings)) + } +} +impl EnvironmentConfig { + fn deadline(self) -> Instant { + Instant::now() + self.0.io + } +} + +/// Environment errors retain typed sources and never display peer payload bytes. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum EnvironmentError { + /// Invalid settings, scope, captured invocation or lifecycle operation. + #[error("invalid environment configuration")] + Configuration, + /// Allocation or command capacity is exhausted. + #[error("environment capacity exhausted")] + Busy, + /// Function plus relay cannot fit the configured total memory, even when idle. + #[error("function and relay memory exceed the configured environment memory limit")] + MemoryLimit, + /// Explicit cancellation or abandoned owner/caller. + #[error("environment cancelled")] + Cancelled, + /// Startup, call or waiting deadline expired. + #[error("environment deadline exceeded")] + Deadline, + /// Function bootstrap did not poll before its independent initialization window. + #[error("function bootstrap initialization deadline exceeded")] + InitializationTimeout, + /// The captured function timeout expired after Runtime API delivery. + #[error("function timed out after {0:?}")] + Timeout(Duration), + /// This owner no longer accepts work. + #[error("environment is stopped")] + Stopped, + /// A worker/root/cleanup supervisor failed. + #[error("environment supervision failed")] + Worker, + /// Persistent container ownership could not be read or durably settled. + #[error("persistent container ownership failed")] + Ownership(#[source] Arc), + /// Function reported a bounded initialization error. + #[error("function initialization failed")] + Initialization(Payload), + /// Function exited before its expected result/readiness. + #[error("function exited with status {exit_code}; oom={oom}")] + Exited { + /// Daemon exit status, without peer text. + exit_code: i64, + /// Daemon-reported OOM kill flag. + oom: bool, + }, + /// Validated local Docker transport failure. + #[error(transparent)] + Docker(#[from] DockerError), + /// Ordered relay session failure. + #[error(transparent)] + Relay(#[from] RelayError), +} +impl From for EnvironmentError { + fn from(error: super::DirectoryLeaseError) -> Self { + Self::Ownership(Arc::new(error)) + } +} + +/// Current ownership/lifecycle state, independent of external API readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum EnvironmentPhase { + /// An allocation is registered and creating its relay. + Allocating, + /// Relay/function initialization is in progress. + Initializing, + /// Ready for one invocation. + Idle, + /// An admitted invocation is active. + Busy, + /// All owned resources are being retired. + Retiring, + /// Failed/uncertain cleanup retains its allocation reservation. + Quarantined, + /// Containers and retained IO are confirmed settled. + Removed, +} +/// Redacted lifecycle projection; a business failure does not imply unowned cleanup. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EnvironmentStatus { + /// Current owner state. + pub phase: EnvironmentPhase, + /// Primary failure, retained across cleanup retries. + pub error: Option, +} +/// Allocation accounting includes every queued, live and quarantined reservation. +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct EnvironmentCapacity { + /// Number of retained allocation slots. + pub allocated: usize, + /// Aggregate reserved memory bytes, including relay overhead. + pub memory_bytes: u64, + /// Environments retaining failed cleanup. + pub quarantined: usize, + /// Whether admission is open and healthy. + pub accepting: bool, + /// A supervision failure permanently closed admission. + pub worker_failed: bool, +} +/// Actual handler result and bounded optional log output; Debug redacts user bytes. +#[derive(Clone)] +#[non_exhaustive] +pub struct EnvironmentResponse { + /// Unmodified handler success or `FunctionError` payload. + pub result: RuntimeResult, + /// Combined observed invocation-window tail when requested, at most 4 KiB. + pub logs: Bytes, +} +impl fmt::Debug for EnvironmentResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentResponse") + .field("result", &self.result) + .field("log_bytes", &self.logs.len()) + .finish() + } +} +struct Call { + invocation: Invocation, + capture_logs: bool, + cancellation: watch::Receiver, + reply: oneshot::Sender>, +} +struct CancelOnDrop { + call: watch::Sender, + environment: watch::Sender, + complete: bool, +} +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if !self.complete { + self.call.send_replace(true); + self.environment.send_replace(true); + } + } +} + +/// Exclusive owner-facing environment handle; Drop requests retained retirement. +#[non_exhaustive] +pub struct EnvironmentHandle { + id: EnvironmentId, + calls: mpsc::Sender, + signals: Signals, + settlement: watch::Receiver>>, +} +impl fmt::Debug for EnvironmentHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentHandle") + .field("id", &self.id) + .field("status", &*self.signals.status.borrow()) + .finish_non_exhaustive() + } +} +impl EnvironmentHandle { + pub(super) fn retirement(&self) -> Retirement { + Retirement { + stop: self.signals.stop.clone(), + status: self.signals.status.subscribe(), + settlement: self.settlement.clone(), + } + } + /// Read the fresh immutable environment identity. + /// + /// ```no_run + /// # fn id(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let id = environment.id(); } + /// ``` + #[must_use] + pub fn id(&self) -> EnvironmentId { + self.id + } + /// Read lifecycle status without waiting for a business mailbox. + /// + /// ```no_run + /// # fn status(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let status = environment.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> EnvironmentStatus { + self.signals.status.borrow().clone() + } + /// Request retirement through its priority stop signal. + /// + /// ```no_run + /// # fn stop(environment: &rustack_lambda_core::docker::EnvironmentHandle) { environment.cancel(); } + /// ``` + pub fn cancel(&self) { + self.signals.stop.send_replace(true); + } + /// Await readiness without cancelling an accepted allocation when dropped. + /// + /// # Errors + /// Returns invalid/expired budget, startup failure or owner failure. The owner + /// remains responsible for shutdown/Drop and retained cleanup. + /// + /// ```no_run + /// # async fn ready(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let ready = environment.wait_ready(tokio::time::Instant::now()+std::time::Duration::from_secs(15)).await; } + /// ``` + pub async fn wait_ready(&self, deadline: Instant) -> Result<(), EnvironmentError> { + check_deadline(deadline)?; + let mut status = self.signals.status.subscribe(); + let observed = timeout_at( + deadline, + status.wait_for(|status| { + !matches!( + status.phase, + EnvironmentPhase::Allocating | EnvironmentPhase::Initializing + ) + }), + ) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)? + .clone(); + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if matches!( + observed.phase, + EnvironmentPhase::Idle | EnvironmentPhase::Busy + ) { + Ok(()) + } else { + Err(observed.error.unwrap_or(EnvironmentError::Stopped)) + } + } + /// Submit one runtime invocation to a ready environment, without replay. + /// + /// # Errors + /// Returns bounded admission, deadline/cancellation, protocol or function-exit + /// errors. Dropping this future retires the environment; shutdown separately + /// awaits cleanup. The supplied absolute invocation deadline is authoritative. + /// + /// ```no_run + /// # async fn invoke(environment: &mut rustack_lambda_core::docker::EnvironmentHandle, invocation: rustack_lambda_runtime::session::Invocation) { let result = environment.invoke(invocation, true).await; } + /// ``` + pub async fn invoke( + &mut self, + invocation: Invocation, + capture_logs: bool, + ) -> Result { + let deadline = invocation.parts().1; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if self.status().phase != EnvironmentPhase::Idle { + return Err(EnvironmentError::Stopped); + } + let (stop, cancellation) = watch::channel(false); + let mut cancel = CancelOnDrop { + call: stop, + environment: self.signals.stop.clone(), + complete: false, + }; + let (reply, response) = oneshot::channel(); + self.calls + .try_send(Call { + invocation, + capture_logs, + cancellation, + reply, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => EnvironmentError::Stopped, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + cancel.complete = result.is_ok(); + result + } + /// Request stop and await retained container/IO settlement under one deadline. + /// + /// # Errors + /// Returns timeout while ownership remains retained, or cached supervision + /// failure. Business failure remains in status, independent of clean retirement. + /// + /// ```no_run + /// # async fn stop(environment: &mut rustack_lambda_core::docker::EnvironmentHandle) { let clean = environment.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.cancel(); + check_deadline(deadline)?; + let settled = timeout_at(deadline, self.settlement.wait_for(Option::is_some)) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)? + .clone() + .ok_or(EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + settled + } +} +impl Drop for EnvironmentHandle { + fn drop(&mut self) { + self.cancel(); + } +} + +/// Cloneable bounded allocation port; retain its manager through shutdown. +#[derive(Clone, TypedBuilder)] +#[builder( + builder_method(vis = ""), + build_method(vis = ""), + builder_type(vis = "") +)] +#[non_exhaustive] +pub struct EnvironmentPort { + sender: mpsc::Sender, + directory: DirectoryLease, + config: EnvironmentConfig, + control: watch::Sender, + budget: Arc, + health: Arc, +} +impl fmt::Debug for EnvironmentPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPort") + .field("capacity", &self.status()) + .finish_non_exhaustive() + } +} +impl EnvironmentPort { + pub(super) fn same_service(&self, other: &Self) -> bool { + self.sender.same_channel(&other.sender) + } + pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { + self.directory.owner_id() == directory.owner_id() + && self.directory.run_id() == directory.run_id() + } + /// Reserve count/memory and register a plan before any container operation. + /// + /// # Errors + /// Returns invalid scope/deadline, closed/unhealthy owner or bounded capacity. + /// Startup runs under the earlier of admission deadline and configured budget. + /// + /// ```no_run + /// # fn allocate(port: &rustack_lambda_core::docker::EnvironmentPort, plan: rustack_lambda_core::docker::EnvironmentPlan) { let environment = port.spawn(plan,tokio::time::Instant::now()+std::time::Duration::from_secs(15)); } + /// ``` + pub fn spawn( + &self, + plan: EnvironmentPlan, + deadline: Instant, + ) -> Result { + self.spawn_shared(Arc::new(plan), deadline) + } + pub(super) fn check_memory(&self, memory: u64) -> Result<(), EnvironmentError> { + self.budget.check_memory(memory) + } + pub(super) fn spawn_shared( + &self, + plan: Arc, + deadline: Instant, + ) -> Result { + let deadline = deadline.min(Instant::now() + self.config.0.start); + check_deadline(deadline)?; + if !plan.same_scope(&self.directory) { + return Err(EnvironmentError::Configuration); + } + if *self.control.borrow() || self.health.failed.load(Ordering::Acquire) { + return Err(EnvironmentError::Stopped); + } + if self.health.quarantined.load(Ordering::Acquire) > 0 { + return Err(DockerError::Quarantined.into()); + } + let reservation = self.budget.reserve(plan.memory_bytes())?; + let code_claim = plan.claim_code()?; + let (calls, mailbox) = mpsc::channel(4); + let signals = Signals::new(); + let (done, settlement) = watch::channel(None); + let handle = EnvironmentHandle { + id: plan.environment_id(), + calls, + signals: signals.clone(), + settlement, + }; + self.sender + .try_send(Envelope { + code_claim, + plan, + mailbox, + signals, + reservation, + done, + deadline: deadline.min(Instant::now() + self.config.0.start), + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => EnvironmentError::Stopped, + })?; + Ok(handle) + } + /// Read bounded resource accounting, including quarantine reservations. + /// + /// ```no_run + /// # fn status(port: &rustack_lambda_core::docker::EnvironmentPort) { let capacity = port.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> EnvironmentCapacity { + EnvironmentCapacity { + allocated: self.budget.count(), + memory_bytes: self.budget.memory.load(Ordering::Acquire), + quarantined: self.health.quarantined.load(Ordering::Acquire), + accepting: !*self.control.borrow() + && !self.health.failed.load(Ordering::Acquire) + && self.health.quarantined.load(Ordering::Acquire) == 0, + worker_failed: self.health.failed.load(Ordering::Acquire), + } + } +} + +/// A pool keeps this receipt above workers which own exclusive handles. +#[derive(Debug, Clone)] +pub(super) struct Retirement { + stop: watch::Sender, + status: watch::Receiver, + settlement: watch::Receiver>>, +} +impl Retirement { + pub(super) fn cancel(&self) { + self.stop.send_replace(true); + } + pub(super) async fn settle(mut self) -> Result<(), EnvironmentError> { + self.cancel(); + self.settlement + .wait_for(Option::is_some) + .await + .map_err(|_| EnvironmentError::Worker)? + .clone() + .ok_or(EnvironmentError::Worker)? + } + pub(super) fn removed(&self) -> bool { + self.status.borrow().phase == EnvironmentPhase::Removed + } +} + +/// Retained root owner; startup reconciliation/probes and executor pooling are separate. +#[non_exhaustive] +pub struct EnvironmentManager { + port: EnvironmentPort, + task: Option>>, + primary: Option>, + settled: Option>, + emergency: Arc, +} +impl fmt::Debug for EnvironmentManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentManager") + .field("port", &self.port) + .field("settled", &self.settled) + .finish_non_exhaustive() + } +} +impl EnvironmentManager { + #[cfg(test)] + pub(super) fn abort_test_worker(&self) { + if let Some(task) = &self.task { + task.abort(); + } + } + + pub(super) fn physically_settled(&self) -> bool { + self.primary.is_some() + && self.task.is_none() + && self.emergency.is_finished() + && self.port.status().allocated == 0 + } + + /// Start ownership on the current runtime under a real directory capability. + /// + /// # Errors + /// Rejects a port not bound to this exact directory, or returns Worker when + /// a Tokio runtime is unavailable. This alone does not establish historical + /// reconciliation or production Docker readiness. + /// + /// ```no_run + /// # fn start(docker: rustack_lambda_core::docker::DockerPort, lease: rustack_lambda_core::docker::DirectoryLease) { let manager = rustack_lambda_core::docker::EnvironmentManager::start(docker,lease,Default::default()); } + /// ``` + pub fn start( + docker: DockerPort, + directory: DirectoryLease, + config: EnvironmentConfig, + ) -> Result { + docker.require_directory(&directory)?; + let runtime = Handle::try_current().map_err(|_| EnvironmentError::Worker)?; + let (sender, inbox) = mpsc::channel(32); + let (control, _) = watch::channel(false); + let health = Arc::new(Health::new(control.clone())); + let budget = Arc::new(Budget::new(config)); + let emergency = Arc::new(CleanupTask::default()); + let port = EnvironmentPort { + sender, + directory, + config, + control: control.clone(), + budget, + health: health.clone(), + }; + let actor = Actor::builder() + .docker(docker) + .config(config) + .inbox(inbox) + .control(control) + .health(health) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + Ok(Self { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + }) + } + /// Clone the allocation port without transferring root shutdown ownership. + /// + /// ```no_run + /// # fn port(manager: &rustack_lambda_core::docker::EnvironmentManager) { let port = manager.port(); } + /// ``` + #[must_use] + pub fn port(&self) -> EnvironmentPort { + self.port.clone() + } + /// Stop admission and join normal/emergency cleanup without dropping records. + /// + /// # Errors + /// Returns timeout while cleanup stays owned, or cached worker/root failure. + /// Must finish before image owner or `DockerWire` shutdown. + /// + /// ```no_run + /// # async fn stop(manager: &mut rustack_lambda_core::docker::EnvironmentManager) { let result = manager.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.port.control.send_replace(true); + check_deadline(deadline)?; + if let Some(settled) = &self.settled { + return settled.clone(); + } + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker) + .and_then(|result| result), + None => Err(EnvironmentError::Worker), + }; + self.task.take(); + self.primary = Some(result); + } + timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + let result = self.primary.clone().ok_or(EnvironmentError::Worker)?; + self.settled = Some(result.clone()); + result + } +} +impl Drop for EnvironmentManager { + fn drop(&mut self) { + self.port.control.send_replace(true); + } +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +pub(in crate::docker) mod test_peer; diff --git a/crates/rustack-lambda-core/src/docker/environment/test_peer.rs b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs new file mode 100644 index 00000000..b88a9254 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs @@ -0,0 +1,89 @@ +//! Test-only peer for exercising public handle ownership without external processes. +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use rustack_lambda_runtime::session::RuntimeResult; + +use super::*; + +pub(in crate::docker) struct Peer { + calls: mpsc::Receiver, + signals: Signals, + done: watch::Sender>>, +} +pub(in crate::docker) fn pair() -> (EnvironmentHandle, Peer) { + let (calls, mailbox) = mpsc::channel(4); + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (done, settlement) = watch::channel(None); + ( + EnvironmentHandle { + id: EnvironmentId::default(), + calls, + signals: signals.clone(), + settlement, + }, + Peer { + calls: mailbox, + signals, + done, + }, + ) +} +impl Peer { + pub(in crate::docker) async fn serve_pending( + mut self, + entered: oneshot::Sender, + release: oneshot::Receiver<()>, + stopping: oneshot::Sender<()>, + ) { + let call = self.calls.recv().await.unwrap(); + entered.send(call.invocation.clone()).unwrap(); + let mut stop = self.signals.stop.subscribe(); + let _ = stop.wait_for(|stop| *stop).await; + let _ = stopping.send(()); + let _ = release.await; + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Ok(()))); + drop(call); + } + pub(in crate::docker) async fn serve(self, count: Arc) { + self.serve_inner(count, None, None).await; + } + pub(in crate::docker) async fn serve_gated( + self, + count: Arc, + release: oneshot::Receiver<()>, + stopping: oneshot::Sender<()>, + ) { + self.serve_inner(count, Some(release), Some(stopping)).await; + } + async fn serve_inner( + mut self, + count: Arc, + release: Option>, + stopping: Option>, + ) { + let mut stop = self.signals.stop.subscribe(); + loop { + tokio::select! { biased; + _ = stop.wait_for(|stop| *stop) => break, + call = self.calls.recv() => { + let Some(call) = call else { break; }; + count.fetch_add(1, Ordering::AcqRel); + let _ = call.reply.send(Ok(EnvironmentResponse { result: RuntimeResult::Success(Payload::try_from(Bytes::from_static(b"{}" )).unwrap()), logs: Bytes::new() })); + } + } + } + if let Some(stopping) = stopping { + let _ = stopping.send(()); + } + if let Some(release) = release { + let _ = release.await; + } + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Ok(()))); + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/tests.rs b/crates/rustack-lambda-core/src/docker/environment/tests.rs new file mode 100644 index 00000000..d1caf6c2 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/tests.rs @@ -0,0 +1,744 @@ +use serde_json::{Value, json}; +use tokio::time::timeout; + +use super::*; +use crate::docker::{ + plan::tests::{image_with, policy, revision}, + tests::Daemon, +}; + +mod initialization; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn directory() -> DirectoryLease { + DirectoryLease::testing(uuid::Uuid::new_v4(), uuid::Uuid::new_v4()) +} +fn plan(directory: &DirectoryLease) -> EnvironmentPlan { + EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap() +} +fn inspected(plan: &EnvironmentPlan) -> Value { + let revision = plan.revision(); + let image = match &revision.artifact { + crate::artifact::DeploymentArtifact::Image(image) => { + image.resolved().metadata().image_id.as_str() + } + _ => panic!("image fixture"), + }; + json!({"Id":"a".repeat(64),"Name":format!("/{}",plan.relay_key().as_str()),"Config":{"Image":image,"Labels":{ + "rustack.lambda.ownerId":plan.directory.owner_id().to_string(),"rustack.lambda.runId":plan.directory.run_id().to_string(),"rustack.lambda.environmentId":plan.environment_id().to_string(),"rustack.lambda.functionResourceId":revision.resource_id.to_string(),"rustack.lambda.role":"relay"}},"State":{"Running":false,"OOMKilled":false,"ExitCode":0}}) +} +async fn quarantined(handle: &EnvironmentHandle) { + timeout(Duration::from_secs(3), async { + while handle.status().phase != EnvironmentPhase::Quarantined { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn finish_relay(daemon: &mut Daemon, owned: &Value) { + let observed = daemon.next().await; + assert!(observed.head.starts_with("GET ")); + observed + .reply(200, &serde_json::to_vec(owned).unwrap()) + .await; + let checked = daemon.next().await; + assert!(checked.head.contains(&"a".repeat(64))); + checked + .reply(200, &serde_json::to_vec(owned).unwrap()) + .await; + let removed = daemon.next().await; + assert!(removed.head.starts_with("DELETE ")); + removed.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; +} + +#[test] +fn test_should_validate_environment_budgets_and_reserve_count_and_memory_transactionally() { + for settings in [ + EnvironmentSettings::builder().environments(0).build(), + EnvironmentSettings::builder().environments(33).build(), + EnvironmentSettings::builder().memory_bytes(1).build(), + EnvironmentSettings::builder().io(Duration::ZERO).build(), + EnvironmentSettings::builder() + .start(Duration::from_secs(61)) + .build(), + EnvironmentSettings::builder() + .idle(Duration::from_millis(1500)) + .build(), + EnvironmentSettings::builder().log_ring_bytes(4095).build(), + EnvironmentSettings::builder().init(Duration::ZERO).build(), + EnvironmentSettings::builder() + .init(Duration::from_secs(901)) + .build(), + ] { + assert!(EnvironmentConfig::try_from(settings).is_err()); + } + let config = EnvironmentConfig::try_from( + EnvironmentSettings::builder() + .environments(2) + .memory_bytes(256 * 1024 * 1024) + .build(), + ) + .unwrap(); + let budget = Arc::new(Budget::new(config)); + let first = budget.reserve(256 * 1024 * 1024).unwrap(); + assert!(budget.reserve(1).is_err()); + assert_eq!(budget.count(), 1); + drop(first); + assert_eq!(budget.count(), 0); + assert_eq!(budget.memory.load(Ordering::Acquire), 0); +} +#[tokio::test] +async fn test_should_reject_foreign_scope_without_admitting_or_mutating() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + assert!(matches!( + manager.port().spawn(plan(&directory()), deadline()), + Err(EnvironmentError::Configuration) + )); + assert_eq!(manager.port().status().allocated, 0); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_settle_definitive_create_rejection_without_containers() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + daemon.next().await.reply(400, b"{}").await; + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_retain_quota_after_500_and_initial_absence_until_late_create_is_removed() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + daemon.next().await.reply(500, b"{}").await; + daemon.next().await.reply(404, b"{}").await; + quarantined(&handle).await; + assert_eq!(manager.port().status().allocated, 1); + assert_eq!(manager.port().status().memory_bytes, 256 * 1024 * 1024); + let capacity = manager.port().status(); + assert_eq!( + serde_json::to_value(capacity).unwrap(), + serde_json::json!({"allocated":1, + "memoryBytes":256 * 1024 * 1024,"quarantined":capacity.quarantined, + "accepting":capacity.accepting,"workerFailed":capacity.worker_failed}) + ); + assert!(matches!( + handle + .shutdown(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + finish_relay(&mut daemon, &owned).await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case("labels")] +#[case("image")] +#[case("name")] +#[case("id")] +#[tokio::test] +async fn test_should_quarantine_foreign_or_malformed_objects_without_deleting_them( + #[case] conflict: &str, +) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut foreign = owned.clone(); + match conflict { + "labels" => { + foreign["Config"]["Labels"]["rustack.lambda.ownerId"] = + json!(uuid::Uuid::new_v4().to_string()); + } + "image" => foreign["Config"]["Image"] = json!("sha256:foreign"), + "name" => foreign["Name"] = json!("/foreign"), + "id" => foreign["Id"] = json!("not-an-id"), + _ => panic!("case"), + } + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + daemon.next().await.reply(500, b"{}").await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&foreign).unwrap()) + .await; + quarantined(&handle).await; + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!(manager.port().status().allocated, 1); + finish_relay(&mut daemon, &owned).await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_finish_critical_create_after_caller_cancel_before_cleanup() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + let create = daemon.next().await; + handle.cancel(); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!(manager.port().status().allocated, 1); + create + .reply( + 201, + format!("{{\"Id\":\"{}\",\"Warnings\":[]}}", "a".repeat(64)).as_bytes(), + ) + .await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + daemon.next().await.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_recover_retained_record_after_root_abort_with_create_in_flight() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + let create = daemon.next().await; + manager.task.as_ref().unwrap().abort(); + daemon.next().await.reply(404, b"{}").await; + assert_eq!(manager.port().status().allocated, 1); + create + .reply( + 201, + format!("{{\"Id\":\"{}\",\"Warnings\":[]}}", "a".repeat(64)).as_bytes(), + ) + .await; + finish_relay(&mut daemon, &owned).await; + assert!(matches!( + handle.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + manager.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + manager.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_settle_revoked_before_issue_without_any_daemon_mutation() { + use std::sync::atomic::AtomicBool; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let (release, _) = watch::channel(()); + let (stop, stopped) = watch::channel(false); + let relay = crate::artifact::PinnedImage::new( + image_with("1000", vec![]).resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped, + ); + let planned = + EnvironmentPlan::new(lease.clone(), relay, Arc::new(revision()), policy()).unwrap(); + stop.send_replace(true); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_release_unissued_allocation_when_socket_connection_fails() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + daemon.close().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_admit_a_queued_call_after_health_io_takes_longer_than_its_cadence() { + use tokio::io::duplex; + + use crate::docker::{DockerOutputReader, FunctionLogs, plan::ContainerRole}; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = Arc::new(plan(&lease)); + let environment: EnvironmentId = planned.environment_id(); + let mut owned = inspected(&planned); + owned["Name"] = json!(format!("/{}", planned.function_key().as_str())); + owned["Config"]["Labels"]["rustack.lambda.role"] = json!("function"); + owned["State"]["Running"] = json!(true); + let (client, peer) = ready_client(environment).await; + let (_quiet, reader) = duplex(8); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let io_task = Arc::new(CleanupTask::default()); + let signals = Signals::new(); + let health = Arc::new(Health::default()); + let config = EnvironmentConfig::default(); + let progress = Arc::new(cleanup::Progress::new()); + progress + .phase(ContainerRole::Function) + .send_replace(cleanup::Phase::Known("a".repeat(64).parse().unwrap())); + let cleanup = Arc::new(tokio::sync::Semaphore::new(2)); + let (global, _) = watch::channel(false); + let cleaner = cleanup::Cleaner::builder() + .docker(wire.port()) + .config(config) + .plan(planned.clone()) + .progress(progress.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .signals(signals.clone()) + .build(); + let worker = worker::Worker::builder() + .docker(wire.port()) + .config(config) + .plan(planned) + .progress(progress) + .cleanup(cleanup) + .health(health) + .signals(signals.clone()) + .global(global) + .io(io_task.clone()) + .runtime(Handle::current()) + .build(); + let mut io = io::IoGuard::new(Handle::current(), io_task.clone(), config, signals); + io.test_streams(client, logs); + let (calls, mut mailbox) = mpsc::channel(4); + let task = tokio::spawn(async move { + if let Some((reply, result)) = worker.serve_test(&mut io, &cleaner, &mut mailbox).await { + let _ = reply.send(result); + } + io.begin_cleanup(); + }); + let inspection = daemon.next().await; + let (stop, cancellation) = watch::channel(false); + let (reply, response) = oneshot::channel(); + calls + .send(Call { + invocation: foreign_invocation(), + capture_logs: false, + cancellation, + reply, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1100)).await; + inspection + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + assert!(matches!( + timeout(Duration::from_millis(500), response) + .await + .unwrap() + .unwrap(), + Err(EnvironmentError::Configuration) + )); + task.await.unwrap(); + io_task.wait().await.unwrap(); + peer.await.unwrap(); + drop(stop); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::normal(false, false)] +#[case::root_abort(true, false)] +#[case::wire_failure(true, true)] +#[tokio::test] +async fn test_should_keep_queued_handle_failure_distinct_from_clean_shutdown( + #[case] failed: bool, + #[case] wire_failure: bool, +) { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + if wire_failure { + wire.port().raw.stop.send_replace(true); + } else if failed { + manager.task.as_ref().unwrap().abort(); + } else { + manager.port.control.send_replace(true); + } + let result = handle.shutdown(deadline()).await; + let joined = manager.shutdown(deadline()).await; + if failed { + assert!(matches!(result, Err(EnvironmentError::Worker))); + assert!(matches!(joined, Err(EnvironmentError::Worker))); + } else { + result.unwrap(); + joined.unwrap(); + } + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_allow_final_log_tail_at_eof_but_never_reuse_the_closed_environment() { + use rustack_lambda_runtime::{ + client::{ClientConfig, ControlClient}, + protocol::{Architecture, EnvironmentId}, + }; + + use crate::docker::{DockerOutputReader, FunctionLogs}; + let (reader, peer) = tokio::io::duplex(8); + drop(peer); + let client = ControlClient::start( + ClientConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(1), + ) + .unwrap(), + reader, + tokio::io::sink(), + ) + .unwrap(); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(b"".as_slice(), Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let cleanup = Arc::new(CleanupTask::default()); + let mut io = io::IoGuard::new( + Handle::current(), + cleanup.clone(), + EnvironmentConfig::default(), + Signals::new(), + ); + io.test_streams(client, logs); + timeout(Duration::from_secs(1), async { + loop { + assert!(io.tail().await.unwrap().is_empty()); + if !io.reusable() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(io.healthy().await.is_err()); + io.begin_cleanup(); + cleanup.wait().await.unwrap(); +} + +async fn ready_client( + environment: EnvironmentId, +) -> ( + rustack_lambda_runtime::client::ControlClient, + tokio::task::JoinHandle<()>, +) { + use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + protocol::{Acknowledgement, Architecture, Frame, Message, read_frame, write_frame}, + }; + use tokio::io::{duplex, split}; + let (stream, mut peer) = duplex(65536); + let (reader, writer) = split(stream); + let peer = tokio::spawn(async move { + for message in [ + Message::Hello { + architecture: Architecture::Arm64, + }, + Message::Ready, + Message::BootstrapReady, + ] { + write_frame(&mut peer, &Frame::new(environment, message), deadline()) + .await + .unwrap(); + } + let close = read_frame(&mut peer, deadline()).await.unwrap().unwrap(); + assert!(matches!(close.message(), Message::Close)); + write_frame( + &mut peer, + &Frame::new(environment, Message::Ack(Acknowledgement::Close)), + deadline(), + ) + .await + .unwrap(); + }); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + writer, + ) + .unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + (client, peer) +} + +fn foreign_invocation() -> rustack_lambda_runtime::session::Invocation { + use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, Payload}, + session::Invocation, + }; + Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:other" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn test_should_bound_long_caller_deadline_by_start_budget_before_admission_validation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager + .port() + .spawn(plan(&lease), Instant::now() + Duration::from_mins(15)) + .unwrap(); + handle.cancel(); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::abandoned(false)] +#[case::consumed(true)] +#[tokio::test] +async fn test_should_retire_ready_reply_only_when_its_waiter_is_abandoned(#[case] consumed: bool) { + use std::{ + future::Future, + task::{Context, Waker}, + }; + + use rustack_lambda_runtime::protocol::Payload; + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (calls, mut mailbox) = mpsc::channel(4); + let (_done, settlement) = watch::channel(None); + let mut handle = EnvironmentHandle { + id: EnvironmentId::default(), + calls, + signals: signals.clone(), + settlement, + }; + { + let invoke = handle.invoke(foreign_invocation(), false); + tokio::pin!(invoke); + assert!( + invoke + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + let call = mailbox.recv().await.unwrap(); + call.reply + .send(Ok(EnvironmentResponse { + result: rustack_lambda_runtime::session::RuntimeResult::Success( + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ), + logs: Bytes::new(), + })) + .unwrap(); + if consumed { + assert!(matches!( + invoke + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + std::task::Poll::Ready(Ok(_)) + )); + } + } + assert_eq!( + *signals.stop.borrow(), + !consumed, + "only abandoned replies retire the environment" + ); +} + +#[tokio::test] +async fn test_should_publish_intent_before_create_and_retain_reservation_until_durable_erase() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut manager = EnvironmentManager::start( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + directory.clone(), + EnvironmentConfig::default(), + ) + .unwrap(); + let mut handle = manager.port().spawn(plan(&directory), deadline()).unwrap(); + let create = daemon.next().await; + assert!(create.head.starts_with("POST ")); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 1 + ); + let staging = root + .path() + .join(".rustack-lambda-docker/containers.pending"); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, &staging).unwrap(); + create.reply(400, b"{}").await; + quarantined(&handle).await; + assert_eq!(manager.port().status().allocated, 1); + tokio::fs::remove_file(staging).await.unwrap(); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 1 + ); + handle.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + assert_eq!(tokio::fs::read(outside).await.unwrap(), b"keep"); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_distinguish_impossible_memory_from_temporary_contention_without_spending_capacity() { + let config = EnvironmentSettings::builder() + .environments(1) + .memory_bytes(256 * 1024 * 1024) + .build() + .try_into() + .unwrap(); + let budget = Arc::new(Budget::new(config)); + assert!(matches!( + budget.reserve(256 * 1024 * 1024 + 1), + Err(EnvironmentError::MemoryLimit) + )); + assert_eq!(budget.count(), 0); + assert_eq!(budget.memory.load(Ordering::Acquire), 0); + let exact = budget.reserve(256 * 1024 * 1024).unwrap(); + assert!(matches!(budget.reserve(1), Err(EnvironmentError::Busy))); + assert!(matches!( + budget.reserve(256 * 1024 * 1024 + 1), + Err(EnvironmentError::MemoryLimit) + )); + assert_eq!(budget.count(), 1); + drop(exact); + assert_eq!(budget.count(), 0); + assert_eq!(budget.memory.load(Ordering::Acquire), 0); +} diff --git a/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs b/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs new file mode 100644 index 00000000..7a4a3d69 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs @@ -0,0 +1,290 @@ +//! Post-bootstrap initialization failures through the actual runtime stack. + +use std::{ + future::Future, + net::SocketAddr, + task::{Context, Wake, Waker}, +}; + +use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + http, + metadata::local_trace_id, + protocol::{Architecture, AttemptId}, + relay::{self, RelayConfig}, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, duplex, split}, + net::TcpStream, + sync::{Notify, Semaphore}, +}; + +use super::*; +use crate::docker::{DockerOutputReader, FunctionLogs, plan::ContainerRole}; + +struct Offered(Notify); +impl Wake for Offered { + fn wake(self: Arc) { + self.0.notify_one(); + } + fn wake_by_ref(self: &Arc) { + self.0.notify_one(); + } +} + +#[rstest::rstest] +#[case::idle_bounded(false, 32768)] +#[case::idle_oversized(false, 32769)] +#[case::busy_bounded(true, 32768)] +#[case::busy_oversized(true, 32769)] +#[tokio::test] +async fn test_should_preserve_init_error_after_abandoned_bootstrap_poll_and_join_retirement( + #[case] busy: bool, + #[case] bytes: usize, +) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let planned = Arc::new(plan(&directory())); + let RuntimeFixture { + address, + session, + mut relay, + client, + } = runtime(planned.environment_id()).await; + + let offered = Arc::new(Offered(Notify::new())); + let waker = Waker::from(offered.clone()); + let mut observer = Box::pin(session.next(deadline())); + if busy { + assert!( + observer + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + } + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (mut task, io_task, calls, mut quiet) = + start_worker(wire.port(), planned.clone(), client, signals.clone()); + let inspected = function_inspection(&planned); + let daemon_signals = signals.clone(); + let (retiring, observed) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let container_cleanup = tokio::spawn(async move { + loop { + let request = daemon.next().await; + assert!(request.head.starts_with("GET ")); + assert!(request.head.contains(&"a".repeat(64))); + if daemon_signals.status.borrow().phase == EnvironmentPhase::Retiring { + retiring.send(()).unwrap(); + timeout_at(deadline(), released).await.unwrap().unwrap(); + request.reply(404, b"{}").await; + break; + } + request.reply(200, &inspected).await; + } + daemon.close().await; + }); + let (stop, cancellation) = watch::channel(false); + let (reply, mut response) = oneshot::channel(); + if busy { + calls + .send(Call { + invocation: Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ) + .unwrap(), + capture_logs: false, + cancellation, + reply, + }) + .await + .unwrap(); + timeout_at(deadline(), offered.0.notified()).await.unwrap(); + assert_eq!(signals.status.borrow().phase, EnvironmentPhase::Busy); + } else { + drop(reply); + } + // A wake proves an offer exists. Never repoll: that would acknowledge delivery. + drop(observer); + let marker = b"private initialization cause"; + let mut payload = vec![b'x'; bytes]; + payload + .get_mut(..marker.len()) + .unwrap() + .copy_from_slice(marker); + post_init_error(address, &payload).await; + timeout_at(deadline(), observed).await.unwrap().unwrap(); + assert!(!task.is_finished()); + if busy { + assert!(matches!( + response.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + } + release.send(()).unwrap(); + timeout_at(deadline(), &mut task).await.unwrap().unwrap(); + io_task.wait().await.unwrap(); + timeout_at(deadline(), relay.wait()).await.unwrap().unwrap(); + container_cleanup.await.unwrap(); + let error = signals.status.borrow().error.clone().unwrap(); + assert_class(&error, &payload); + if busy { + assert_class(&response.await.unwrap().unwrap_err(), &payload); + } + assert!(!format!("{error:?} {error}").contains(std::str::from_utf8(marker).unwrap())); + assert_eq!(quiet.read(&mut [0; 1]).await.unwrap(), 0); + assert!(TcpStream::connect(address).await.is_err()); + drop(stop); + drop(calls); + wire.shutdown().await.unwrap(); +} + +fn start_worker( + docker: DockerPort, + planned: Arc, + client: ControlClient, + signals: Signals, +) -> ( + tokio::task::JoinHandle<()>, + Arc, + mpsc::Sender, + tokio::io::DuplexStream, +) { + let (quiet, reader) = duplex(8); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let config = EnvironmentConfig::default(); + let io_task = Arc::new(CleanupTask::default()); + let progress = Arc::new(cleanup::Progress::new()); + progress + .phase(ContainerRole::Function) + .send_replace(cleanup::Phase::Known("a".repeat(64).parse().unwrap())); + let worker = worker::Worker::builder() + .docker(docker) + .config(config) + .plan(planned) + .progress(progress) + .cleanup(Arc::new(Semaphore::new(2))) + .health(Arc::new(Health::default())) + .signals(signals.clone()) + .global(watch::channel(false).0) + .io(io_task.clone()) + .runtime(Handle::current()) + .build(); + let mut io = io::IoGuard::new(Handle::current(), io_task.clone(), config, signals); + io.test_streams(client, logs); + let (calls, mut mailbox) = mpsc::channel(4); + let task = tokio::spawn(async move { + worker.serve_and_retire_test(&mut io, &mut mailbox).await; + }); + (task, io_task, calls, quiet) +} + +fn function_inspection(plan: &EnvironmentPlan) -> Vec { + let mut owned = inspected(plan); + owned["Name"] = json!(format!("/{}", plan.function_key().as_str())); + owned["Config"]["Labels"]["rustack.lambda.role"] = json!("function"); + owned["State"]["Running"] = json!(true); + serde_json::to_vec(&owned).unwrap() +} + +async fn post_init_error(address: SocketAddr, payload: &[u8]) { + timeout_at(deadline(), async { + let mut stream = TcpStream::connect(address).await.unwrap(); + let head = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n", + payload.len() + ); + stream.write_all(head.as_bytes()).await.unwrap(); + stream.write_all(payload).await.unwrap(); + let mut response = Vec::new(); + stream.take(16384).read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 202 "), "{response:?}"); + }) + .await + .unwrap(); +} + +fn assert_class(error: &EnvironmentError, payload: &[u8]) { + if payload.len() <= 32768 { + let EnvironmentError::Initialization(received) = error else { + panic!("{error:?}"); + }; + assert_eq!(received.as_ref(), payload); + } else { + assert!( + matches!(error, EnvironmentError::Docker(DockerError::Protocol)), + "{error:?}" + ); + } +} + +struct RuntimeFixture { + address: SocketAddr, + session: rustack_lambda_runtime::session::SessionHandle, + relay: relay::RelayHandle, + client: ControlClient, +} +async fn runtime(environment: EnvironmentId) -> RuntimeFixture { + let api = http::start().await.unwrap(); + let address = api.addr(); + let session = api.session().clone(); + let (host, peer) = duplex(65536); + let (reader, writer) = split(peer); + let relay = relay::start( + api, + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(10), + ) + .unwrap(), + reader, + writer, + ) + .unwrap(); + let (reader, writer) = split(host); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + writer, + ) + .unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + let mut first_poll = Box::pin(session.next(deadline())); + assert!( + first_poll + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + drop(first_poll); + + RuntimeFixture { + address, + session, + relay, + client, + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs new file mode 100644 index 00000000..fba8b9fa --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -0,0 +1,535 @@ +//! One ordered environment worker; all resources remain registered above its task. + +use std::{future::pending, sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::StatusCode; +use rustack_lambda_runtime::{ + client::ClientEvent, + protocol::{Architecture as RuntimeArchitecture, EnvironmentId, HandlerTimeout}, + session::RuntimeResult, +}; +use tokio::{ + runtime::Handle, + sync::{Semaphore, mpsc, oneshot, watch}, + time::{Instant, interval, sleep_until}, +}; +use typed_builder::TypedBuilder; + +use super::{ + Call, EnvironmentConfig, EnvironmentError, EnvironmentPhase, EnvironmentResponse, + actor::{Health, Signals}, + cleanup::{Cleaner, Phase, Progress}, + io::IoGuard, +}; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + docker::{ + DockerError, DockerKey, DockerPort, EnvironmentPlan, + connection::stopped as wait_for_stop, + mutation::MutationGuard, + ownership::ContainerRecord, + plan::ContainerRole, + zip::{CodeStageError, StageInput}, + }, + invocation::CleanupTask, +}; + +#[derive(TypedBuilder)] +pub(super) struct Worker { + docker: DockerPort, + config: EnvironmentConfig, + plan: Arc, + progress: Arc, + cleanup: Arc, + health: Arc, + signals: Signals, + global: watch::Sender, + io: Arc, + runtime: Handle, +} +pub(super) type FinalReply = ( + oneshot::Sender>, + Result, +); +impl Worker { + #[cfg(test)] + pub(super) async fn serve_and_retire_test( + &self, + io: &mut IoGuard, + mailbox: &mut mpsc::Receiver, + ) { + let cleaner = self.cleaner(); + let pending = self.serve(io, &cleaner, mailbox).await; + self.finish(io, &cleaner, mailbox, pending).await; + } + #[cfg(test)] + pub(super) async fn serve_test( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + ) -> Option { + self.serve(io, cleaner, mailbox).await + } + + pub(super) async fn run( + self, + mut mailbox: mpsc::Receiver, + deadline: Instant, + ) -> Result<(), EnvironmentError> { + let mut io = IoGuard::new( + self.runtime.clone(), + self.io.clone(), + self.config, + self.signals.clone(), + ); + let cleaner = self.cleaner(); + let pending = match self.initialize(&mut io, deadline).await { + Ok(()) => { + self.signals.phase(EnvironmentPhase::Idle); + self.serve(&mut io, &cleaner, &mut mailbox).await + } + Err(error) => { + self.signals.error(error); + None + } + }; + self.finish(&mut io, &cleaner, &mut mailbox, pending).await; + Ok(()) + } + async fn finish( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + pending: Option, + ) { + self.signals.phase(EnvironmentPhase::Retiring); + mailbox.close(); + while let Ok(call) = mailbox.try_recv() { + let _ = call.reply.send(Err(EnvironmentError::Stopped)); + } + cleaner.retire(&self.io, Some(io)).await; + if let Some((reply, result)) = pending { + let _ = reply.send(result); + } + } + fn cleaner(&self) -> Cleaner { + Cleaner::builder() + .docker(self.docker.clone()) + .config(self.config) + .plan(self.plan.clone()) + .progress(self.progress.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .signals(self.signals.clone()) + .build() + } + fn check(&self, deadline: Instant) -> Result<(), EnvironmentError> { + if *self.global.borrow() || *self.signals.stop.borrow() { + Err(EnvironmentError::Cancelled) + } else if Instant::now() >= deadline { + Err(EnvironmentError::Deadline) + } else { + Ok(()) + } + } + async fn create( + &self, + role: ContainerRole, + deadline: Instant, + ) -> Result { + self.check(deadline)?; + let mutation = MutationGuard::new(); + self.progress + .phase(role) + .send_replace(Phase::Uncertain(mutation.receipt())); + let record = ContainerRecord::from_plan(&self.plan, role)?; + self.plan + .directory + .prepare_container(record.clone(), self.config.deadline()) + .await?; + let created = self + .plan + .create_tracked(role, &self.docker, self.config.deadline(), mutation) + .await; + if matches!( + &created, + Err(DockerError::Busy + | DockerError::Status(StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND)) + ) { + self.progress.phase(role).send_replace(Phase::Clean); + } + let created = created?; + let inspected = self + .docker + .inspect_container(self.plan.key(role), self.config.deadline()) + .await?; + let id = self.plan.owned(role, &inspected)?; + if id.as_str() != created.id { + return Err(DockerError::Protocol.into()); + } + self.plan + .directory + .know_container(record, id.clone(), self.config.deadline()) + .await?; + self.progress + .phase(role) + .send_replace(Phase::Known(id.clone())); + self.check(deadline)?; + Ok(id) + } + async fn initialize( + &self, + io: &mut IoGuard, + deadline: Instant, + ) -> Result<(), EnvironmentError> { + let initialization_started = Instant::now(); + if let Some(zip) = &self.plan.zip { + let DeploymentArtifact::Zip(artifact) = &self.plan.revision().artifact else { + return Err(EnvironmentError::Configuration); + }; + zip.start( + StageInput::builder() + .artifact(artifact.clone()) + .deadline(deadline) + .global(self.global.subscribe()) + .stop(self.signals.stop.subscribe()) + .build(), + &self.runtime, + )?; + if let Err(error) = zip.wait_ready(deadline).await { + return Err(match error { + CodeStageError::AdmissionBusy => EnvironmentError::Busy, + CodeStageError::Docker(DockerError::Worker) => { + self.health.fail(); + EnvironmentError::Worker + } + CodeStageError::Docker(other) => other.into(), + }); + } + self.check(deadline)?; + } + let relay = self.create(ContainerRole::Relay, deadline).await?; + self.signals.phase(EnvironmentPhase::Initializing); + let attached = self.docker.attach(&relay, self.config.deadline()).await?; + let architecture = match self.plan.revision().architecture { + Architecture::Arm64 => RuntimeArchitecture::Arm64, + Architecture::X86_64 => RuntimeArchitecture::X86_64, + }; + io.control(attached, self.plan.environment_id(), architecture)?; + self.check(deadline)?; + self.docker + .start_container(&relay, self.config.deadline()) + .await?; + self.check(deadline)?; + match self.next(io, deadline, None, None).await? { + ClientEvent::RelayReady => {} + _ => return Err(DockerError::Protocol.into()), + } + let relay_ready = Instant::now(); + let function = self.create(ContainerRole::Function, deadline).await?; + io.logs( + self.docker + .attach_logs(&function, self.config.deadline()) + .await?, + )?; + self.check(deadline)?; + self.docker + .start_container(&function, self.config.deadline()) + .await?; + let bootstrap_deadline = deadline.min(Instant::now() + self.config.0.init); + let ready = self + .next(io, bootstrap_deadline, Some(&self.cleaner()), None) + .await + .map_err(|error| { + if matches!(error, EnvironmentError::Deadline) && bootstrap_deadline < deadline { + EnvironmentError::InitializationTimeout + } else { + error + } + })?; + finish_initialization( + ready, + self.plan.zip.is_none().then(|| self.plan.environment_id()), + initialization_started, + relay_ready, + ) + } + async fn next( + &self, + io: &mut IoGuard, + deadline: Instant, + health: Option<&Cleaner>, + mut cancellation: Option<&mut watch::Receiver>, + ) -> Result { + let mut global = self.global.subscribe(); + let mut stopped = self.signals.stop.subscribe(); + let mut tick = interval(Duration::from_secs(1)); + tick.tick().await; + loop { + self.check(deadline)?; + if cancellation + .as_ref() + .is_some_and(|cancel| *cancel.borrow() || cancel.has_changed().is_err()) + { + return Err(EnvironmentError::Cancelled); + } + tokio::select! {biased; + ()=wait_for_stop(&mut global)=>return Err(EnvironmentError::Cancelled), + ()=wait_for_stop(&mut stopped)=>return Err(EnvironmentError::Cancelled), + ()=sleep_until(deadline)=>return Err(EnvironmentError::Deadline), + ()=async{match &mut cancellation{Some(cancel)=>{let _=cancel.wait_for(|stop|*stop).await;},None=>pending().await,}}=>return Err(EnvironmentError::Cancelled), + result=io.client()?.next(deadline)=>return result.map_err(EnvironmentError::from), + _=tick.tick(),if health.is_some()=>{if let Some(health)=health{health.inspect_health().await?;io.healthy().await?;tick.reset();}}, + } + } + } + async fn serve( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + ) -> Option { + let mut global = self.global.subscribe(); + let mut stopped = self.signals.stop.subscribe(); + let mut tick = interval(Duration::from_secs(1)); + tick.tick().await; + let mut idle = Instant::now() + self.config.0.idle; + loop { + let action = tokio::select! {biased; + ()=wait_for_stop(&mut global)=>return None, + ()=wait_for_stop(&mut stopped)=>return None, + ()=sleep_until(idle)=>return None, + _=tick.tick()=>{let result=async{cleaner.inspect_health().await?;io.healthy().await}.await;tick.reset();if let Err(error)=result{self.signals.error(error);return None;}continue;}, + result=io.client().ok()?.next(idle)=>{self.signals.error(match result {Ok(event)=>Self::event_failure(event),Err(error)=>error.into()});return None;}, + call=mailbox.recv()=>call, + }; + let mut call = action?; + self.signals.phase(EnvironmentPhase::Busy); + let result = self.call(io, cleaner, &mut call).await; + let reusable = matches!( + &result, + Ok(EnvironmentResponse { + result: RuntimeResult::Success(_), + .. + }) + ) && !call.reply.is_closed() + && io.reusable(); + if reusable { + self.signals.phase(EnvironmentPhase::Idle); + let _ = call.reply.send(result); + idle = Instant::now() + self.config.0.idle; + } else { + if let Err(error) = &result { + self.signals.error(error.clone()); + } + return Some((call.reply, result)); + } + } + } + async fn call( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + call: &mut Call, + ) -> Result { + let (attempt, deadline, arn, _, _) = call.invocation.parts(); + let name = arn + .as_ref() + .split_once(":function:") + .and_then(|(_, name)| name.split(':').next()); + if name != Some(self.plan.revision().function_name.as_str()) { + return Err(EnvironmentError::Configuration); + } + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } + let log_window = io.begin_logs(attempt).await?; + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } + io.client()? + .invoke( + call.invocation.clone().with_handler_timeout( + HandlerTimeout::try_from(self.plan.revision().timeout) + .map_err(|_| EnvironmentError::Configuration)?, + ), + deadline.min(self.config.deadline()), + ) + .await?; + let result = self + .next(io, deadline, Some(cleaner), Some(&mut call.cancellation)) + .await; + match result { + Ok(ClientEvent::InvocationTimedOut(returned)) if returned == attempt => { + Err(EnvironmentError::Timeout(self.plan.revision().timeout)) + } + Ok(ClientEvent::InvocationResult { + attempt: returned, + result, + }) if returned == attempt => { + let tail = io.finish_logs(log_window).await?; + let logs = if call.capture_logs { + tail + } else { + Bytes::new() + }; + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } + Ok(EnvironmentResponse { result, logs }) + } + Ok(event) => Err(Self::event_failure(event)), + Err(error) => { + if io + .client()? + .cancel(attempt, self.config.deadline()) + .await + .is_ok() + { + let _ = io.client()?.next(self.config.deadline()).await; + } + Err(error) + } + } + } + fn event_failure(event: ClientEvent) -> EnvironmentError { + match event { + ClientEvent::InitError(payload) if payload.as_ref().len() <= 32768 => { + EnvironmentError::Initialization(payload) + } + _ => DockerError::Protocol.into(), + } + } +} + +fn micros(duration: Duration) -> u64 { + u64::try_from(duration.as_nanos().saturating_add(999) / 1000).unwrap_or(u64::MAX) +} + +fn finish_initialization( + event: ClientEvent, + image: Option, + started: Instant, + relay: Instant, +) -> Result<(), EnvironmentError> { + match event { + ClientEvent::BootstrapReady => { + if let Some(environment) = image { + tracing::info!( + target: "rustack_lambda_core::docker::initialization", + environment_id = %environment, + relay_micros = micros(relay.duration_since(started)), + ric_micros = micros(relay.elapsed()), + "Image environment ready" + ); + } + Ok(()) + } + event => Err(Worker::event_failure(event)), + } +} + +#[cfg(test)] +mod timing_tests { + use std::{ + error::Error, + io::{self, Write}, + mem, + sync::mpsc::{SyncSender, sync_channel}, + time::Duration, + }; + + use bytes::Bytes; + use rustack_lambda_runtime::{ + client::ClientEvent, + protocol::{EnvironmentId, Payload}, + }; + use tokio::time::{Instant, advance}; + + use super::{finish_initialization, micros}; + + #[derive(Debug)] + struct Line { + bytes: Vec, + sender: SyncSender>, + } + impl Write for Line { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.bytes.len().saturating_add(bytes.len()) > 1024 { + return Err(io::Error::other("bounded initialization diagnostic")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + impl Drop for Line { + fn drop(&mut self) { + let _ = self.sender.try_send(mem::take(&mut self.bytes)); + } + } + + #[test] + fn test_should_round_initialization_durations_up_without_overflow() { + assert_eq!(micros(Duration::ZERO), 0); + assert_eq!(micros(Duration::from_nanos(1)), 1); + assert_eq!(micros(Duration::from_nanos(1000)), 1); + assert_eq!(micros(Duration::from_nanos(1001)), 2); + assert_eq!(micros(Duration::MAX), u64::MAX); + } + + #[tokio::test(start_paused = true)] + async fn test_should_report_only_successful_image_initialization_without_payloads() + -> Result<(), Box> { + let (sender, received) = sync_channel(4); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(move || Line { + bytes: Vec::new(), + sender: sender.clone(), + }) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let started = Instant::now(); + advance(Duration::from_millis(1)).await; + let relay = Instant::now(); + advance(Duration::from_millis(2)).await; + let environment = EnvironmentId::default(); + finish_initialization( + ClientEvent::BootstrapReady, + Some(environment), + started, + relay, + )?; + let line = String::from_utf8(received.try_recv()?)?; + assert!(line.contains("relay_micros=1000") && line.contains("ric_micros=2000")); + assert!(line.contains(&environment.to_string())); + let secret = Payload::try_from(Bytes::from_static(b"runtime-boundary-secret"))?; + assert!( + finish_initialization( + ClientEvent::InitError(secret), + Some(environment), + started, + relay + ) + .is_err() + ); + assert!( + finish_initialization(ClientEvent::RelayReady, Some(environment), started, relay) + .is_err() + ); + finish_initialization(ClientEvent::BootstrapReady, None, started, relay)?; + assert!(received.try_recv().is_err()); + assert!(!line.contains("runtime-boundary-secret")); + Ok(()) + } +} diff --git a/crates/rustack-lambda-core/src/docker/executor.rs b/crates/rustack-lambda-core/src/docker/executor.rs new file mode 100644 index 00000000..e555e8f3 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/executor.rs @@ -0,0 +1,460 @@ +//! Trusted runtime composition; startup qualification remains coordinator-owned. +//! +//! The synchronous readiness trait is object-safe for trusted in-process startup +//! injection. Executor uses async-trait for the existing dynamic backend boundary. + +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; +use parking_lot::Mutex; +use rustack_lambda_runtime::{ + metadata::{MetadataError, local_trace_id}, + protocol::{FunctionArn, Payload, ProtocolError}, + session::{Invocation, RuntimeResult, SessionError}, +}; +use thiserror::Error; +use tokio::{ + runtime::Handle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + ContainerPolicy, DirectoryLease, DockerError, DockerSdkEnvironment, DockerZipRuntime, + EnvironmentError, EnvironmentPlan, EnvironmentPool, EnvironmentPort, EnvironmentResponse, + PoolConfig, PoolPort, +}; +use crate::{ + artifact::{Architecture, DeploymentArtifact, PinnedImage}, + executor::{Executor, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::{CleanupTask, InvocationContext}, +}; + +/// Redacted typed failures at the Docker Executor composition boundary. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum DockerExecutorError { + /// Startup/shutdown owner preserves its first diagnostic after physical retirement. + #[error("Docker runtime owner failed")] + Owner(#[source] Arc), + /// Invalid trusted settings, runtime scope or captured invocation metadata. + #[error("invalid Docker executor configuration")] + Configuration, + /// Startup qualification was not granted or admission has been revoked. + #[error("Docker execution is not ready")] + Unavailable, + /// Invalid bounded invocation payload, ARN or attempt identity. + #[error("invalid Docker invocation metadata")] + Request(#[source] Arc), + /// Wall-clock tracing metadata could not be generated. + #[error("Docker invocation clock failed")] + Clock(#[from] MetadataError), + /// Runtime session rejected construction before pool admission. + #[error("Docker invocation session failed")] + Session(#[from] SessionError), + /// Retained environment preserves initialization, exit/OOM and wire classes. + #[error("Docker environment failed: {0}")] + Environment(#[from] EnvironmentError), +} +impl fmt::Debug for DockerExecutorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl From for DockerExecutorError { + fn from(error: ProtocolError) -> Self { + Self::Request(Arc::new(error)) + } +} + +/// Trusted startup evidence port for the exact runtime inputs injected below. +/// +/// Implementations must make bounded, nonblocking checks without IO. The startup +/// coordinator owns actual reconciliation and probe evidence; this trait alone +/// cannot establish it. Revocation rejects new calls; shutdown retires active work. +pub trait DockerReadiness: fmt::Debug + Send + Sync + 'static { + /// Check coordinator-owned qualification for this runtime. + /// + /// # Errors + /// Returns Unavailable or a typed qualification failure after revocation. + /// + /// ```no_run + /// # fn ready(evidence: &dyn rustack_lambda_core::executor::DockerReadiness) { + /// let qualified = evidence.check().is_ok(); + /// # } + /// ``` + fn check(&self) -> Result<(), DockerExecutorError>; +} + +/// Validated pool and independent public cleanup-wait budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct DockerExecutorSettings { + pool: PoolConfig, + cleanup: Duration, +} +impl Default for DockerExecutorSettings { + fn default() -> Self { + Self { + pool: PoolConfig::default(), + cleanup: Duration::from_secs(10), + } + } +} +impl DockerExecutorSettings { + /// Validate a whole-second cleanup budget in1..60s. + /// + /// # Errors + /// Rejects zero, fractional or excessive cleanup duration. + /// + /// ``` + /// let settings = rustack_lambda_core::executor::DockerExecutorSettings::new( + /// Default::default(), std::time::Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_core::executor::DockerExecutorError>(()) + /// ``` + pub fn new(pool: PoolConfig, cleanup: Duration) -> Result { + if cleanup.subsec_nanos() != 0 || !(1..=60).contains(&cleanup.as_secs()) { + return Err(DockerExecutorError::Configuration); + } + Ok(Self { pool, cleanup }) + } +} + +/// Trusted startup inputs; readiness must qualify this exact scope/relay/policy. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct DockerExecutorConfig { + /// Persistent namespace capability retained through cleanup. + pub directory: DirectoryLease, + /// Immutable live startup relay image. + pub relay: PinnedImage, + /// Validated operator allocation policy. + pub policy: ContainerPolicy, + /// Retained environment service for the same directory owner and run. + pub environments: EnvironmentPort, + /// Trusted startup evidence, never supplied by an AWS request. + pub readiness: Arc, + /// Validated dispatch/cleanup budgets. + #[builder(default)] + pub settings: DockerExecutorSettings, + /// Optional affiliated provided.al2023 source, qualified by production startup. + #[builder(default)] + pub zip: Option, + /// Optional opaque handler SDK context issued by the same private startup proof. + #[builder(default)] + pub sdk: Option, +} + +#[derive(Debug)] +struct PoolOwner { + root: Mutex>, + port: PoolPort, + cleanup: CleanupTask, + runtime: Handle, + stopping: AtomicBool, +} + +/// Actual image Lambda Executor over retained pool/environment ownership. +#[derive(Debug)] +#[non_exhaustive] +pub struct DockerExecutor { + config: DockerExecutorConfig, + pool: PoolOwner, +} +impl DockerExecutor { + pub(super) fn physically_settled(&self) -> bool { + self.pool.stopping.load(Ordering::Acquire) && self.pool.cleanup.is_finished() + } + + /// Construct dispatch over matching live runtime inputs without daemon IO. + /// + /// This does not qualify production startup. Retain the environment/image + /// managers until shutdown has joined this executor's pool. + /// + /// # Errors + /// Rejects scope mismatch, revoked relay or missing Tokio supervision. + /// + /// ```no_run + /// # async fn compose(config: rustack_lambda_core::executor::DockerExecutorConfig) -> Result<(), rustack_lambda_core::executor::ExecutorError> { + /// use rustack_lambda_core::executor::{DockerExecutor, Executor}; + /// let executor = DockerExecutor::new(config)?; + /// executor.shutdown().await?; + /// # Ok(()) } + /// ``` + pub fn new(config: DockerExecutorConfig) -> Result { + if !config.environments.same_scope(&config.directory) || !config.relay.is_live() { + return Err(DockerExecutorError::Configuration); + } + if let Some(zip) = &config.zip { + if !zip.same_scope(&config.directory) + || zip.base.resolved().metadata().architecture + != config.relay.resolved().metadata().architecture + { + return Err(DockerExecutorError::Configuration); + } + zip.available().map_err(EnvironmentError::from)?; + } + if config.sdk.as_ref().is_some_and(|sdk| !sdk.matches(&config)) { + return Err(DockerExecutorError::Configuration); + } + let runtime = Handle::try_current().map_err(|_| DockerExecutorError::Unavailable)?; + let root = EnvironmentPool::start(config.environments.clone(), config.settings.pool)?; + let port = root.port(); + Ok(Self::with_pool(config, root, port, runtime)) + } + fn with_pool( + config: DockerExecutorConfig, + root: EnvironmentPool, + port: PoolPort, + runtime: Handle, + ) -> Self { + Self { + config, + pool: PoolOwner { + root: Mutex::new(Some(root)), + port, + cleanup: CleanupTask::default(), + runtime, + stopping: AtomicBool::new(false), + }, + } + } + fn plan(&self, request: &InvokeRequest) -> Result { + if self + .config + .sdk + .as_ref() + .is_some_and(|sdk| !sdk.matches(&self.config)) + { + return Err(DockerExecutorError::Unavailable); + } + let plan = match &request.revision.artifact { + DeploymentArtifact::Image(_) => EnvironmentPlan::new( + self.config.directory.clone(), + self.config.relay.clone(), + request.revision.clone(), + self.config.policy.clone(), + ), + DeploymentArtifact::Zip(_) => EnvironmentPlan::new_zip( + self.config.directory.clone(), + self.config.relay.clone(), + request.revision.clone(), + self.config.policy.clone(), + self.config + .zip + .clone() + .ok_or(DockerExecutorError::Unavailable)?, + ), + } + .map_err(EnvironmentError::from)?; + let plan = match &self.config.sdk { + Some(sdk) => plan + .with_sdk_environment(sdk) + .map_err(EnvironmentError::from)?, + None => plan, + }; + let _: FunctionArn = request.function_arn.parse()?; + let (_, function) = request + .function_arn + .split_once(":function:") + .ok_or(DockerExecutorError::Configuration)?; + let (name, qualifier) = function + .split_once(':') + .map_or((function, None), |(name, q)| (name, Some(q))); + let version = request.revision.qualifier.as_str(); + let matches = match qualifier { + None | Some("$LATEST") => version == "$LATEST", + Some(value) if value.bytes().all(|b| b.is_ascii_digit()) => value == version, + Some(_) => version != "$LATEST", + }; + let region = request.function_arn.split(':').nth(3); + if name != request.revision.function_name + || !matches + || region + != request + .revision + .environment + .get("AWS_REGION") + .map(String::as_str) + { + return Err(DockerExecutorError::Configuration); + } + Payload::try_from(request.payload.clone())?; + self.config.environments.check_memory(plan.memory_bytes())?; + Ok(plan) + } + fn start_cleanup(&self) { + self.pool.stopping.store(true, Ordering::Release); + self.pool.cleanup.start_on(&self.pool.runtime, || { + let root = self.pool.root.lock().take(); + async move { + let Some(mut root) = root else { + return Err(cleanup_error()); + }; + loop { + match root + .shutdown(Instant::now() + Duration::from_secs(60)) + .await + { + Ok(()) => return Ok(()), + Err( + EnvironmentError::Deadline + | EnvironmentError::Docker(DockerError::Deadline), + ) => {} + Err(_) => return Err(cleanup_error()), + } + } + } + }); + } +} +impl Drop for DockerExecutor { + fn drop(&mut self) { + self.start_cleanup(); + } +} + +#[async_trait] +impl Executor for DockerExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + self.config.readiness.check()?; + let environment = self.config.environments.status(); + let pool = self.pool.port.status(); + if environment.worker_failed || pool.worker_failed { + return Err(DockerExecutorError::Environment(EnvironmentError::Worker).into()); + } + if self.pool.stopping.load(Ordering::Acquire) + || !self.config.relay.is_live() + || !environment.accepting + || !pool.accepting + { + return Err(DockerExecutorError::Unavailable.into()); + } + Ok(()) + } + fn docker_zip_architecture(&self) -> Option { + self.config + .zip + .as_ref() + .map(|source| source.base.resolved().metadata().architecture) + } + + fn zip_available(&self, architecture: Architecture) -> Result<(), ExecutorError> { + self.image_available()?; + let source = self.config.zip.as_ref().ok_or_else(|| { + ExecutorError::Unsupported("Docker ZIP runtime is not configured".into()) + })?; + source + .available() + .map_err(EnvironmentError::from) + .map_err(DockerExecutorError::from)?; + if source.base.resolved().metadata().architecture != architecture { + return Err(ExecutorError::Unsupported( + "Docker ZIP architecture does not match the selected base".into(), + )); + } + Ok(()) + } + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + self.image_available()?; + self.plan(request)?; + Ok(()) + } + async fn invoke( + &self, + request: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.image_available()?; + context.check()?; + let plan = self.plan(&request)?; + let invocation = Invocation::new( + context + .id() + .to_string() + .parse() + .map_err(DockerExecutorError::from)?, + context.deadline(), + request + .function_arn + .parse() + .map_err(DockerExecutorError::from)?, + local_trace_id().map_err(DockerExecutorError::from)?, + Payload::try_from(request.payload.clone()).map_err(DockerExecutorError::from)?, + ) + .map_err(|error| match error { + SessionError::Deadline => ExecutorError::DeadlineExceeded, + error => DockerExecutorError::Session(error).into(), + })?; + context.check()?; + let mut call = self + .pool + .port + .start(plan, invocation, request.capture_logs) + .map_err(environment_error)?; + let result = tokio::select! { biased; + reason = context.cancelled() => Err(reason.error()), + result = call.wait() => result.map_err(environment_error), + }; + call.cancel_and_wait(Instant::now() + self.config.settings.cleanup) + .await + .map_err(|_| cleanup_error())?; + context.check()?; + response(result?, &request) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.start_cleanup(); + timeout_at( + Instant::now() + self.config.settings.cleanup, + self.pool.cleanup.wait(), + ) + .await + .map_err(|_| cleanup_error())? + } +} + +fn cleanup_error() -> ExecutorError { + ExecutorError::CleanupFailed("Docker resource settlement is not confirmed".into()) +} +fn environment_error(error: EnvironmentError) -> ExecutorError { + match error { + EnvironmentError::Cancelled => ExecutorError::Cancelled, + EnvironmentError::Deadline => ExecutorError::DeadlineExceeded, + EnvironmentError::Timeout(timeout) => ExecutorError::Timeout(timeout), + error => DockerExecutorError::Environment(error).into(), + } +} +fn response( + result: EnvironmentResponse, + request: &InvokeRequest, +) -> Result { + if result.logs.len() > 4096 { + return Err(DockerExecutorError::Environment(DockerError::Protocol.into()).into()); + } + let (payload, function_error) = match result.result { + RuntimeResult::Success(payload) => (payload, None), + RuntimeResult::FunctionError(payload) => (payload, Some("Unhandled".into())), + RuntimeResult::InitError(payload) if payload.as_ref().len() <= 32768 => { + return Err(environment_error(EnvironmentError::Initialization(payload))); + } + _ => return Err(environment_error(DockerError::Protocol.into())), + }; + Ok(InvokeResponse { + status: 200, + payload: payload.into(), + function_error, + log_tail: request.capture_logs.then(|| STANDARD.encode(result.logs)), + executed_version: request.revision.qualifier.clone(), + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/executor/tests.rs b/crates/rustack-lambda-core/src/docker/executor/tests.rs new file mode 100644 index 00000000..d7c1dfc4 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/executor/tests.rs @@ -0,0 +1,711 @@ +use std::{ + collections::HashMap, + error::Error as _, + future::Future, + task::{Context, Poll, Waker}, +}; + +use bytes::Bytes; +use serde_json::{to_string, to_value}; +use tokio::sync::{mpsc::error::TryRecvError, oneshot, watch}; +use uuid::Uuid; + +use super::*; +use crate::{ + artifact::ImageArtifact, + config::LambdaConfig, + deployment::ImageDeploymentConfig, + docker::{ + DockerWire, EnvironmentConfig, EnvironmentManager, ImageLeaseConfig, ImageLeaseManager, + ImagePinScope, RelayBudgets, + environment::test_peer, + plan::tests::{function_body, image_with, policy, relay_body, revision}, + sdk::{LocalSdkCredentials, SdkAffiliation}, + tests::Daemon, + }, + error::LambdaServiceError, + executor::ExecutorBackend, + provider::RustackLambda, + storage::{FunctionStore, tests::sample_record}, + work::WorkManager, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +#[derive(Debug)] +struct Readiness(AtomicBool); +impl DockerReadiness for Readiness { + fn check(&self) -> Result<(), DockerExecutorError> { + if self.0.load(Ordering::Acquire) { + Ok(()) + } else { + Err(DockerExecutorError::Unavailable) + } + } +} +struct Fixture { + daemon: Daemon, + wire: DockerWire, + environments: EnvironmentManager, + directory: DirectoryLease, + ready: Arc, +} +impl Fixture { + async fn new() -> Self { + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let environments = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + Self { + daemon, + wire, + environments, + directory, + ready: Arc::new(Readiness(AtomicBool::new(true))), + } + } + fn config(&self) -> DockerExecutorConfig { + DockerExecutorConfig::builder() + .directory(self.directory.clone()) + .relay(image_with("1000", vec![])) + .policy(policy()) + .environments(self.environments.port()) + .readiness(self.ready.clone()) + .settings( + DockerExecutorSettings::new(PoolConfig::default(), Duration::from_secs(1)).unwrap(), + ) + .build() + } + async fn close(mut self) { + self.environments.shutdown(deadline()).await.unwrap(); + self.wire.shutdown().await.unwrap(); + self.daemon.close().await; + } +} +fn request() -> InvokeRequest { + InvokeRequest::new( + Arc::new(revision()), + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture".into(), + Bytes::from_static(b"\x00\xffraw-event"), + ) +} + +#[test] +fn test_should_bound_executor_cleanup_settings() { + for invalid in [ + Duration::ZERO, + Duration::from_millis(1500), + Duration::from_secs(61), + ] { + assert!(matches!( + DockerExecutorSettings::new(PoolConfig::default(), invalid), + Err(DockerExecutorError::Configuration) + )); + } +} + +#[rstest::rstest] +#[case::name("arn:aws:lambda:us-east-1:000000000000:function:other")] +#[case::region("arn:aws:lambda:us-west-2:000000000000:function:policy-fixture")] +#[case::alias_latest("arn:aws:lambda:us-east-1:000000000000:function:policy-fixture:prod")] +#[case::version_latest("arn:aws:lambda:us-east-1:000000000000:function:policy-fixture:2")] +#[case::malformed("private-invalid-metadata")] +#[tokio::test] +async fn test_should_reject_invalid_request_metadata_before_daemon_allocation(#[case] arn: &str) { + let fixture = Fixture::new().await; + let executor = DockerExecutor::new(fixture.config()).unwrap(); + let mut request = request(); + request.function_arn = arn.into(); + assert!(executor.available(&request).is_err()); + assert_eq!(fixture.environments.port().status().allocated, 0); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_validate_scope_readiness_and_payload_without_allocating() { + let fixture = Fixture::new().await; + let mut wrong = fixture.config(); + wrong.directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + assert!(matches!( + DockerExecutor::new(wrong), + Err(DockerExecutorError::Configuration) + )); + let mut revoked = fixture.config(); + revoked.relay = PinnedImage::new( + revoked.relay.resolved().clone(), + revoked.relay.unpacked_size(), + watch::channel(()).0, + Arc::new(AtomicBool::new(false)), + watch::channel(false).1, + ); + assert!(matches!( + DockerExecutor::new(revoked), + Err(DockerExecutorError::Configuration) + )); + let executor = DockerExecutor::new(fixture.config()).unwrap(); + executor.available(&request()).unwrap(); + fixture.ready.0.store(false, Ordering::Release); + assert!(matches!( + executor.image_available(), + Err(ExecutorError::Docker(DockerExecutorError::Unavailable)) + )); + fixture.ready.0.store(true, Ordering::Release); + let mut invalid = request(); + invalid.payload = Bytes::from(vec![0; 6 * 1024 * 1024 + 1]); + assert!(executor.available(&invalid).is_err()); + let mut invalid = request(); + Arc::make_mut(&mut invalid.revision).backend = crate::executor::ExecutorBackend::Native; + assert!(executor.available(&invalid).is_err()); + let mut published = request(); + Arc::make_mut(&mut published.revision).qualifier = "1".into(); + assert!(executor.available(&published).is_err()); + published.function_arn.push_str(":prod"); + executor.available(&published).unwrap(); + published.function_arn = published.function_arn.replace(":prod", ":1"); + executor.available(&published).unwrap(); + assert_eq!(fixture.environments.port().status().allocated, 0); + executor.shutdown().await.unwrap(); + assert!(executor.image_available().is_err()); + fixture.close().await; +} + +#[rstest::rstest] +#[case::success(false, false)] +#[case::error(false, true)] +#[case::tail(true, false)] +#[case::tail_error(true, true)] +#[test] +fn test_should_preserve_binary_outcomes_and_encode_requested_window_once( + #[case] tail: bool, + #[case] failed: bool, +) { + let mut request = request(); + request.capture_logs = tail; + Arc::make_mut(&mut request.revision).qualifier = "17".into(); + request.function_arn.push_str(":prod"); + let raw = Bytes::from_static(b"\x00\xffprivate-handler-data"); + let payload = Payload::try_from(raw.clone()).unwrap(); + let logs = Bytes::from_static(b"\xff\x00private-log-data"); + let result = if failed { + RuntimeResult::FunctionError(payload) + } else { + RuntimeResult::Success(payload) + }; + let response = response( + EnvironmentResponse { + result, + logs: logs.clone(), + }, + &request, + ) + .unwrap(); + assert_eq!(response.payload, raw); + assert_eq!(response.status, 200); + assert_eq!( + response.function_error.as_deref(), + if failed { Some("Unhandled") } else { None } + ); + assert_eq!(response.log_tail, tail.then(|| STANDARD.encode(&logs))); + assert_eq!(response.executed_version, "17"); + assert!(!format!("{response:?}").contains("private")); +} +#[test] +fn test_should_preserve_empty_tail_and_reject_oversized_logs_or_init_success() { + let mut request = request(); + request.capture_logs = true; + let value = || RuntimeResult::Success(Payload::try_from(Bytes::new()).unwrap()); + assert_eq!( + response( + EnvironmentResponse { + result: value(), + logs: Bytes::new() + }, + &request + ) + .unwrap() + .log_tail + .as_deref(), + Some("") + ); + assert!( + response( + EnvironmentResponse { + result: value(), + logs: Bytes::from(vec![0; 4097]) + }, + &request + ) + .is_err() + ); + for size in [32768, 32769] { + let result = + RuntimeResult::InitError(Payload::try_from(Bytes::from(vec![0; size])).unwrap()); + let error = response( + EnvironmentResponse { + result, + logs: Bytes::new(), + }, + &request, + ) + .unwrap_err(); + if size == 32768 { + assert!(matches!( + error, + ExecutorError::Docker(DockerExecutorError::Environment( + EnvironmentError::Initialization(_) + )) + )); + } else { + assert!(matches!( + error, + ExecutorError::Docker(DockerExecutorError::Environment(EnvironmentError::Docker( + DockerError::Protocol + ))) + )); + } + } +} + +#[test] +fn test_should_preserve_typed_environment_failures_and_static_service_mapping() { + let source = DockerExecutorError::from(ProtocolError::Io(std::io::Error::other( + "private-transport-data", + ))); + assert!(source.source().is_some()); + assert!(!format!("{source:?} {source}").contains("private-transport-data")); + assert!(matches!( + environment_error(EnvironmentError::Timeout(Duration::from_secs(3))), + ExecutorError::Timeout(_) + )); + assert!(matches!( + environment_error(EnvironmentError::Deadline), + ExecutorError::DeadlineExceeded + )); + assert!(matches!( + environment_error(EnvironmentError::Cancelled), + ExecutorError::Cancelled + )); + assert!(matches!( + LambdaServiceError::from(environment_error(EnvironmentError::Busy)), + LambdaServiceError::TooManyRequests + )); + let nested = environment_error(EnvironmentError::Docker(DockerError::Busy)); + assert!(matches!( + &nested, + ExecutorError::Docker(DockerExecutorError::Environment(EnvironmentError::Docker( + DockerError::Busy + ))) + )); + assert!(matches!( + LambdaServiceError::from(nested), + LambdaServiceError::Internal { .. } + )); + assert!(matches!( + LambdaServiceError::from(ExecutorError::from(DockerExecutorError::Unavailable)), + LambdaServiceError::DockerNotAvailable + )); + for error in [ + EnvironmentError::Initialization( + Payload::try_from(Bytes::from_static(b"private-error")).unwrap(), + ), + EnvironmentError::Exited { + exit_code: 137, + oom: true, + }, + EnvironmentError::Docker(DockerError::Protocol), + ] { + let error = environment_error(error); + assert!(error.source().is_some()); + assert!(!format!("{error:?} {error}").contains("private-error")); + } +} + +#[rstest::rstest] +#[case::released(false)] +#[case::cleanup_expired(true)] +#[tokio::test] +async fn test_should_hold_work_supervisor_concurrency_through_docker_cancellation_settlement( + #[case] expired: bool, +) { + let fixture = Fixture::new().await; + let config = fixture.config(); + let request = request(); + let plan = EnvironmentPlan::new( + config.directory.clone(), + config.relay.clone(), + request.revision.clone(), + config.policy.clone(), + ) + .unwrap(); + let (environment, peer) = test_peer::pair(); + let (entered, observed) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let (release, gate) = oneshot::channel(); + let peer = tokio::spawn(peer.serve_pending(entered, gate, stopping)); + let pool = super::super::pool::testing_start( + fixture.environments.port(), + PoolConfig::default(), + vec![(plan.warm_key().into(), environment)], + ); + let port = pool.port(); + let executor = Arc::new(DockerExecutor::with_pool( + config, + pool, + port, + Handle::current(), + )); + let work = WorkManager::new(); + let handle = work + .submit(executor.clone(), request.clone(), Some(1), false, None) + .await + .unwrap(); + let id = handle.id(); + let actual = observed.await.unwrap(); + assert_eq!(actual.parts().0.to_string(), id.to_string()); + assert_eq!(actual.parts().2.as_ref(), request.function_arn.as_str()); + assert_eq!(actual.parts().4.as_ref(), request.payload.as_ref()); + let mut waiting = Box::pin(handle.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + stopped.await.unwrap(); + let rejected = work + .submit(executor.clone(), request, Some(1), false, None) + .await; + assert!( + matches!(rejected, Err(LambdaServiceError::TooManyRequests)), + "{rejected:?}" + ); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + let mut shutdown = Box::pin(executor.shutdown()); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(shutdown); + assert!(executor.image_available().is_err()); + if expired { + assert!(matches!( + executor.shutdown().await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(matches!( + waiting.await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(work.quiesce(Duration::from_secs(1)).await.is_err()); + release.send(()).unwrap(); + } else { + release.send(()).unwrap(); + waiting.await.unwrap(); + work.quiesce(Duration::from_secs(10)).await.unwrap(); + } + peer.await.unwrap(); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_reject_impossible_memory_before_provider_admission_as_a_throttle() { + let fixture = Fixture::new().await; + let executor = Arc::new(DockerExecutor::new(fixture.config()).unwrap()); + let mut oversized = request(); + Arc::make_mut(&mut oversized.revision).memory_mb = 10240; + assert!(matches!( + executor.available(&oversized), + Err(ExecutorError::Docker(DockerExecutorError::Environment( + EnvironmentError::MemoryLimit + ))) + )); + let work = WorkManager::new(); + assert!(matches!( + work.submit(executor.clone(), oversized, Some(1), false, None) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + assert_eq!(fixture.environments.port().status().allocated, 0); + assert_eq!(fixture.environments.port().status().memory_bytes, 0); + assert!(executor.available(&request()).is_ok()); + work.quiesce(Duration::from_secs(1)).await.unwrap(); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_distinguish_unconfigured_zip_from_revoked_backend_without_allocating() { + let fixture = Fixture::new().await; + let executor = DockerExecutor::new(fixture.config()).unwrap(); + assert!(matches!( + executor.zip_available(Architecture::Arm64), + Err(ExecutorError::Unsupported(_)) + )); + fixture.ready.0.store(false, Ordering::Release); + assert!(matches!( + executor.zip_available(Architecture::Arm64), + Err(ExecutorError::Docker(DockerExecutorError::Unavailable)) + )); + assert_eq!(fixture.environments.port().status().allocated, 0); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +fn sdk_context(config: &DockerExecutorConfig) -> DockerSdkEnvironment { + let credentials: LocalSdkCredentials = ("private-sdk-key".into(), "private-sdk-secret".into()) + .try_into() + .unwrap(); + DockerSdkEnvironment::new( + credentials, + "http://127.0.0.1:4566".parse().unwrap(), + SdkAffiliation::new( + config.directory.clone(), + config.relay.clone(), + config.policy.clone(), + config.environments.clone(), + config.readiness.clone(), + ), + ) + .unwrap() +} +#[rstest::rstest] +#[case::qualification("qualification")] +#[case::acquisition("acquisition")] +#[case::manager("manager")] +#[case::network("network")] +#[case::relay("relay")] +#[tokio::test] +async fn test_should_refuse_foreign_sdk_affiliation_before_pool_or_container_allocation( + #[case] changed: &str, +) { + let fixture = Fixture::new().await; + let original = fixture.config(); + let sdk = sdk_context(&original); + let mut other_manager = EnvironmentManager::start( + fixture.wire.port(), + original.directory.clone(), + EnvironmentConfig::default(), + ) + .unwrap(); + let mut foreign = original.clone(); + match changed { + "qualification" => foreign.readiness = Arc::new(Readiness(AtomicBool::new(true))), + "acquisition" => { + foreign.directory = + DirectoryLease::testing(original.directory.owner_id(), original.directory.run_id()); + } + "manager" => foreign.environments = other_manager.port(), + "network" => { + foreign.policy = ContainerPolicy::new( + "other-bridge".parse().unwrap(), + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap(); + } + "relay" => foreign.relay = image_with("1000", vec!["CHANGED=value".into()]), + _ => unreachable!("fixed affiliation cases"), + } + assert!(!sdk.matches(&foreign)); + foreign.sdk = Some(sdk.clone()); + assert!(matches!( + DockerExecutor::new(foreign), + Err(DockerExecutorError::Configuration) + )); + assert_eq!(fixture.environments.port().status().allocated, 0); + assert_eq!(other_manager.port().status().allocated, 0); + other_manager.shutdown(deadline()).await.unwrap(); + fixture.close().await; +} +#[tokio::test] +async fn test_should_isolate_sdk_warm_identity_without_mutating_revision_or_relay_environment() { + let fixture = Fixture::new().await; + let mut config = fixture.config(); + let first = sdk_context(&config); + let second = sdk_context(&config); + assert_eq!(first.identity(), first.clone().identity()); + assert_ne!(first.identity(), second.identity()); + assert_eq!(format!("{first:?}"), "DockerSdkEnvironment()"); + let mut call = request(); + Arc::make_mut(&mut call.revision).environment.insert( + "AWS_ENDPOINT_URL".into(), + "http://user-selected:4566".into(), + ); + let original_env = call.revision.environment.clone(); + let plain = DockerExecutor::new(config.clone()).unwrap(); + let without_sdk = plain.plan(&call).unwrap(); + assert!( + function_body(&without_sdk) + .env + .as_ref() + .unwrap() + .contains(&"AWS_ENDPOINT_URL=http://user-selected:4566".to_owned()) + ); + config.sdk = Some(first.clone()); + let enabled = DockerExecutor::new(config.clone()).unwrap(); + let first_plan = enabled.plan(&call).unwrap(); + let cloned_plan = enabled.plan(&call).unwrap(); + assert_ne!(without_sdk.warm_key(), first_plan.warm_key()); + assert_eq!(first_plan.warm_key(), cloned_plan.warm_key()); + assert!( + function_body(&first_plan) + .env + .as_ref() + .unwrap() + .contains(&"AWS_ENDPOINT_URL=http://127.0.0.1:4566/".to_owned()) + ); + assert!( + relay_body(&first_plan) + .env + .as_ref() + .unwrap() + .iter() + .all(|entry| !entry.starts_with("AWS_") && !entry.contains("private-sdk")) + ); + assert_eq!(call.revision.environment, original_env); + config.sdk = Some(second); + let other = DockerExecutor::new(config).unwrap(); + assert_ne!(first_plan.warm_key(), other.plan(&call).unwrap().warm_key()); + fixture.ready.0.store(false, Ordering::Release); + assert!(matches!( + enabled.plan(&call), + Err(DockerExecutorError::Unavailable) + )); + assert_eq!(fixture.environments.port().status().allocated, 0); + drop((without_sdk, first_plan, cloned_plan)); + plain.shutdown().await.unwrap(); + enabled.shutdown().await.unwrap(); + other.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_retain_sdk_revocation_guard_in_a_prepared_plan_and_refuse_create_without_rpc() +{ + let mut fixture = Fixture::new().await; + let mut config = fixture.config(); + config.sdk = Some(sdk_context(&config)); + let executor = DockerExecutor::new(config).unwrap(); + let plan = executor.plan(&request()).unwrap(); + plan.live().unwrap(); + fixture.ready.0.store(false, Ordering::Release); + assert!(matches!(plan.live(), Err(DockerError::Stopped))); + let port = fixture.wire.port(); + let mut relay = Box::pin(plan.create_relay(&port, deadline())); + assert!(matches!( + relay.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Ready(Err(DockerError::Stopped)) + )); + drop(relay); + let mut function = Box::pin(plan.create_function(&port, deadline())); + assert!(matches!( + function + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Ready(Err(DockerError::Stopped)) + )); + drop(function); + assert!(matches!( + fixture.daemon.requests.try_recv(), + Err(TryRecvError::Empty) + )); + assert_eq!(fixture.environments.port().status().allocated, 0); + drop(plan); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_export_real_sdk_provider_snapshot_without_injected_keys_or_credentials() { + let fixture = Fixture::new().await; + let mut config = fixture.config(); + config.sdk = Some(sdk_context(&config)); + let executor = Arc::new(DockerExecutor::new(config).unwrap()); + let mut call = request(); + let user_env = HashMap::from([ + ("BUSINESS".to_owned(), "retained".to_owned()), + ( + "AWS_ENDPOINT_URL".to_owned(), + "http://user-selected:4566".to_owned(), + ), + ]); + Arc::make_mut(&mut call.revision) + .environment + .extend(user_env.clone()); + let plan = executor.plan(&call).unwrap(); + assert!( + function_body(&plan) + .env + .as_ref() + .unwrap() + .iter() + .any(|entry| entry == "AWS_SECRET_ACCESS_KEY=private-sdk-secret") + ); + let DeploymentArtifact::Image(image) = &call.revision.artifact else { + panic!("fixed image fixture"); + }; + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let mut record = sample_record(&call.revision.function_name); + record.resource_id = call.revision.resource_id; + record.arn = call.function_arn.clone(); + record.latest.package_type = "Image".into(); + record.latest.runtime = None; + record.latest.handler = None; + record.latest.image_uri = Some(image.resolved().metadata().requested_reference.to_string()); + record.latest.image = ImageArtifact::Pinned(image.clone()); + record.latest.architectures = vec!["arm64".into()]; + record.latest.environment = user_env.clone(); + record.latest.code_sha256 = image.resolved().code_sha256().unwrap(); + record.latest.code_size = image.unpacked_size().get(); + store.insert(record).unwrap(); + let mut images = ImageLeaseManager::start( + fixture.wire.port(), + ImagePinScope::new(fixture.directory.clone()), + ImageLeaseConfig::default(), + ) + .unwrap(); + let provider = RustackLambda::with_executor( + store, + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + executor, + ) + .with_image_preparation(images.port(), ImageDeploymentConfig::default()) + .unwrap(); + let snapshot = provider.export_snapshot_before(deadline()).await.unwrap(); + let encoded = to_string(&snapshot).unwrap(); + for private in [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "private-sdk-key", + "private-sdk-secret", + ] { + assert!(!encoded.contains(private)); + } + let projected = to_value(&snapshot).unwrap(); + assert_eq!( + projected["functions"]["functions"][0]["latest"]["environment"], + to_value(user_env).unwrap() + ); + drop(plan); + provider.shutdown_before(deadline()).await.unwrap(); + images.shutdown(deadline()).await.unwrap(); + fixture.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/framing.rs b/crates/rustack-lambda-core/src/docker/framing.rs new file mode 100644 index 00000000..b95e2e5a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/framing.rs @@ -0,0 +1,221 @@ +//! Strict streaming non-TTY Docker output records; no full-record allocation. + +use std::{fmt, time::Duration}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncRead, AsyncReadExt}, + time::{Instant, timeout_at}, +}; + +use super::DockerError; + +/// Maximum outer Docker record, distinct from the inner CBOR frame allowance. +pub const MAX_DOCKER_RECORD: usize = 8 * 1024 * 1024; +const CHUNK_BYTES: usize = 16 * 1024; + +/// Docker's validated non-TTY output channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum OutputChannel { + /// Standard output; relay output is binary control data. + Stdout, + /// Standard error; function logs are kept separately from relay control. + Stderr, +} + +/// At most16 KiB from one validated Docker record; Debug redacts content. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct OutputChunk { + /// Output channel advertised by the validated record header. + pub channel: OutputChannel, + /// Raw bytes; no implicit UTF-8 conversion or line splitting. + pub bytes: Bytes, +} +impl fmt::Debug for OutputChunk { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutputChunk") + .field("channel", &self.channel) + .field("bytes", &self.bytes.len()) + .finish() + } +} + +#[derive(Debug)] +struct Record { + channel: OutputChannel, + remaining: usize, + deadline: Instant, +} + +/// One retained parser with a fixed deadline for each partial record. +#[non_exhaustive] +pub struct DockerOutputReader { + reader: R, + record: Option, + io_budget: Duration, + poisoned: bool, + closed: bool, +} +impl fmt::Debug for DockerOutputReader { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DockerOutputReader") + .field("record", &self.record) + .field("poisoned", &self.poisoned) + .field("closed", &self.closed) + .finish_non_exhaustive() + } +} + +impl DockerOutputReader { + /// Own the reader with a nonzero partial-record budget≤60s. + /// + /// # Errors + /// Rejects a zero/excessive IO budget. + /// + /// ``` + /// let reader = rustack_lambda_core::docker::DockerOutputReader::new( + /// b"".as_slice(), std::time::Duration::from_secs(5))?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(reader: R, io_budget: Duration) -> Result { + if io_budget.is_zero() || io_budget > Duration::from_secs(60) { + return Err(DockerError::Configuration); + } + Ok(Self { + reader, + record: None, + io_budget, + poisoned: false, + closed: false, + }) + } + + /// Read one bounded chunk. Cancelling this future invalidates the reader. + /// + /// Boundary EOF returns None; a partial header/payload fails. The caller's + /// absolute deadline may tighten, but never extend, a partial record lease. + /// + /// # Errors + /// Returns framing, size, deadline, IO or interrupted-reader errors. + /// + /// ```no_run + /// # async fn example(reader: &mut rustack_lambda_core::docker::DockerOutputReader) { + /// let chunk = reader.next(tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await; + /// # } + /// ``` + pub async fn next(&mut self, deadline: Instant) -> Result, DockerError> { + if self.poisoned { + return Err(DockerError::Poisoned); + } + if self.closed { + return Ok(None); + } + self.poisoned = true; + let result = self.read(deadline).await; + if result.is_ok() { + self.poisoned = false; + } + result + } + + async fn read(&mut self, deadline: Instant) -> Result, DockerError> { + let mut empty = 0_u8; + loop { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if self.record.is_none() { + let mut header = [0_u8; 8]; + let first = header.get_mut(..1).ok_or(DockerError::Protocol)?; + let count = timeout_at(deadline, self.reader.read(first)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if count == 0 { + self.closed = true; + return Ok(None); + } + let record_deadline = Instant::now() + .checked_add(self.io_budget) + .ok_or(DockerError::Configuration)? + .min(deadline); + timeout_at( + record_deadline, + self.reader + .read_exact(header.get_mut(1..).ok_or(DockerError::Protocol)?), + ) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= record_deadline { + return Err(DockerError::Deadline); + } + let channel = match header.first() { + Some(1) => OutputChannel::Stdout, + Some(2) => OutputChannel::Stderr, + _ => return Err(DockerError::Protocol), + }; + if header.get(1..4) != Some([0_u8; 3].as_slice()) { + return Err(DockerError::Protocol); + } + let length: [u8; 4] = header + .get(4..) + .ok_or(DockerError::Protocol)? + .try_into() + .map_err(|_| DockerError::Protocol)?; + let remaining = usize::try_from(u32::from_be_bytes(length)) + .map_err(|_| DockerError::TooLarge)?; + if remaining > MAX_DOCKER_RECORD { + return Err(DockerError::TooLarge); + } + if remaining == 0 { + empty = empty.saturating_add(1); + if empty == 64 { + tokio::task::yield_now().await; + empty = 0; + } + continue; + } + self.record = Some(Record { + channel, + remaining, + deadline: record_deadline, + }); + } + let record = self.record.as_mut().ok_or(DockerError::Protocol)?; + let until = record.deadline.min(deadline); + if Instant::now() >= until { + return Err(DockerError::Deadline); + } + let size = record.remaining.min(CHUNK_BYTES); + let mut output = Vec::new(); + output + .try_reserve_exact(size) + .map_err(|_| DockerError::Allocation)?; + output.resize(size, 0); + timeout_at(until, self.reader.read_exact(&mut output)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= until { + return Err(DockerError::Deadline); + } + record.remaining = record + .remaining + .checked_sub(size) + .ok_or(DockerError::Protocol)?; + let channel = record.channel; + if record.remaining == 0 { + self.record = None; + } + return Ok(Some(OutputChunk { + channel, + bytes: Bytes::from(output), + })); + } + } +} diff --git a/crates/rustack-lambda-core/src/docker/image.rs b/crates/rustack-lambda-core/src/docker/image.rs new file mode 100644 index 00000000..8fdcfc72 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/image.rs @@ -0,0 +1,782 @@ +//! Read-only platform selection and executable-image validation before deployment. + +use bytes::Bytes; +use http::{Method, StatusCode}; +use rustack_lambda_model::types::ImageConfig; +use tokio::time::Instant; + +use super::{ + DockerError, DockerPort, Operation, decode, escaped, + models::{ImageInspect, ImageManifestSummaryKindEnum, OciDescriptor, SystemInfo}, +}; +use crate::{ + artifact::{ + Architecture, ArtifactError, ImageExecutionDefaults, ImageExecutionMetadata, ImageMetadata, + ImageReference, ResolvedImage, Sha256Digest, UnpackedImageSize, + }, + function_config::validate_image_config, +}; + +/// Read-only local resolution; deployment publication remains with its actor. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ImageResolver { + port: DockerPort, +} + +impl ImageResolver { + /// Use an already negotiated bounded Docker port; never discover a daemon. + /// + /// ```no_run + /// # fn resolver(port: rustack_lambda_core::docker::DockerPort) { + /// let resolver = rustack_lambda_core::docker::ImageResolver::new(port); + /// # } + /// ``` + #[must_use] + pub fn new(port: DockerPort) -> Self { + Self { port } + } + + /// Resolve a preloaded image and validate effective entry configuration. + /// + /// This never pulls, builds or reads registry credentials. Function settings + /// and deployment status/CAS are validated and owned by the provider. + /// + /// # Errors + /// Returns wire failure, invalid overrides or invalid/ambiguous inspection + /// metadata. No error includes image argv, environment or daemon text. + /// + /// ```no_run + /// # async fn example(port: rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// use rustack_lambda_core::{artifact::Architecture, docker::ImageResolver}; + /// let image = ImageResolver::new(port).resolve_local(&"worker:v1".parse()?, + /// Architecture::Arm64, None, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn resolve_local( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + ) -> Result { + super::check_deadline(deadline)?; + if let Some(overrides) = overrides { + validate_image_config(overrides).map_err(|_| DockerError::Configuration)?; + } + let root = self.port.inspect_image(reference, deadline).await?; + self.resolve_captured(reference, architecture, overrides, root, deadline) + .await + } + + /// Resolve the one already captured initial/post-pull root without another tag lookup. + pub(super) async fn resolve_captured( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + root: ImageInspect, + deadline: Instant, + ) -> Result { + super::check_deadline(deadline)?; + if let Some(overrides) = overrides { + validate_image_config(overrides).map_err(|_| DockerError::Configuration)?; + } + let launch_id: Sha256Digest = root.id.as_deref().ok_or_else(invalid)?.parse()?; + let inspected = if self.port.api_version().minor() >= 49 { + drop(root); + let platform = match architecture { + Architecture::X86_64 => r#"{"os":"linux","architecture":"amd64"}"#, + Architecture::Arm64 => r#"{"os":"linux","architecture":"arm64"}"#, + }; + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("platform", platform) + .finish(); + let path = format!("/images/{}/json?{query}", escaped(launch_id.as_str())); + let selected = self + .port + .call( + Operation::Image, + Method::GET, + path, + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await + .map_err(|error| match error { + // The reference already resolved to this captured ID. This + // operation asks whether it supplies the requested platform. + DockerError::Status(StatusCode::NOT_FOUND) => invalid(), + other => other, + })?; + decode(selected, deadline)? + } else { + root + }; + if inspected.descriptor.is_none() { + let info = self.port.info(deadline).await?; + verify_classic_store(&inspected, &info)?; + } + let result = resolve(reference, architecture, overrides, inspected, &launch_id); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + + /// Prove the selected image's unpacked-storage quota without running code. + /// + /// # Errors + /// Rejects missing, ambiguous, incompatible or out-of-quota unpacked proof, + /// classic-store ambiguity, expiry or bounded Docker metadata/IO failure. + /// + /// ```no_run + /// # async fn size(port:rustack_lambda_core::docker::DockerPort,image:&rustack_lambda_core::artifact::ResolvedImage) { + /// let bytes = rustack_lambda_core::docker::ImageResolver::new(port).unpacked_size(image, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn unpacked_size( + &self, + image: &ResolvedImage, + deadline: Instant, + ) -> Result { + super::check_deadline(deadline)?; + let root = self + .port + .image_manifests(&image.metadata().image_id, deadline) + .await?; + if image.metadata().selected_manifest.is_none() { + verify_classic_store(&root, &self.port.info(deadline).await?)?; + } + let size = unpacked_proof(image, &root)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + Ok(size) + } +} + +fn unpacked_proof( + image: &ResolvedImage, + root: &ImageInspect, +) -> Result { + let metadata = image.metadata(); + if root.id.as_deref() != Some(metadata.image_id.as_str()) { + return Err(invalid()); + } + let Some(manifest) = &metadata.selected_manifest else { + if root.os.as_deref() != Some("linux") + || root.architecture.as_deref() != Some(docker_arch(metadata.architecture)) + { + return Err(invalid()); + } + let size = root + .size + .and_then(|size| u64::try_from(size).ok()) + .ok_or_else(invalid)?; + return size.try_into().map_err(DockerError::from); + }; + let summaries = root.manifests.as_deref().ok_or_else(invalid)?; + if summaries.len() > 128 { + return Err(invalid()); + } + let mut matching = summaries + .iter() + .filter(|summary| summary.id == manifest.as_str()); + let summary = matching.next().ok_or_else(invalid)?; + if matching.next().is_some() + || !summary.available + || summary.kind != Some(ImageManifestSummaryKindEnum::IMAGE) + || image_descriptor(&summary.descriptor, docker_arch(metadata.architecture))? != *manifest + { + return Err(invalid()); + } + let data = summary.image_data.as_ref().ok_or_else(invalid)?; + if data.platform.os.as_deref() != Some("linux") + || data.platform.architecture.as_deref() != Some(docker_arch(metadata.architecture)) + { + return Err(invalid()); + } + let bytes = u64::try_from(data.size.unpacked).map_err(|_| invalid())?; + bytes.try_into().map_err(DockerError::from) +} + +fn docker_arch(architecture: Architecture) -> &'static str { + match architecture { + Architecture::X86_64 => "amd64", + Architecture::Arm64 => "arm64", + } +} + +fn verify_classic_store(inspected: &ImageInspect, info: &SystemInfo) -> Result<(), DockerError> { + let driver = inspected.graph_driver.as_ref().ok_or_else(invalid)?; + if inspected.descriptor.is_some() + || !matches!( + driver.name.as_str(), + "overlay2" | "aufs" | "btrfs" | "devicemapper" | "vfs" | "zfs" | "fuse-overlayfs" + ) + || info.driver.as_deref() != Some(driver.name.as_str()) + || inspected + .manifests + .as_ref() + .is_some_and(|manifests| !manifests.is_empty()) + { + return Err(invalid()); + } + let statuses = info.driver_status.as_deref().unwrap_or_default(); + if statuses.len() > 128 { + return Err(invalid()); + } + for status in statuses { + let [key, value] = status.as_slice() else { + return Err(invalid()); + }; + if key.len() > 256 + || value.len() > 16 * 1024 + || (key.eq_ignore_ascii_case("driver-type") && value == "io.containerd.snapshotter.v1") + { + return Err(invalid()); + } + } + Ok(()) +} + +fn invalid() -> DockerError { + ArtifactError::Inspection.into() +} + +fn resolve( + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + inspected: ImageInspect, + launch_id: &Sha256Digest, +) -> Result { + let docker_arch = match architecture { + Architecture::X86_64 => "amd64", + Architecture::Arm64 => "arm64", + }; + if inspected.os.as_deref() != Some("linux") + || inspected.architecture.as_deref() != Some(docker_arch) + { + return Err(invalid()); + } + let _: Sha256Digest = inspected.id.as_deref().ok_or_else(invalid)?.parse()?; + let size = inspected + .size + .and_then(|value| u64::try_from(value).ok()) + .filter(|size| (1..=10 * 1024 * 1024 * 1024).contains(size)) + .ok_or_else(invalid)?; + let descriptor = selected_descriptor(&inspected, docker_arch)?; + let registry = registry_manifest(reference, &inspected, descriptor.as_ref())?; + let config = inspected.config.ok_or_else(invalid)?; + if config + .volumes + .as_ref() + .is_some_and(|volumes| !volumes.is_empty()) + { + return Err(invalid()); + } + let defaults: ImageExecutionDefaults = ImageExecutionMetadata::builder() + .entrypoint(config.entrypoint.unwrap_or_default()) + .command(config.cmd.unwrap_or_default()) + .working_directory(config.working_dir.unwrap_or_default()) + .environment(config.env.unwrap_or_default()) + .user(config.user.unwrap_or_default()) + .build() + .try_into()?; + defaults.plan(overrides)?; + let inspection_hash = ImageMetadata::execution_fingerprint( + launch_id, + descriptor.as_ref(), + architecture, + size, + &defaults, + )?; + ImageMetadata::builder() + .requested_reference(reference.clone()) + .image_id(launch_id.clone()) + .selected_manifest(descriptor) + .registry_manifest(registry) + .os("linux".into()) + .architecture(architecture) + .inspection_hash(inspection_hash) + .size_bytes(size) + .execution(defaults) + .build() + .try_into() + .map_err(DockerError::from) +} + +fn selected_descriptor( + inspected: &ImageInspect, + architecture: &str, +) -> Result, DockerError> { + if inspected + .manifests + .as_ref() + .is_some_and(|manifests| !manifests.is_empty()) + { + return Err(invalid()); + } + inspected + .descriptor + .as_ref() + .map(|descriptor| image_descriptor(descriptor, architecture)) + .transpose() +} + +fn image_descriptor( + descriptor: &OciDescriptor, + architecture: &str, +) -> Result { + if !matches!( + descriptor.media_type.as_deref(), + Some( + "application/vnd.oci.image.manifest.v1+json" + | "application/vnd.docker.distribution.manifest.v2+json" + ) + ) || descriptor.artifact_type.is_some() + || descriptor.platform.as_ref().is_some_and(|platform| { + platform.os.as_deref().is_some_and(|os| os != "linux") + || platform + .architecture + .as_deref() + .is_some_and(|arch| arch != architecture) + }) + || descriptor.annotations.as_ref().is_some_and(|annotations| { + annotations + .get("vnd.docker.reference.type") + .is_some_and(|kind| kind == "attestation-manifest") + }) + { + return Err(invalid()); + } + descriptor + .digest + .as_deref() + .ok_or_else(invalid)? + .parse() + .map_err(DockerError::from) +} + +fn repository_identity(reference: &ImageReference) -> String { + let (registry, repository) = + reference + .registry() + .map_or(("docker.io", reference.repository()), |registry| { + ( + registry, + reference + .repository() + .split_once('/') + .map_or("", |(_, repository)| repository), + ) + }); + let registry = registry.to_ascii_lowercase(); + let registry = match registry.as_str() { + "index.docker.io" => "docker.io", + value => value, + }; + if registry == "docker.io" && !repository.contains('/') { + format!("{registry}/library/{repository}") + } else { + format!("{registry}/{repository}") + } +} + +fn registry_manifest( + reference: &ImageReference, + inspected: &ImageInspect, + descriptor: Option<&Sha256Digest>, +) -> Result, DockerError> { + let digests = inspected.repo_digests.as_deref().unwrap_or_default(); + if digests.len() > 128 { + return Err(invalid()); + } + let repository = repository_identity(reference); + let mut matches = Vec::new(); + for candidate in digests { + let candidate: ImageReference = candidate.parse()?; + let digest: Sha256Digest = candidate.digest().ok_or_else(invalid)?.parse()?; + if repository_identity(&candidate) == repository { + matches.push(digest); + } + } + if let Some(requested) = reference.digest() { + if descriptor.is_none_or(|digest| digest.as_str() != requested) + && !matches.iter().any(|digest| digest.as_str() == requested) + { + return Err(invalid()); + } + } + if reference.registry().is_none() || matches.is_empty() { + return Ok(None); + } + let selected = if let Some(descriptor) = descriptor { + descriptor + } else if let Some(requested) = reference.digest() { + matches + .iter() + .find(|digest| digest.as_str() == requested) + .ok_or_else(invalid)? + } else { + matches + .iter() + .min_by(|left, right| left.as_str().cmp(right.as_str())) + .ok_or_else(invalid)? + }; + Ok(Some( + format!("{}@{}", reference.repository(), selected.as_str()).parse()?, + )) +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn digest(letter: char) -> String { + format!("sha256:{}", letter.to_string().repeat(64)) + } + fn inspection() -> Value { + json!({"Id":digest('a'),"Os":"linux","Architecture":"arm64","Size":1024, + "Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json", "digest":digest('b'), + "platform":{"os":"linux","architecture":"arm64"}}, + "RepoDigests":[format!("registry.test/team/worker@{}",digest('c'))], + "Config":{"Entrypoint":["/entry"],"Cmd":["handler"],"WorkingDir":"/task","Env":["SECRET=secret_value"],"User":"1000"}}) + } + fn selected(reference: &str, value: Value) -> Result { + resolve( + &reference.parse()?, + Architecture::Arm64, + None, + serde_json::from_value(value).unwrap(), + &digest('a').parse()?, + ) + } + + #[test] + fn test_should_pin_selected_child_and_preserve_validated_defaults() { + for suffix in [ + ":v1".to_owned(), + format!("@{}", digest('c')), + format!("@{}", digest('b')), + ] { + let image = + selected(&format!("registry.test/team/worker{suffix}"), inspection()).unwrap(); + assert_eq!(image.metadata().image_id.as_str(), digest('a')); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('b')) + ); + assert_eq!( + image.metadata().execution.plan(None).unwrap().entrypoint(), + ["/entry"] + ); + assert_eq!( + image.metadata().execution.metadata().environment, + ["SECRET=secret_value"] + ); + assert!(!format!("{image:?}").contains("secret_value")); + let restored: ResolvedImage = + serde_json::from_slice(&serde_json::to_vec(&image).unwrap()).unwrap(); + assert_eq!( + restored.metadata().execution.plan(None).unwrap().command(), + ["handler"] + ); + } + assert!( + selected( + &format!("registry.test/team/worker@{}", digest('d')), + inspection() + ) + .is_err() + ); + } + + #[test] + fn test_should_keep_short_local_provenance_and_ignore_unrelated_repository_digests() { + let mut value = inspection(); + value["RepoDigests"] = json!([format!("team/worker@{}", digest('c'))]); + let image = selected("team/worker:v1", value).unwrap(); + assert_eq!(image.resolved_uri(), format!("local://{}", digest('a'))); + let mut value = inspection(); + value["RepoDigests"] = json!([format!("other.test/team/worker@{}", digest('c'))]); + assert!( + selected("registry.test/team/worker:v1", value) + .unwrap() + .metadata() + .registry_manifest + .is_none() + ); + } + + #[rstest::rstest] + #[case("docker.io/library/python:v1", "python")] + #[case("index.docker.io/python:v1", "docker.io/library/python")] + #[case("Registry.test/team/worker:v1", "registry.test/team/worker")] + fn test_should_normalize_familiar_repository_evidence( + #[case] requested: &str, + #[case] repository: &str, + ) { + let mut value = inspection(); + value["RepoDigests"] = json!([format!("{repository}@{}", digest('c'))]); + let image = selected(requested, value).unwrap(); + assert_eq!( + image + .metadata() + .registry_manifest + .as_ref() + .unwrap() + .digest(), + Some(digest('b').as_str()) + ); + } + + #[test] + fn test_should_select_legacy_manifest_evidence_deterministically() { + let mut value = inspection(); + value.as_object_mut().unwrap().remove("Descriptor"); + value["RepoDigests"] = json!([ + format!("registry.test/team/worker@{}", digest('d')), + format!("registry.test/team/worker@{}", digest('c')) + ]); + let image = selected("registry.test/team/worker:v1", value.clone()).unwrap(); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('c')) + ); + let image = selected(&format!("registry.test/team/worker@{}", digest('d')), value).unwrap(); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('d')) + ); + } + + #[rstest::rstest] + #[case("os")] + #[case("architecture")] + #[case("id")] + #[case("size_zero")] + #[case("size_negative")] + #[case("size_large")] + #[case("descriptor_index")] + #[case("descriptor_unknown")] + #[case("descriptor_digest")] + #[case("descriptor_arch")] + #[case("descriptor_os")] + #[case("descriptor_artifact")] + #[case("descriptor_attestation")] + #[case("digests_count")] + #[case("digests_invalid")] + #[case("digests_tag_only")] + #[case("config_missing")] + #[case("config_volumes")] + #[case("config_nul")] + #[case("config_empty_executable")] + fn test_should_reject_incompatible_or_ambiguous_image_metadata(#[case] field: &str) { + let mut value = inspection(); + match field { + "os" => value["Os"] = json!("windows"), + "architecture" => value["Architecture"] = json!("amd64"), + "id" => value["Id"] = json!("worker:v1"), + "size_zero" => value["Size"] = json!(0), + "size_negative" => value["Size"] = json!(-1), + "size_large" => value["Size"] = json!(10_u64 * 1024 * 1024 * 1024 + 1), + "descriptor_index" => { + value["Descriptor"]["mediaType"] = json!("application/vnd.oci.image.index.v1+json"); + } + "descriptor_unknown" => value["Descriptor"]["mediaType"] = json!("application/unknown"), + "descriptor_digest" => value["Descriptor"]["digest"] = json!("bad"), + "descriptor_arch" => value["Descriptor"]["platform"]["architecture"] = json!("amd64"), + "descriptor_os" => value["Descriptor"]["platform"]["os"] = json!("windows"), + "descriptor_artifact" => { + value["Descriptor"]["artifactType"] = json!("application/attestation"); + } + "descriptor_attestation" => { + value["Descriptor"]["annotations"] = + json!({"vnd.docker.reference.type":"attestation-manifest"}); + } + "digests_count" => { + value["RepoDigests"] = + json!(vec![ + format!("registry.test/team/worker@{}", digest('c')); + 129 + ]); + } + "digests_invalid" => value["RepoDigests"] = json!(["garbage@digest"]), + "digests_tag_only" => value["RepoDigests"] = json!(["registry.test/team/worker:v1"]), + "config_missing" => value["Config"] = Value::Null, + "config_volumes" => value["Config"]["Volumes"] = json!({"/data":{}}), + "config_nul" => value["Config"]["Cmd"] = json!(["bad\0"]), + "config_empty_executable" => { + value["Config"]["Entrypoint"] = json!([]); + value["Config"]["Cmd"] = json!([]); + } + _ => panic!("unknown test case"), + } + assert!(selected("registry.test/team/worker:v1", value).is_err()); + } + + #[test] + fn test_should_hash_only_normalized_base_execution_identity() { + let base = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let mut value = inspection(); + value["Config"]["Labels"] = json!({"ignored":"changed"}); + value["Created"] = json!("2025-01-01T00:00:00Z"); + value["RepoDigests"] = json!([ + format!("unrelated.test/other@{}", digest('e')), + format!("registry.test/team/worker@{}", digest('c')) + ]); + let same = selected("registry.test/team/worker:v2", value).unwrap(); + assert_eq!( + base.metadata().inspection_hash, + same.metadata().inspection_hash + ); + for field in ["Entrypoint", "Cmd", "Env", "User", "WorkingDir"] { + let mut value = inspection(); + value["Config"][field] = match field { + "Entrypoint" | "Cmd" => json!(["changed"]), + "Env" => json!(["SECRET=changed"]), + "User" => json!("1001"), + _ => json!("/changed"), + }; + let changed = selected("registry.test/team/worker:v1", value).unwrap(); + assert_ne!( + base.metadata().inspection_hash, + changed.metadata().inspection_hash + ); + } + let mut value = inspection(); + value["Config"]["Entrypoint"] = json!([]); + value["Config"]["Cmd"] = json!([]); + let overrides = ImageConfig { + command: Some(vec!["/provided".into()]), + ..Default::default() + }; + let image = resolve( + &"registry.test/team/worker:v1".parse().unwrap(), + Architecture::Arm64, + Some(&overrides), + serde_json::from_value(value).unwrap(), + &digest('a').parse().unwrap(), + ) + .unwrap(); + assert!(image.metadata().execution.plan(None).is_err()); + assert_eq!( + image + .metadata() + .execution + .plan(Some(&overrides)) + .unwrap() + .command(), + ["/provided"] + ); + } + #[test] + fn test_should_require_consistent_classic_evidence_and_reject_manifest_summaries() { + let mut value = inspection(); + value.as_object_mut().unwrap().remove("Descriptor"); + value["GraphDriver"] = json!({"Name":"overlay2","Data":{}}); + let inspected: ImageInspect = serde_json::from_value(value.clone()).unwrap(); + let classic: SystemInfo = serde_json::from_value( + json!({"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"]]}), + ) + .unwrap(); + verify_classic_store(&inspected, &classic).unwrap(); + for statuses in [ + json!({"Driver":"vfs"}), + json!({"Driver":"vfs","DriverStatus":null}), + ] { + let mut vfs = inspected.clone(); + vfs.graph_driver.as_mut().unwrap().name = "vfs".into(); + verify_classic_store(&vfs, &serde_json::from_value(statuses).unwrap()).unwrap(); + } + for name in ["btrfs", "zfs"] { + let mut containerd = inspected.clone(); + containerd.graph_driver.as_mut().unwrap().name = name.into(); + let info:SystemInfo = serde_json::from_value(json!({"Driver":name,"DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]})).unwrap(); + assert!(verify_classic_store(&containerd, &info).is_err()); + } + for info in [ + json!({"Driver":"overlayfs","DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]}), + json!({"Driver":"overlay2","DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]}), + json!({"Driver":"overlay2","DriverStatus":[["malformed"]]}), + ] { + assert!( + verify_classic_store(&inspected, &serde_json::from_value(info).unwrap()).is_err() + ); + } + value["Manifests"] = json!([{"ID":digest('b'),"Descriptor":{},"Available":true,"Kind":"image","ImageData":null,"AttestationData":null,"Size":{"Content":1,"Total":1}}]); + let inspected: ImageInspect = serde_json::from_value(value).unwrap(); + assert!(verify_classic_store(&inspected, &classic).is_err()); + assert!(selected_descriptor(&inspected, "arm64").is_err()); + } + + fn storage_proof() -> Value { + json!({"Id":digest('a'),"Manifests":[{"ID":digest('b'), + "Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":digest('b'),"platform":{"os":"linux","architecture":"arm64"}}, + "Available":true,"Kind":"image","Size":{"Content":64,"Total":99999}, + "ImageData":{"Containers":[],"Platform":{"os":"linux","architecture":"arm64"},"Size":{"Unpacked":4096}}}]}) + } + + #[test] + fn test_should_measure_selected_unpacked_storage_instead_of_compressed_or_total_size() { + let image = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let proof: ImageInspect = serde_json::from_value(storage_proof()).unwrap(); + assert_eq!(unpacked_proof(&image, &proof).unwrap().get(), 4096); + let legacy = ResolvedImage::fixture(Architecture::Arm64); + let proof: ImageInspect = serde_json::from_value( + json!({"Id":digest('a'),"Size":8192,"Os":"linux","Architecture":"arm64"}), + ) + .unwrap(); + assert_eq!(unpacked_proof(&legacy, &proof).unwrap().get(), 8192); + } + + #[rstest::rstest] + #[case("parent")] + #[case("missing")] + #[case("duplicate")] + #[case("unavailable")] + #[case("attestation")] + #[case("id")] + #[case("descriptor")] + #[case("platform")] + #[case("data")] + #[case("zero")] + #[case("negative")] + #[case("large")] + fn test_should_reject_missing_or_incompatible_unpacked_storage_evidence(#[case] fault: &str) { + let image = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let mut proof = storage_proof(); + match fault { + "parent" => proof["Id"] = json!(digest('e')), + "missing" => proof["Manifests"] = json!([]), + "duplicate" => { + let duplicate = proof["Manifests"][0].clone(); + proof["Manifests"].as_array_mut().unwrap().push(duplicate); + } + "unavailable" => proof["Manifests"][0]["Available"] = json!(false), + "attestation" => proof["Manifests"][0]["Kind"] = json!("attestation"), + "id" => proof["Manifests"][0]["ID"] = json!(digest('e')), + "descriptor" => proof["Manifests"][0]["Descriptor"]["digest"] = json!(digest('e')), + "platform" => { + proof["Manifests"][0]["ImageData"]["Platform"]["architecture"] = json!("amd64"); + } + "data" => proof["Manifests"][0]["ImageData"] = Value::Null, + "zero" => proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = json!(0), + "negative" => proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = json!(-1), + "large" => { + proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = + json!(10_u64 * 1024 * 1024 * 1024 + 1); + } + _ => panic!("unknown test fault"), + } + assert!(unpacked_proof(&image, &serde_json::from_value(proof).unwrap()).is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/docker/lease.rs b/crates/rustack-lambda-core/src/docker/lease.rs new file mode 100644 index 00000000..ee73d4c5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease.rs @@ -0,0 +1,1214 @@ +//! Retained pin preparation, shared live leases and phase-aware cleanup recovery. + +use std::{ + collections::HashMap, + fmt, mem, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use http::StatusCode; +use rustack_lambda_model::types::ImageConfig; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinError, JoinHandle, JoinSet}, + time::{Instant, sleep, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + DockerError, DockerPort, ImagePin, ImagePinScope, ImagePullPolicy, ImageResolver, + check_deadline, + connection::stopped, + mutation::{MutationGuard, MutationReceipt}, + ownership::ImageRecord, +}; +use crate::{ + artifact::{ + Architecture, ArtifactError, ImageLeaseContext, ImageReference, PinnedImage, + PreparationAdmission, PreparationReceipt, ResolvedImage, Sha256Digest, UnpackedImageSize, + }, + executor::ExecutorError, + function_config::validate_image_config, + invocation::CleanupTask, +}; + +fn check_preparation_deadline(deadline: Instant) -> Result<(), DockerError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|value| !value.is_zero()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(3600) { + return Err(DockerError::Configuration); + } + Ok(()) +} + +/// Validated image-preparation and live-pin admission/IO budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ImageLeaseConfig { + pins: u16, + jobs: u8, + io: Duration, +} +impl Default for ImageLeaseConfig { + fn default() -> Self { + Self { + pins: 1024, + jobs: 8, + io: Duration::from_secs(5), + } + } +} +impl ImageLeaseConfig { + /// Validate bounded pin/job counts and the independent mutation IO budget. + /// + /// # Errors + /// Rejects pins outside1..8192, jobs outside1..32 or exceeding pins, and + /// zero/over60s IO budgets. + /// + /// ``` + /// let config = rustack_lambda_core::docker::ImageLeaseConfig::new(1024,8,std::time::Duration::from_secs(5))?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(pins: u16, jobs: u8, io: Duration) -> Result { + if !(1..=8192).contains(&pins) + || !(1..=32).contains(&jobs) + || u16::from(jobs) > pins + || io < Duration::from_secs(1) + || io > Duration::from_secs(60) + { + return Err(DockerError::Configuration); + } + Ok(Self { pins, jobs, io }) + } + fn deadline(self) -> Instant { + Instant::now() + self.io + } +} + +/// Diagnostic counts; successful cleanup is established by joined ownership. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ImageLeaseStatus { + /// Slots retained by queued, preparing, live or quarantined pins. + pub pins: usize, + /// Accepted preparations, including their failed-preparation cleanup. + pub preparing: usize, + /// Workers retaining uncertain/mismatching or failed cleanup. + pub quarantined: usize, + /// Whether new preparations can be accepted. + pub accepting: bool, + /// A worker/root supervision failure permanently closes admission. + pub worker_failed: bool, +} +#[derive(Debug, Default)] +struct Health { + failed: AtomicBool, + quarantined: AtomicUsize, +} + +#[derive(Debug)] +struct PinBudget { + capacity: Arc, + count: AtomicUsize, + changed: watch::Sender<()>, +} +impl PinBudget { + fn new(max: u16) -> Self { + let (changed, _) = watch::channel(()); + Self { + capacity: Arc::new(Semaphore::new(usize::from(max))), + count: AtomicUsize::new(0), + changed, + } + } + fn acquire(self: &Arc) -> Result { + let permit = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + self.count.fetch_add(permit.num_permits(), Ordering::AcqRel); + self.changed.send_replace(()); + Ok(PinPermit { + permit, + budget: self.clone(), + }) + } +} +#[derive(Debug)] +struct PinPermit { + permit: OwnedSemaphorePermit, + budget: Arc, +} +impl Drop for PinPermit { + fn drop(&mut self) { + self.budget + .count + .fetch_sub(self.permit.num_permits(), Ordering::AcqRel); + self.budget.changed.send_replace(()); + } +} +impl Health { + fn check(&self) -> Result<(), DockerError> { + if self.failed.load(Ordering::Acquire) { + return Err(DockerError::Worker); + } + if self.quarantined.load(Ordering::Acquire) > 0 { + return Err(DockerError::Quarantined); + } + Ok(()) + } +} + +#[derive(Debug)] +enum PreparationSource { + Reference { + reference: ImageReference, + architecture: Architecture, + }, + Restore(Box), +} +impl PreparationSource { + fn architecture(&self) -> Architecture { + match self { + Self::Reference { architecture, .. } => *architecture, + Self::Restore(saved) => saved.metadata().architecture, + } + } +} + +#[derive(TypedBuilder)] +struct Request { + source: PreparationSource, + overrides: Option, + deadline: Instant, + cancelled: watch::Receiver<()>, + reply: oneshot::Sender>, + slot: Option, + #[builder(default)] + retained_admission: Option>, +} +impl fmt::Debug for Request { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ImagePreparation") + .field("deadline", &self.deadline) + .finish_non_exhaustive() + } +} +#[derive(Debug)] +struct Envelope { + request: Request, + pin: ImagePin, + permit: PinPermit, +} + +/// Cloneable bounded preparation port; it does not own manager shutdown. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct ImageLeasePort { + #[builder(default)] + platform: Option, + sender: mpsc::Sender, + slots: Arc, + pins: Arc, + stop: watch::Sender, + health: Arc, + scope: ImagePinScope, + config: ImageLeaseConfig, +} +impl ImageLeasePort { + pub(crate) fn available(&self) -> Result<(), DockerError> { + self.health.check()?; + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + Ok(()) + } + pub(crate) fn preparation_jobs(&self) -> u8 { + self.config.jobs + } + + /// Prepare a live pinned artifact; dropping this future signals cancellation. + /// + /// # Errors + /// Returns invalid settings, bounded admission, deadline, metadata, mutation + /// or cleanup failure. In-flight mutations and cleanup remain supervised. + /// + /// ```no_run + /// # async fn prepare(port:&rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::docker::DockerError> { + /// let image = port.prepare(&"worker:v1".parse()?,rustack_lambda_core::artifact::Architecture::Arm64,None, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await?; + /// assert!(image.is_live()); + /// # Ok(()) } + /// ``` + pub async fn prepare( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + ) -> Result { + self.prepare_guarded( + PreparationSource::Reference { + reference: reference.clone(), + architecture, + }, + overrides, + deadline, + None, + ) + .await + } + + /// Reacquire local immutable content without following source tags or pulling. + /// + /// The saved ID, selected platform and execution fingerprint must match fresh + /// inspection before a new durable private pin can be created. Saved metadata + /// alone never grants availability. This does not publish an archive store. + /// + /// # Errors + /// Returns unavailable/exhausted admission, deadline, missing immutable content, + /// changed inspection metadata or retained mutation/cleanup failure. Cancellation + /// never discards an uncertain mutation or its ownership. + /// + /// ```no_run + /// # async fn restore(port: &rustack_lambda_core::docker::ImageLeasePort, saved: &rustack_lambda_core::artifact::ResolvedImage) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let image = port.restore(saved, None, tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; + /// assert!(image.is_live()); + /// # Ok(()) } + /// ``` + pub async fn restore( + &self, + saved: &ResolvedImage, + overrides: Option<&ImageConfig>, + deadline: Instant, + ) -> Result { + self.prepare_guarded( + PreparationSource::Restore(Box::new(saved.clone())), + overrides, + deadline, + None, + ) + .await + } + + pub(crate) async fn prepare_deployment( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + admission: Arc, + ) -> Result { + self.prepare_guarded( + PreparationSource::Reference { + reference: reference.clone(), + architecture, + }, + overrides, + deadline, + Some(admission), + ) + .await + } + + async fn prepare_guarded( + &self, + source: PreparationSource, + overrides: Option<&ImageConfig>, + deadline: Instant, + retained_admission: Option>, + ) -> Result { + let deployment = retained_admission.is_some(); + self.available()?; + if self + .platform + .is_some_and(|platform| platform != source.architecture()) + { + return Err(DockerError::Configuration); + } + check_preparation_deadline(deadline)?; + if let Some(config) = overrides { + validate_image_config(config).map_err(|_| DockerError::Configuration)?; + } + let slot = self + .slots + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + let permit = self.pins.acquire()?; + let pin = self.scope.allocate()?; + let (cancel, cancelled) = watch::channel(()); + let (reply, response) = oneshot::channel(); + let request = Request::builder() + .source(source) + .overrides(overrides.cloned()) + .deadline(deadline) + .cancelled(cancelled) + .reply(reply) + .slot(Some(slot)) + .retained_admission(retained_admission) + .build(); + self.sender + .try_send(Envelope { + request, + pin, + permit, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Stopped, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + drop(cancel); + self.available()?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let image = result?; + if !deployment { + image.accept_preparation(); + } + Ok(image) + } + /// Observe release of every pin reservation without stopping admission. + /// + /// # Errors + /// Returns expiry or supervision failure. A timeout never drops cleanup. + /// Concurrent new preparations can delay this observation. + /// + /// ```no_run + /// # async fn empty(port:&rustack_lambda_core::docker::ImageLeasePort) { + /// let clean=port.wait_empty(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn wait_empty(&self, deadline: Instant) -> Result<(), DockerError> { + check_deadline(deadline)?; + let mut changed = self.pins.changed.subscribe(); + while self.pins.count.load(Ordering::Acquire) > 0 { + timeout_at(deadline, changed.changed()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + } + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + self.health.check() + } + /// Inspect bounded resource counts without logging source image defaults. + /// + /// ```no_run + /// # fn status(port:&rustack_lambda_core::docker::ImageLeasePort) { + /// let status=port.status(); + /// # } + /// ``` + #[must_use] + pub fn status(&self) -> ImageLeaseStatus { + let worker_failed = self.health.failed.load(Ordering::Acquire); + let quarantined = self.health.quarantined.load(Ordering::Acquire); + ImageLeaseStatus { + pins: self.pins.count.load(Ordering::Acquire), + preparing: usize::from(self.config.jobs).saturating_sub(self.slots.available_permits()), + quarantined, + accepting: !worker_failed && quarantined == 0 && !*self.stop.borrow(), + worker_failed, + } + } +} + +/// Owner of every preparation, live alias and retained cleanup worker. +#[derive(Debug)] +#[non_exhaustive] +pub struct ImageLeaseManager { + port: ImageLeasePort, + task: Option>>, + primary: Option>, + settlement: Option>, + emergency: Arc, +} +impl ImageLeaseManager { + pub(super) fn physically_settled(&self) -> bool { + self.primary.is_some() + && self.task.is_none() + && self.emergency.is_finished() + && self.port.status().pins == 0 + && self.port.status().preparing == 0 + } + + /// Start under the current Tokio runtime with an explicit scoped Docker port. + /// + /// # Errors + /// Rejects absence of a Tokio runtime or a port not bound to this exact + /// directory lease. The scope is never taken from a function request. + /// + /// ```no_run + /// # fn start(port:rustack_lambda_core::docker::DockerPort,scope:rustack_lambda_core::docker::ImagePinScope) -> Result<(),rustack_lambda_core::docker::DockerError> { + /// let owner=rustack_lambda_core::docker::ImageLeaseManager::start(port,scope,Default::default())?; + /// # Ok(()) } + /// ``` + pub fn start( + docker: DockerPort, + scope: ImagePinScope, + config: ImageLeaseConfig, + ) -> Result { + Self::start_with_pull(docker, scope, config, ImagePullPolicy::default()) + } + + /// Start with an explicit validated operator pull policy. + /// + /// # Errors + /// Rejects absent runtime or a port not bound to this exact directory. + /// No startup readiness is granted by this constructor. + /// + /// ```no_run + /// # fn start(port:rustack_lambda_core::docker::DockerPort,scope:rustack_lambda_core::docker::ImagePinScope)->Result<(),rustack_lambda_core::docker::DockerError> { + /// let policy=rustack_lambda_core::docker::ImagePullPolicy::if_missing(vec!["registry.example".parse()?],Default::default(),std::time::Duration::from_secs(300))?; + /// let images=rustack_lambda_core::docker::ImageLeaseManager::start_with_pull(port,scope,Default::default(),policy)?; + /// # Ok(()) } + /// ``` + pub fn start_with_pull( + docker: DockerPort, + scope: ImagePinScope, + config: ImageLeaseConfig, + policy: ImagePullPolicy, + ) -> Result { + Self::start_platform(docker, scope, config, policy, None) + } + pub(super) fn start_qualified( + docker: DockerPort, + scope: ImagePinScope, + config: ImageLeaseConfig, + policy: ImagePullPolicy, + platform: Architecture, + ) -> Result { + Self::start_platform(docker, scope, config, policy, Some(platform)) + } + fn start_platform( + docker: DockerPort, + scope: ImagePinScope, + config: ImageLeaseConfig, + policy: ImagePullPolicy, + platform: Option, + ) -> Result { + docker.require_directory(scope.directory())?; + let runtime = Handle::try_current().map_err(|_| DockerError::Worker)?; + let (sender, requests) = mpsc::channel(32); + let (stop, _) = watch::channel(false); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let port = ImageLeasePort::builder() + .platform(platform) + .sender(sender) + .slots(Arc::new(Semaphore::new(usize::from(config.jobs)))) + .pins(Arc::new(PinBudget::new(config.pins))) + .stop(stop.clone()) + .health(health.clone()) + .scope(scope) + .config(config) + .build(); + let actor = Actor::builder() + .docker(docker) + .policy(policy) + .config(config) + .inbox(requests) + .control(stop) + .health(health) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + let task = runtime.spawn(actor.run()); + Ok(Self { + port, + task: Some(task), + primary: None, + settlement: None, + emergency, + }) + } + /// Clone a preparation port; retain this owner through consumer shutdown. + /// + /// ```no_run + /// # fn port(owner:&rustack_lambda_core::docker::ImageLeaseManager) { + /// let port=owner.port(); + /// # } + /// ``` + #[must_use] + pub fn port(&self) -> ImageLeasePort { + self.port.clone() + } + + /// Revoke leases, stop admission and join pin cleanup before `DockerWire` stops. + /// + /// # Errors + /// Returns timeout while retaining cleanup, or cached supervision failure. + /// The runtime must first join invocation/environment consumers of the leases. + /// + /// ```no_run + /// # async fn stop(owner:&mut rustack_lambda_core::docker::ImageLeaseManager) { + /// let result=owner.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), DockerError> { + self.port.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + check_deadline(deadline)?; + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Err(DockerError::Worker), + }; + self.primary = Some(result); + self.task.take(); + } + let emergency = timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let result = self + .primary + .clone() + .ok_or(DockerError::Worker)? + .and(emergency); + self.settlement = Some(result.clone()); + result + } +} +impl Drop for ImageLeaseManager { + fn drop(&mut self) { + self.port.stop.send_replace(true); + } +} + +#[derive(Debug, Clone)] +enum Phase { + Unissued, + Preparing(Sha256Digest), + Uncertain(Sha256Digest, MutationReceipt), + Committing(Sha256Digest), + Known(Sha256Digest), + NoEffect(Sha256Digest), + Clean, +} +#[derive(Debug)] +struct Progress { + phase: watch::Sender, + released: watch::Receiver<()>, + active: Arc, +} +#[derive(Debug)] +struct Record { + pin: ImagePin, + progress: Arc, + permit: PinPermit, + _admission: PreparationReceipt, +} + +type PinJob = (Uuid, Result<(), DockerError>); +type PinJoin = Result<(Id, PinJob), JoinError>; + +#[derive(Debug, TypedBuilder)] +struct Actor { + docker: DockerPort, + #[builder(default)] + policy: ImagePullPolicy, + config: ImageLeaseConfig, + inbox: mpsc::Receiver, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default = Arc::new(Semaphore::new(2)))] + cleanup: Arc, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + records: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + async fn run(mut self) -> Result<(), DockerError> { + let mut stop = self.control.subscribe(); + let mut wire_stop = self.docker.raw.stop.subscribe(); + loop { + if *stop.borrow() { + self.inbox.close(); + while let Ok(envelope) = self.inbox.try_recv() { + let _ = envelope.request.reply.send(Err(DockerError::Stopped)); + } + for record in self.records.values() { + record.progress.active.store(false, Ordering::Release); + } + if self.jobs.is_empty() { + break; + } + if let Some(result) = self.jobs.join_next_with_id().await { + self.finished(&result); + } + continue; + } + tokio::select! { + biased; + ()=stopped(&mut wire_stop)=>{ self.health.failed.store(true,Ordering::Release);self.control.send_replace(true); }, + ()=stopped(&mut stop)=>{}, + result=self.jobs.join_next_with_id(),if !self.jobs.is_empty()=>if let Some(result)=result {self.finished(&result);}, + envelope=self.inbox.recv()=>match envelope {Some(envelope)=>self.start(envelope),None=>{self.control.send_replace(true);}}, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(DockerError::Worker) + } else { + Ok(()) + } + } + fn start(&mut self, mut envelope: Envelope) { + if self.health.check().is_err() + || *self.control.borrow() + || envelope.request.reply.is_closed() + || envelope.request.cancelled.has_changed().is_err() + || Instant::now() >= envelope.request.deadline + { + let _ = envelope.request.reply.send(Err(DockerError::Stopped)); + return; + } + let Some(slot) = envelope.request.slot.take() else { + self.health.failed.store(true, Ordering::Release); + self.control.send_replace(true); + let _ = envelope.request.reply.send(Err(DockerError::Worker)); + return; + }; + let (admission, _) = watch::channel(Some(PreparationAdmission::new( + slot, + envelope.request.retained_admission.take(), + ))); + let (phase, _) = watch::channel(Phase::Unissued); + let (release, released) = watch::channel(()); + let progress = Arc::new(Progress { + phase, + released, + active: Arc::new(AtomicBool::new(false)), + }); + let id = envelope.pin.id; + let worker = Worker::builder() + .docker(self.docker.clone()) + .policy(self.policy.clone()) + .config(self.config) + .pin(envelope.pin.clone()) + .progress(progress.clone()) + .control(self.control.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .build(); + self.records.insert( + id, + Record { + pin: envelope.pin, + progress, + permit: envelope.permit, + _admission: admission.clone(), + }, + ); + let health = self.health.clone(); + let control = self.control.clone(); + let job = self.jobs.spawn(async move { + let mut guard = WorkerGuard { + health, + control, + finished: false, + }; + let result = worker.run(envelope.request, release, admission).await; + guard.finished = true; + (id, result) + }); + self.index.insert(job.id(), id); + } + fn finished(&mut self, result: &PinJoin) { + let task = match result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(id) = self.index.remove(&task) else { + self.fail(); + return; + }; + let clean = self.records.get(&id).is_some_and(|record| { + matches!( + *record.progress.phase.borrow(), + Phase::Unissued | Phase::Clean + ) + }); + if matches!(result, Ok((_, (returned, Ok(())))) if *returned==id) && clean { + if let Some(record) = self.records.remove(&id) { + drop(record.permit); + } + return; + } + self.fail(); + let Some(record) = self.records.get(&id) else { + return; + }; + record.progress.active.store(false, Ordering::Release); + let worker = Worker::builder() + .docker(self.docker.clone()) + .config(self.config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(self.control.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .build(); + let job = self.jobs.spawn(async move { + worker.retire().await; + (id, Ok(())) + }); + self.index.insert(job.id(), id); + } + fn fail(&self) { + self.health.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + if self.records.is_empty() { + return; + } + self.fail(); + for record in self.records.values() { + record.progress.active.store(false, Ordering::Release); + } + let records = mem::take(&mut self.records); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + let docker = self.docker.clone(); + let config = self.config; + let control = self.control.clone(); + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover_all(records, docker, config, control, health, cleanup).await + }); + } +} + +async fn recover_all( + mut records: HashMap, + docker: DockerPort, + config: ImageLeaseConfig, + control: watch::Sender, + health: Arc, + cleanup: Arc, +) -> Result<(), ExecutorError> { + let mut jobs = JoinSet::new(); + let mut index = HashMap::new(); + for (id, record) in &records { + let worker = Worker::builder() + .docker(docker.clone()) + .config(config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(control.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .build(); + let id = *id; + let job = jobs.spawn(async move { + worker.retire().await; + id + }); + index.insert(job.id(), id); + } + while let Some(result) = jobs.join_next_with_id().await { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some(id) = index.remove(&task) else { + continue; + }; + let Some(record) = records.get(&id) else { + continue; + }; + if matches!( + *record.progress.phase.borrow(), + Phase::Unissued | Phase::Clean + ) { + if let Some(record) = records.remove(&id) { + drop(record.permit); + } + } else { + let worker = Worker::builder() + .docker(docker.clone()) + .config(config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(control.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .build(); + let job = jobs.spawn(async move { + worker.retire().await; + id + }); + index.insert(job.id(), id); + } + } + if records.is_empty() { + Ok(()) + } else { + Err(ExecutorError::CleanupFailed( + "Image pin recovery lost supervision".into(), + )) + } +} + +struct WorkerGuard { + health: Arc, + control: watch::Sender, + finished: bool, +} +impl Drop for WorkerGuard { + fn drop(&mut self) { + if !self.finished { + self.health.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } + } +} +struct Dirty(Arc); +impl Dirty { + fn new(health: Arc) -> Self { + health.quarantined.fetch_add(1, Ordering::AcqRel); + Self(health) + } +} +impl Drop for Dirty { + fn drop(&mut self) { + self.0.quarantined.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Debug, TypedBuilder)] +struct Worker { + docker: DockerPort, + #[builder(default)] + policy: ImagePullPolicy, + config: ImageLeaseConfig, + pin: ImagePin, + progress: Arc, + control: watch::Sender, + health: Arc, + cleanup: Arc, +} +impl Worker { + fn check(&self, deadline: Instant, cancel: &watch::Receiver<()>) -> Result<(), DockerError> { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if *self.control.borrow() { + return Err(DockerError::Stopped); + } + if cancel.has_changed().is_err() { + return Err(DockerError::Cancelled); + } + Ok(()) + } + async fn run( + mut self, + request: Request, + release: watch::Sender<()>, + admission: PreparationReceipt, + ) -> Result<(), DockerError> { + let Request { + source, + overrides, + deadline, + mut cancelled, + reply, + slot: _, + retained_admission: _, + } = request; + let staged = self + .stage(&source, overrides.as_ref(), deadline, &mut cancelled) + .await; + drop(source); + drop(overrides); + match staged { + Ok((resolved, size)) => { + self.progress.active.store(true, Ordering::Release); + let image = PinnedImage::with_context( + resolved, + size, + release, + self.progress.active.clone(), + self.control.subscribe(), + ImageLeaseContext::Owned { + directory: self.pin.directory().clone(), + preparation: Some(admission), + }, + ); + let delivered = match self.check(deadline, &cancelled) { + Ok(()) => reply.send(Ok(image)).is_ok(), + Err(error) => { + self.progress.active.store(false, Ordering::Release); + drop(image); + let _ = reply.send(Err(error)); + false + } + }; + drop(cancelled); + if !delivered { + self.progress.active.store(false, Ordering::Release); + self.retire().await; + return Ok(()); + } + let mut released = self.progress.released.clone(); + let mut stop = self.control.subscribe(); + tokio::select! {biased;()=stopped(&mut stop)=>{},_ = released.changed()=>{}} + self.retire().await; + } + Err(error) => { + let _ = reply.send(Err(error)); + self.retire().await; + } + } + Ok(()) + } + async fn resolve_source( + &self, + source: &PreparationSource, + overrides: Option<&ImageConfig>, + deadline: Instant, + cancel: &mut watch::Receiver<()>, + ) -> Result { + self.check(deadline, cancel)?; + let architecture = source.architecture(); + let (reference, lookup) = match source { + PreparationSource::Reference { reference, .. } => (reference, reference.clone()), + PreparationSource::Restore(saved) => ( + &saved.metadata().requested_reference, + saved.metadata().image_id.as_str().parse()?, + ), + }; + let resolver = ImageResolver::new(self.docker.clone()); + let mut stop = self.control.subscribe(); + let root = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(&lookup,deadline.min(self.config.deadline()))=>result,}; + let root = match root { + Err(DockerError::Status(StatusCode::NOT_FOUND)) + if matches!(source, PreparationSource::Reference { .. }) => + { + self.policy + .pull( + &self.docker, + reference, + architecture, + cancel.clone(), + self.control.subscribe(), + ) + .await?; + self.check(deadline, cancel)?; + tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(reference,deadline.min(self.config.deadline()))=>result?,} + } + result => result?, + }; + if let PreparationSource::Restore(saved) = source { + if root.id.as_deref() != Some(saved.metadata().image_id.as_str()) { + return Err(ArtifactError::Inspection.into()); + } + } + let actual = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.resolve_captured(reference,architecture,overrides,root,deadline.min(self.config.deadline()))=>result?,}; + self.check(deadline, cancel)?; + match source { + PreparationSource::Restore(saved) => { + if actual.metadata().inspection_hash != saved.metadata().inspection_hash { + return Err(ArtifactError::Inspection.into()); + } + Ok(saved.as_ref().clone()) + } + PreparationSource::Reference { .. } => Ok(actual), + } + } + + async fn stage( + &mut self, + source: &PreparationSource, + overrides: Option<&ImageConfig>, + deadline: Instant, + cancel: &mut watch::Receiver<()>, + ) -> Result<(ResolvedImage, UnpackedImageSize), DockerError> { + let image = self + .resolve_source(source, overrides, deadline, cancel) + .await?; + let resolver = ImageResolver::new(self.docker.clone()); + let mut stop = self.control.subscribe(); + self.check(deadline, cancel)?; + let expected = image.metadata().image_id.clone(); + self.progress + .phase + .send_replace(Phase::Preparing(expected.clone())); + self.pin + .directory() + .prepare_image( + ImageRecord::new(&self.pin, &expected)?, + self.config.deadline(), + ) + .await?; + self.check(deadline, cancel)?; + let mutation = MutationGuard::new(); + self.progress + .phase + .send_replace(Phase::Uncertain(expected.clone(), mutation.receipt())); + let tagged = self + .docker + .tag_image_pin_tracked(&expected, &self.pin, self.config.deadline(), mutation) + .await; + if matches!( + &tagged, + Err(DockerError::Busy + | DockerError::Status(StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND)) + ) { + self.progress + .phase + .send_replace(Phase::NoEffect(expected.clone())); + } + tagged?; + self.persist_commit(&expected).await?; + self.check(deadline, cancel)?; + let actual = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(self.pin.reference(),deadline.min(self.config.deadline()))=>result?,}; + if actual.id.as_deref() != Some(expected.as_str()) { + return Err(DockerError::Quarantined); + } + let size = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.unpacked_size(&image,deadline.min(self.config.deadline()))=>result?,}; + self.check(deadline, cancel)?; + Ok((image, size)) + } + async fn retire(self) { + self.progress.active.store(false, Ordering::Release); + let mut dirty = matches!( + *self.progress.phase.borrow(), + Phase::Uncertain(..) | Phase::Committing(_) + ) + .then(|| Dirty::new(self.health.clone())); + let mut delay = Duration::from_secs(1); + loop { + if self.cleanup_step().await { + self.progress.phase.send_replace(Phase::Clean); + return; + } + if dirty.is_none() { + dirty = Some(Dirty::new(self.health.clone())); + } + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + async fn cleanup_step(&self) -> bool { + let phase = self.progress.phase.borrow().clone(); + let (expected, known) = match phase { + Phase::Unissued | Phase::Clean => return true, + Phase::Preparing(id) | Phase::NoEffect(id) => return self.erase(&id).await, + Phase::Committing(id) => { + if self.persist_commit(&id).await.is_err() { + return false; + } + (id, true) + } + Phase::Known(id) => (id, true), + Phase::Uncertain(id, receipt) => { + if receipt.definitely_unissued() { + return self.erase(&id).await; + } + (id, false) + } + }; + let Ok(_cleanup) = self.cleanup.acquire().await else { + return false; + }; + let inspected = self + .docker + .inspect_image_pin(&self.pin, self.config.deadline()) + .await; + match inspected { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => known && self.erase(&expected).await, + Ok(image) if image.id.as_deref() == Some(expected.as_str()) => { + if self.persist_commit(&expected).await.is_err() { + return false; + } + if !matches!(self.docker.inspect_image_pin(&self.pin, self.config.deadline()).await, + Ok(image) if image.id.as_deref() == Some(expected.as_str())) + { + return false; + } + let _ = self + .docker + .remove_image_pin(&self.pin, self.config.deadline()) + .await; + matches!( + self.docker + .inspect_image_pin(&self.pin, self.config.deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ) && self.erase(&expected).await + } + _ => false, + } + } + async fn persist_commit(&self, expected: &Sha256Digest) -> Result<(), DockerError> { + self.progress + .phase + .send_replace(Phase::Committing(expected.clone())); + self.pin + .directory() + .commit_image( + ImageRecord::new(&self.pin, expected)?, + self.config.deadline(), + ) + .await?; + self.progress + .phase + .send_replace(Phase::Known(expected.clone())); + Ok(()) + } + async fn erase(&self, expected: &Sha256Digest) -> bool { + let Ok(record) = ImageRecord::new(&self.pin, expected) else { + return false; + }; + self.pin + .directory() + .settle_image(record, self.config.deadline()) + .await + .is_ok() + } +} + +#[cfg(test)] +#[path = "lease_tests.rs"] +pub(crate) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/lease_tests.rs b/crates/rustack-lambda-core/src/docker/lease_tests.rs new file mode 100644 index 00000000..d892c05f --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease_tests.rs @@ -0,0 +1,1012 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use serde_json::{Value, json}; +use tokio::{ + sync::Notify, + time::{sleep_until, timeout}, +}; + +use super::*; +use crate::docker::{ + DirectoryLease, DirectoryLeaseError, DockerWire, + tests::{Daemon, IncomingRequest}, +}; + +fn digest(letter: char) -> String { + format!("sha256:{}", letter.to_string().repeat(64)) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn scope() -> ImagePinScope { + ImagePinScope::testing(Uuid::new_v4(), Uuid::new_v4()).unwrap() +} +fn config(pins: u16, jobs: u8) -> ImageLeaseConfig { + ImageLeaseConfig::new(pins, jobs, Duration::from_secs(1)).unwrap() +} +fn metadata(parent: &str, platform: bool) -> Value { + let child = if parent == digest('a') { + digest('c') + } else { + digest('d') + }; + json!({"Id":if platform {&child} else {parent},"Os":"linux","Architecture":"arm64","Size":64, + "Descriptor":{"mediaType":if platform {"application/vnd.oci.image.manifest.v1+json"} else {"application/vnd.oci.image.index.v1+json"}, + "digest":if platform {&child} else {parent},"platform":{"os":"linux","architecture":"arm64"}}, + "Config":{"Entrypoint":["/entry"],"Cmd":[if parent==digest('a') {"old_handler"} else {"new_handler"}],"Env":["SECRET=private_value"],"WorkingDir":"/task","User":"1000"}}) +} +fn manifests(parent: &str, zero: bool) -> Value { + let mut value = metadata(parent, false); + let child = if parent == digest('a') { + digest('c') + } else { + digest('d') + }; + value["Manifests"] = json!([{"ID":child,"Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":child,"platform":{"os":"linux","architecture":"arm64"}}, + "Available":true,"Kind":"image","Size":{"Content":64,"Total":99999}, + "ImageData":{"Containers":[],"Platform":{"os":"linux","architecture":"arm64"},"Size":{"Unpacked":if zero {0} else {4096}}}}]); + value +} + +#[derive(Debug, Default)] +pub(crate) struct Stats { + pub(crate) source_lookups: AtomicUsize, + pub(crate) immutable_lookups: AtomicUsize, + pub(crate) platform_lookups: AtomicUsize, + pub(crate) tags: AtomicUsize, + pub(crate) deletes: AtomicUsize, + pub(crate) absent: AtomicUsize, + pub(crate) aliases: AtomicUsize, + changed: Notify, +} +impl Stats { + pub(crate) async fn until(&self, condition: impl Fn() -> bool) { + timeout(Duration::from_secs(10), async { + loop { + let changed = self.changed.notified(); + if condition() { + break; + } + changed.await; + } + }) + .await + .unwrap(); + } + fn notify(&self) { + self.changed.notify_waiters(); + } +} +#[derive(Debug, Clone, Copy)] +pub(crate) enum Mode { + Normal, + Reject400, + Reject404, + Lost, + PostCommit500, + Late, + Late500, + Never, + Wrong, + WrongRoot, + ZeroSize, + LostDelete, + HoldDelete, +} +pub(crate) enum Command { + Retarget, + Repair, + Commit, + AllowDeletes, +} +struct Pending { + alias: String, + parent: String, + at: Option, + request: Option, +} + +#[derive(TypedBuilder)] +struct Script { + daemon: Daemon, + mode: Mode, + stats: Arc, + commands: mpsc::Receiver, + stop: watch::Receiver, + #[builder(default)] + aliases: HashMap, + #[builder(default)] + expected: HashMap, + #[builder(default)] + pending: Option, + #[builder(default)] + retargeted: bool, + #[builder(default)] + allow_deletes: bool, +} +impl Script { + async fn run(mut self) { + loop { + let at = self.pending.as_ref().and_then(|pending| pending.at); + tokio::select! { + biased; + _ = self.stop.changed()=>break, + () = sleep_until(at.unwrap_or_else(Instant::now)),if at.is_some()=>self.commit(), + command=self.commands.recv()=>match command {Some(Command::Retarget)=>self.retargeted=true,Some(Command::Repair)=>self.aliases=self.expected.clone(),Some(Command::Commit)=>self.commit(),Some(Command::AllowDeletes)=>self.allow_deletes=true,None=>break}, + request=self.daemon.requests.recv()=>match request {Some(request)=>self.request(request).await,None=>break}, + } + self.stats + .aliases + .store(self.aliases.len(), Ordering::Release); + self.stats.notify(); + } + assert_eq!(self.aliases.len(), 0, "fake daemon retains image aliases"); + assert!(self.pending.is_none()); + self.daemon.close().await; + } + fn commit(&mut self) { + if let Some(pending) = self.pending.take() { + self.aliases.insert(pending.alias, pending.parent); + drop(pending.request); + } + } + async fn tag(&mut self, request: IncomingRequest, image: &str, query: &str) { + assert!(image.ends_with("/tag")); + let parent = image.strip_suffix("/tag").unwrap().to_owned(); + let fields: HashMap<_, _> = form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect(); + let alias = format!( + "{}:{}", + fields.get("repo").unwrap(), + fields.get("tag").unwrap() + ); + assert!(alias.starts_with("rustack-internal/lambda-")); + self.stats.tags.fetch_add(1, Ordering::AcqRel); + self.expected.insert(alias.clone(), parent.clone()); + match self.mode { + Mode::Reject400 => request.reply(400, b"{}").await, + Mode::Reject404 => request.reply(404, b"{}").await, + Mode::Late | Mode::Never => { + assert!(self.pending.is_none()); + self.pending = Some(Pending { + alias, + parent, + at: matches!(self.mode, Mode::Late) + .then(|| Instant::now() + Duration::from_secs(2)), + request: Some(request), + }); + } + Mode::Late500 => { + assert!(self.pending.is_none()); + self.pending = Some(Pending { + alias, + parent, + at: Some(Instant::now() + Duration::from_secs(2)), + request: None, + }); + request.reply(500, b"{}").await; + } + _ => { + self.aliases.insert( + alias, + if matches!(self.mode, Mode::Wrong) { + digest('b') + } else { + parent + }, + ); + match self.mode { + Mode::Lost => drop(request), + Mode::PostCommit500 => request.reply(500, b"{}").await, + _ => request.reply(201, b"").await, + } + } + } + } + + async fn request(&mut self, request: IncomingRequest) { + let line = request.head.lines().next().unwrap().to_owned(); + let fields: Vec<_> = line.split_whitespace().collect(); + let method = fields.first().unwrap(); + let target = fields.get(1).unwrap(); + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let path = percent_encoding::percent_decode_str(path) + .decode_utf8() + .unwrap() + .into_owned(); + let image = path.strip_prefix("/v1.53/images/").unwrap(); + if *method == "POST" { + self.tag(request, image, query).await; + } else if *method == "DELETE" { + assert_eq!(query, "force=false&noprune=true"); + assert!(image.starts_with("rustack-internal/lambda-")); + self.stats.deletes.fetch_add(1, Ordering::AcqRel); + if matches!(self.mode, Mode::HoldDelete) && !self.allow_deletes { + request.reply(500, b"{}").await; + return; + } + let removed = self.aliases.remove(image); + assert!(removed.is_some()); + if matches!(self.mode, Mode::LostDelete) { + drop(request); + return; + } + request + .reply( + 200, + &serde_json::to_vec(&json!([{"Untagged":image}])).unwrap(), + ) + .await; + } else { + assert_eq!(*method, "GET"); + let reference = image.strip_suffix("/json").unwrap(); + if reference == "worker:v1" { + self.stats.source_lookups.fetch_add(1, Ordering::AcqRel); + } else if (reference == digest('a') || reference == digest('b')) && query.is_empty() { + self.stats.immutable_lookups.fetch_add(1, Ordering::AcqRel); + } + if !query.is_empty() && query != "manifests=true" { + self.stats.platform_lookups.fetch_add(1, Ordering::AcqRel); + } + let parent = if reference == "worker:v1" { + Some(if self.retargeted { + digest('b') + } else { + digest('a') + }) + } else if let Some(id) = self.aliases.get(reference) { + Some(id.clone()) + } else if reference == digest('a') || reference == digest('b') { + let current = if self.retargeted { + digest('b') + } else { + digest('a') + }; + (current == reference || self.aliases.values().any(|id| id == reference)) + .then(|| reference.to_owned()) + } else { + None + }; + if let Some(parent) = parent { + let parent = if matches!(self.mode, Mode::WrongRoot) + && reference == digest('a') + && query.is_empty() + { + digest('b') + } else { + parent + }; + let value = if query == "manifests=true" { + manifests(&parent, matches!(self.mode, Mode::ZeroSize)) + } else { + metadata(&parent, !query.is_empty()) + }; + request + .reply(200, &serde_json::to_vec(&value).unwrap()) + .await; + } else { + self.stats.absent.fetch_add(1, Ordering::AcqRel); + request.reply(404, b"{}").await; + } + } + } +} + +pub(crate) struct FakePins { + pub(crate) wire: DockerWire, + stop: watch::Sender, + pub(crate) commands: mpsc::Sender, + task: Option>, + pub(crate) stats: Arc, + pub(crate) identity_status: watch::Sender, +} +impl FakePins { + pub(crate) async fn new(mode: Mode) -> Self { + let daemon = Daemon::new(); + let identity_status = daemon.identity_status.clone(); + let wire = daemon.wire().await; + let (stop, stopping) = watch::channel(false); + let (commands, receiver) = mpsc::channel(8); + let stats = Arc::new(Stats::default()); + let script = Script::builder() + .daemon(daemon) + .mode(mode) + .stats(stats.clone()) + .commands(receiver) + .stop(stopping) + .build(); + Self { + wire, + identity_status, + stop, + commands, + task: Some(tokio::spawn(script.run())), + stats, + } + } + async fn saved(&self) -> ResolvedImage { + ImageResolver::new(self.wire.port()) + .resolve_local( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap() + } + pub(crate) fn owner(&self, pins: u16, jobs: u8) -> ImageLeaseManager { + ImageLeaseManager::start(self.wire.port(), scope(), config(pins, jobs)).unwrap() + } + pub(crate) async fn close(mut self) { + self.wire.shutdown().await.unwrap(); + self.stop.send_replace(true); + self.task.take().unwrap().await.unwrap(); + } +} +impl Drop for FakePins { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[tokio::test] +async fn test_should_retain_the_directory_lock_through_abandoned_image_owner_cleanup() { + let fake = FakePins::new(Mode::HoldDelete).await; + let directory = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire(directory.path().into(), deadline()) + .await + .unwrap(); + let identity = lease.owner_id(); + let owner = ImageLeaseManager::start( + fake.wire + .port() + .bind_directory(&lease, deadline()) + .await + .unwrap(), + ImagePinScope::new(lease), + config(2, 1), + ) + .unwrap(); + let image = owner + .port() + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + fake.stats + .until(|| owner.port().status().quarantined > 0) + .await; + drop(owner); + assert!(matches!( + DirectoryLease::acquire(directory.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + timeout(Duration::from_secs(10), async { + loop { + match DirectoryLease::acquire(directory.path().into(), deadline()).await { + Ok(lease) => { + assert_eq!(lease.owner_id(), identity); + break; + } + Err(DirectoryLeaseError::InUse) => { + tokio::time::sleep(Duration::from_millis(5)).await; + } + other => panic!("unexpected lease result: {other:?}"), + } + } + }) + .await + .unwrap(); + assert_eq!(fake.stats.aliases.load(Ordering::Acquire), 0); + fake.close().await; +} + +#[tokio::test] +async fn test_should_pin_old_code_across_tag_retargeting_and_keep_cloned_artifacts_live() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + assert!(image.is_live()); + assert_eq!(image.unpacked_size().get(), 4096); + assert_eq!(port.status().pins, 1); + assert!(!format!("{image:?}").contains("private_value")); + let mut observer = Box::pin(port.wait_empty(deadline())); + let mut context = Context::from_waker(Waker::noop()); + assert!(observer.as_mut().poll(&mut context).is_pending()); + assert_eq!(port.status().pins, 1); + fake.commands.send(Command::Retarget).await.unwrap(); + let newer = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(observer); + assert_eq!(image.resolved().metadata().image_id.as_str(), digest('a')); + assert_eq!(newer.resolved().metadata().image_id.as_str(), digest('b')); + assert_eq!( + image + .resolved() + .metadata() + .execution + .plan(None) + .unwrap() + .command(), + ["old_handler"] + ); + let captured = image.clone(); + drop(image); + drop(newer); + assert!(matches!( + port.wait_empty(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(captured.is_live()); + drop(captured); + port.wait_empty(deadline()).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 2); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 2); + fake.close().await; +} + +#[rstest::rstest] +#[case(Mode::Reject400)] +#[case(Mode::Reject404)] +#[case(Mode::Lost)] +#[case(Mode::PostCommit500)] +#[case(Mode::ZeroSize)] +#[tokio::test] +async fn test_should_reconcile_rejected_and_lost_preparation_outcomes(#[case] mode: Mode) { + let fake = FakePins::new(mode).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(port.status().quarantined, 0); + assert!(port.status().accepting); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!( + fake.stats.deletes.load(Ordering::Acquire), + usize::from(!matches!(mode, Mode::Reject400 | Mode::Reject404)) + ); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_retain_uncertain_tag_through_cancelled_waiter_and_initial_absence( + #[case] restoring: bool, +) { + let fake = FakePins::new(Mode::Late).await; + let expected = fake.saved().await; + let mut owner = fake.owner(1, 1); + let port = owner.port(); + let call = port.clone(); + let caller = tokio::spawn(async move { + if restoring { + call.restore(&expected, None, deadline()).await + } else { + call.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + } + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + fake.stats + .until(|| fake.stats.absent.load(Ordering::Acquire) > 0) + .await; + assert_eq!(port.status().pins, 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_bound_preparation_and_pin_admission_and_retain_shutdown_on_timeout( + #[case] restoring: bool, +) { + let fake = FakePins::new(Mode::Never).await; + let saved = fake.saved().await; + let expected = saved.clone(); + let mut owner = fake.owner(1, 1); + let port = owner.port(); + let call = port.clone(); + let caller = tokio::spawn(async move { + if restoring { + call.restore(&expected, None, deadline()).await + } else { + call.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + } + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(matches!( + port.restore(&saved, None, deadline()).await, + Err(DockerError::Busy) + )); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(owner.task.is_some()); + assert_eq!(port.status().pins, 1); + fake.commands.send(Command::Commit).await.unwrap(); + assert!(caller.await.unwrap().is_err()); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + fake.close().await; +} + +#[tokio::test] +async fn test_should_quarantine_observed_alias_conflicts_until_exclusive_owner_repairs_them() { + let fake = FakePins::new(Mode::Wrong).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + fake.stats.until(|| port.status().quarantined > 0).await; + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + assert!(matches!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await, + Err(DockerError::Quarantined) + )); + fake.commands.send(Command::Repair).await.unwrap(); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(port.status().quarantined, 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_join_after_a_shutdown_waiter_is_dropped_and_cache_root_failure() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + let mut shutdown = Box::pin(owner.shutdown(deadline())); + let mut context = Context::from_waker(Waker::noop()); + assert!(shutdown.as_mut().poll(&mut context).is_pending()); + drop(shutdown); + owner.shutdown(deadline()).await.unwrap(); + assert!(!image.is_live()); + drop(image); + fake.close().await; + + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + owner.task.as_ref().unwrap().abort(); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert!(!image.is_live()); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert_eq!(port.status().pins, 0); + drop(image); + fake.close().await; +} + +#[test] +fn test_should_reject_invalid_pin_scope_and_preparation_configuration() { + assert!(ImagePinScope::testing(Uuid::nil(), Uuid::new_v4()).is_err()); + let pin = scope().allocate().unwrap(); + assert!( + pin.reference() + .as_str() + .starts_with("rustack-internal/lambda-") + ); + for (pins, jobs, io) in [ + (0, 1, Duration::from_secs(1)), + (8193, 1, Duration::from_secs(1)), + (1, 2, Duration::from_secs(1)), + (1, 1, Duration::from_nanos(1)), + (1, 1, Duration::from_secs(61)), + ] { + assert!(ImageLeaseConfig::new(pins, jobs, io).is_err()); + } +} + +#[tokio::test] +async fn test_should_retain_an_unaccepted_prepared_artifact_until_physical_retirement() { + let fake = FakePins::new(Mode::HoldDelete).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let reference = "worker:v1".parse().unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let admission = Arc::new(capacity.clone().acquire_owned().await.unwrap()); + let image = port + .prepare_deployment(&reference, Architecture::Arm64, None, deadline(), admission) + .await + .unwrap(); + assert_eq!(capacity.available_permits(), 0); + assert_eq!(port.status().preparing, 1); + drop(image); + fake.stats + .until(|| fake.stats.deletes.load(Ordering::Acquire) > 0) + .await; + assert_eq!(capacity.available_permits(), 0); + assert_eq!(port.status().preparing, 1); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(capacity.available_permits(), 1); + assert_eq!(port.status().preparing, 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_report_root_failure_even_when_no_pin_has_started() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + owner.task.as_ref().unwrap().abort(); + timeout_at(deadline(), async { + while !owner.task.as_ref().unwrap().is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(matches!( + port.wait_empty(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 0); + fake.close().await; +} + +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_recover_owned_aliases_after_worker_or_off_runtime_root_failure( + #[case] off_runtime: bool, +) { + let fake = FakePins::new(Mode::HoldDelete).await; + let config = config(1, 1); + let pin = scope().allocate().unwrap(); + let resolved = ImageResolver::new(fake.wire.port()) + .resolve_local( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + let intent = ImageRecord::new(&pin, &resolved.metadata().image_id).unwrap(); + pin.directory() + .prepare_image(intent.clone(), deadline()) + .await + .unwrap(); + fake.wire + .port() + .tag_image_pin(&resolved.metadata().image_id, &pin, deadline()) + .await + .unwrap(); + pin.directory() + .commit_image(intent, deadline()) + .await + .unwrap(); + let budget = Arc::new(PinBudget::new(1)); + let permit = budget.acquire().unwrap(); + let (control, stop) = watch::channel(false); + let (release, released) = watch::channel(()); + let (phase, _) = watch::channel(Phase::Known(resolved.metadata().image_id.clone())); + let active = Arc::new(AtomicBool::new(true)); + let progress = Arc::new(Progress { + phase, + released, + active: active.clone(), + }); + let image = PinnedImage::new(resolved, 4096.try_into().unwrap(), release, active, stop); + let (sender, inbox) = mpsc::channel(32); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let id = pin.id; + let capacity = Arc::new(Semaphore::new(1)); + let preparation = Arc::new(Semaphore::new(1)); + let admission = PreparationAdmission::new( + preparation.clone().acquire_owned().await.unwrap(), + Some(Arc::new(capacity.clone().acquire_owned().await.unwrap())), + ); + let mut jobs: JoinSet = JoinSet::new(); + let job = jobs.spawn(async { panic!("controlled image worker failure") }); + let actor = Actor::builder() + .docker(fake.wire.port()) + .config(config) + .inbox(inbox) + .control(control) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(emergency.clone()) + .jobs(jobs) + .records(HashMap::from([( + id, + Record { + pin, + progress, + permit, + _admission: watch::channel(Some(admission)).0, + }, + )])) + .index(HashMap::from([(job.id(), id)])) + .build(); + let joined = if off_runtime { + let thread = std::thread::spawn(move || drop(actor)); + timeout_at( + deadline(), + tokio::task::spawn_blocking(move || thread.join()), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + let emergency = emergency.clone(); + tokio::spawn(async move { emergency.wait().await.map_err(|_| DockerError::Worker) }) + } else { + tokio::spawn(actor.run()) + }; + fake.stats + .until(|| fake.stats.deletes.load(Ordering::Acquire) > 0) + .await; + assert_eq!(capacity.available_permits(), 0); + assert_eq!(preparation.available_permits(), 0); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + let result = timeout_at(deadline(), joined).await.unwrap().unwrap(); + if off_runtime { + assert!(result.is_ok()); + } else { + assert!(matches!(result, Err(DockerError::Worker))); + } + assert_eq!(capacity.available_permits(), 1); + assert_eq!(preparation.available_permits(), 1); + assert!(health.failed.load(Ordering::Acquire)); + assert!(!image.is_live()); + assert_eq!(budget.count.load(Ordering::Acquire), 0); + assert!(fake.stats.deletes.load(Ordering::Acquire) >= 2); + assert_eq!(fake.stats.aliases.load(Ordering::Acquire), 0); + emergency.wait().await.unwrap(); + drop(sender); + drop(image); + fake.close().await; +} + +#[tokio::test] +async fn test_should_retain_a_late_nested_mutation_after_a_complete_http_500() { + let fake = FakePins::new(Mode::Late500).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + fake.stats + .until(|| fake.stats.absent.load(Ordering::Acquire) > 0) + .await; + assert_eq!(port.status().pins, 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_confirm_absence_after_an_acknowledged_tag_and_lost_delete_response() { + let fake = FakePins::new(Mode::LostDelete).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + assert_eq!(port.status().quarantined, 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_retain_cleanup_capacity_and_shutdown_join_until_delete_recovers() { + let fake = FakePins::new(Mode::HoldDelete).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + fake.stats.until(|| port.status().quarantined > 0).await; + assert_eq!(port.status().pins, 1); + assert!(matches!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await, + Err(DockerError::Quarantined) + )); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(owner.task.is_some()); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + assert_eq!(port.status().quarantined, 0); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert!(fake.stats.deletes.load(Ordering::Acquire) >= 2); + fake.close().await; +} + +#[path = "lease_tests/durable.rs"] +mod durable; + +#[tokio::test] +async fn test_should_reject_unqualified_platform_before_pin_or_wire_admission() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut images = ImageLeaseManager::start_qualified( + wire.port(), + scope(), + config(2, 1), + ImagePullPolicy::default(), + Architecture::Arm64, + ) + .unwrap(); + assert!(matches!( + images + .port() + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::X86_64, + None, + deadline() + ) + .await, + Err(DockerError::Configuration) + )); + assert!(matches!( + images + .port() + .restore( + &ResolvedImage::fixture(Architecture::X86_64), + None, + deadline() + ) + .await, + Err(DockerError::Configuration) + )); + assert_eq!(images.port().status().pins, 0); + assert_eq!(images.port().status().preparing, 0); + images.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[path = "lease_tests/restore.rs"] +mod restore; diff --git a/crates/rustack-lambda-core/src/docker/lease_tests/durable.rs b/crates/rustack-lambda-core/src/docker/lease_tests/durable.rs new file mode 100644 index 00000000..b0241fdc --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease_tests/durable.rs @@ -0,0 +1,373 @@ +use std::os::unix::fs::symlink; + +use super::*; + +async fn resolve(daemon: &mut Daemon) { + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&metadata(&digest('a'), false)).unwrap(), + ) + .await; + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&metadata(&digest('a'), true)).unwrap(), + ) + .await; +} +async fn matching(daemon: &mut Daemon) { + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&metadata(&digest('a'), false)).unwrap(), + ) + .await; +} + +#[tokio::test] +async fn test_should_publish_intent_before_tag_and_commit_before_live_lease_or_delete() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut owner = ImageLeaseManager::start( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ImagePinScope::new(directory.clone()), + config(1, 1), + ) + .unwrap(); + let port = owner.port(); + let task = tokio::spawn(async move { + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + resolve(&mut daemon).await; + let tag = daemon.next().await; + assert!(tag.head.starts_with("POST ")); + let records = directory.image_records(deadline()).await.unwrap(); + assert_eq!(records.len(), 1); + assert!(!records[0].committed); + tag.reply(201, b"").await; + let inspected = daemon.next().await; + assert!(directory.image_records(deadline()).await.unwrap()[0].committed); + inspected + .reply( + 200, + &serde_json::to_vec(&metadata(&digest('a'), false)).unwrap(), + ) + .await; + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&manifests(&digest('a'), false)).unwrap(), + ) + .await; + let image = task.await.unwrap().unwrap(); + assert!(image.is_live()); + drop(image); + matching(&mut daemon).await; + matching(&mut daemon).await; + let remove = daemon.next().await; + assert!(remove.head.starts_with("DELETE ")); + assert!(directory.image_records(deadline()).await.unwrap()[0].committed); + remove.reply(200, b"[]").await; + daemon.next().await.reply(404, b"{}").await; + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_retain_pin_capacity_when_no_effect_record_cannot_be_erased() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut owner = ImageLeaseManager::start( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ImagePinScope::new(directory.clone()), + config(1, 1), + ) + .unwrap(); + let port = owner.port(); + let task = tokio::spawn(async move { + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + resolve(&mut daemon).await; + let tag = daemon.next().await; + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + let staging = root.path().join(".rustack-lambda-docker/pins.pending"); + symlink(root.path().join("outside"), &staging).unwrap(); + tag.reply(400, b"{}").await; + assert!(matches!( + task.await.unwrap(), + Err(DockerError::Status(StatusCode::BAD_REQUEST)) + )); + timeout(Duration::from_secs(2), async { + while owner.port().status().quarantined == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(owner.port().status().pins, 1); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + tokio::fs::remove_file(staging).await.unwrap(); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_acknowledgement_through_failed_commit_then_settle_absent_alias() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut owner = ImageLeaseManager::start( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ImagePinScope::new(directory.clone()), + config(1, 1), + ) + .unwrap(); + let port = owner.port(); + let task = tokio::spawn(async move { + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + resolve(&mut daemon).await; + let tag = daemon.next().await; + let staging = root.path().join(".rustack-lambda-docker/pins.pending"); + symlink(root.path().join("outside"), &staging).unwrap(); + tag.reply(201, b"").await; + assert!(matches!( + task.await.unwrap(), + Err(DockerError::Ownership(_) | DockerError::Quarantined) + )); + timeout(Duration::from_secs(2), async { + while owner.port().status().quarantined == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(owner.port().status().pins, 1); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + tokio::fs::remove_file(&staging).await.unwrap(); + let absent = timeout(Duration::from_secs(3), daemon.next()) + .await + .unwrap(); + assert!( + absent.head.starts_with("GET "), + "acknowledged tag was replayed" + ); + assert!( + directory + .image_records(deadline()) + .await + .unwrap() + .first() + .unwrap() + .committed + ); + absent.reply(404, b"{}").await; + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_positive_inspection_through_failed_commit_and_retire_without_retagging() +{ + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let expected: Sha256Digest = digest('a').parse().unwrap(); + directory + .prepare_image(ImageRecord::new(&pin, &expected).unwrap(), deadline()) + .await + .unwrap(); + let mut mutation = MutationGuard::new(); + let receipt = mutation.receipt(); + mutation.issued(); + drop(mutation); + let (_release, released) = watch::channel(()); + let progress = Arc::new(Progress { + phase: watch::channel(Phase::Uncertain(expected, receipt)).0, + released, + active: Arc::new(AtomicBool::new(false)), + }); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let worker = Worker::builder() + .docker( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ) + .config(config(1, 1)) + .pin(pin) + .progress(progress.clone()) + .control(watch::channel(false).0) + .health(Arc::new(Health::default())) + .cleanup(Arc::new(Semaphore::new(2))) + .build(); + let staging = root.path().join(".rustack-lambda-docker/pins.pending"); + symlink(root.path().join("outside"), &staging).unwrap(); + let (clean, ()) = tokio::join!(worker.cleanup_step(), matching(&mut daemon)); + assert!(!clean, "unsafe storage did not block durable commitment"); + assert!(matches!(*progress.phase.borrow(), Phase::Committing(_))); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + tokio::fs::remove_file(staging).await.unwrap(); + tokio::join!(worker.retire(), async { + let absent = daemon.next().await; + assert!( + absent.head.starts_with("GET "), + "matching proof was replaced with a repeated mutation" + ); + assert!( + directory + .image_records(deadline()) + .await + .unwrap() + .first() + .unwrap() + .committed + ); + absent.reply(404, b"{}").await; + }); + assert!(matches!(*progress.phase.borrow(), Phase::Clean)); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_erase_preparing_or_definitely_unissued_intent_without_daemon_mutation( + #[case] receipt: bool, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let expected: Sha256Digest = digest('a').parse().unwrap(); + directory + .prepare_image(ImageRecord::new(&pin, &expected).unwrap(), deadline()) + .await + .unwrap(); + let mutation = MutationGuard::new(); + let unissued = mutation.receipt(); + drop(mutation); + let phase = if receipt { + Phase::Uncertain(expected, unissued) + } else { + Phase::Preparing(expected) + }; + let (_release, released) = watch::channel(()); + let progress = Arc::new(Progress { + phase: watch::channel(phase).0, + released, + active: Arc::new(AtomicBool::new(false)), + }); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + Worker::builder() + .docker( + wire.port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + ) + .config(config(1, 1)) + .pin(pin) + .progress(progress.clone()) + .control(watch::channel(false).0) + .health(Arc::new(Health::default())) + .cleanup(Arc::new(Semaphore::new(2))) + .build() + .retire() + .await; + assert!(matches!(*progress.phase.borrow(), Phase::Clean)); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/lease_tests/restore.rs b/crates/rustack-lambda-core/src/docker/lease_tests/restore.rs new file mode 100644 index 00000000..9f752d72 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease_tests/restore.rs @@ -0,0 +1,142 @@ +//! Immutable-content restoration must never become ordinary tag preparation. + +use super::*; +use crate::artifact::{ImageExecutionMetadata, ImageMetadata}; + +#[tokio::test] +async fn test_should_restore_saved_metadata_without_following_a_retargeted_source() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let original = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + let serialized = serde_json::to_vec(original.resolved()).unwrap(); + let saved: ResolvedImage = serde_json::from_slice(&serialized).unwrap(); + fake.commands.send(Command::Retarget).await.unwrap(); + let restored = port.restore(&saved, None, deadline()).await.unwrap(); + drop(original); + assert_eq!(serde_json::to_vec(restored.resolved()).unwrap(), serialized); + assert_eq!( + restored.resolved().metadata().image_id.as_str(), + digest('a') + ); + assert_eq!( + restored.resolved().code_sha256().unwrap(), + saved.code_sha256().unwrap() + ); + assert_eq!(restored.unpacked_size().get(), 4096); + assert!(restored.is_live()); + assert_eq!(fake.stats.source_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 2); + assert!(fake.stats.immutable_lookups.load(Ordering::Acquire) > 0); + drop(restored); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 2); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_missing_saved_content_without_source_fallback_or_mutation() { + let fake = FakePins::new(Mode::Normal).await; + let saved = fake.saved().await; + let mut owner = fake.owner(1, 1); + let port = owner.port(); + fake.commands.send(Command::Retarget).await.unwrap(); + assert!(matches!( + port.restore(&saved, None, deadline()).await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + )); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.source_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.immutable_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 0); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + // A newer source is positively available, but cannot replace saved A. + let newer = fake.saved().await; + assert_eq!(newer.metadata().image_id.as_str(), digest('b')); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[rstest::rstest] +#[case::size(0)] +#[case::defaults(1)] +#[case::manifest(2)] +#[case::architecture(3)] +#[tokio::test] +async fn test_should_compare_valid_saved_fingerprints_with_actual_metadata_before_tag( + #[case] field: u8, +) { + let fake = FakePins::new(Mode::Normal).await; + let saved = fake.saved().await; + let mut changed: ImageMetadata = saved.clone().into(); + match field { + 0 => changed.size_bytes = 65, + 1 => { + changed.execution = ImageExecutionMetadata::builder() + .entrypoint(vec!["/entry".into()]) + .command(vec!["old_handler".into()]) + .working_directory("/task".into()) + .environment(vec!["SECRET=private_value".into()]) + .user("1001".into()) + .build() + .try_into() + .unwrap(); + } + 2 => changed.selected_manifest = Some(digest('e').parse().unwrap()), + 3 => changed.architecture = Architecture::X86_64, + _ => unreachable!("fixed case table"), + } + changed.inspection_hash = ImageMetadata::execution_fingerprint( + &changed.image_id, + changed.selected_manifest.as_ref(), + changed.architecture, + changed.size_bytes, + &changed.execution, + ) + .unwrap(); + let changed: ResolvedImage = changed.try_into().unwrap(); + let mut owner = fake.owner(1, 1); + let port = owner.port(); + assert!(matches!( + port.restore(&changed, None, deadline()).await, + Err(DockerError::Image(ArtifactError::Inspection)) + )); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 0); + assert_eq!(fake.stats.source_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.immutable_lookups.load(Ordering::Acquire), 1); + let image = port.restore(&saved, None, deadline()).await.unwrap(); + assert!(image.is_live()); + drop(image); + port.wait_empty(deadline()).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_a_different_root_id_before_platform_lookup_or_tag() { + let fake = FakePins::new(Mode::WrongRoot).await; + let saved = fake.saved().await; + let mut owner = fake.owner(1, 1); + let port = owner.port(); + assert!(matches!( + port.restore(&saved, None, deadline()).await, + Err(DockerError::Image(ArtifactError::Inspection)) + )); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.source_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.immutable_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.platform_lookups.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/logs.rs b/crates/rustack-lambda-core/src/docker/logs.rs new file mode 100644 index 00000000..231f53b2 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs.rs @@ -0,0 +1,530 @@ +//! Independent function output capture; socket reads never wait for a consumer. + +mod window; + +use std::{collections::VecDeque, fmt, time::Duration}; + +use bytes::Bytes; +use rustack_lambda_runtime::{diagnostics::runtime_dispatch, protocol::AttemptId}; +use tokio::{ + io::AsyncRead, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; +pub use window::LogWindow; +use window::{ActiveWindow, PendingReceipt, ReceiptGuard, WindowKey, changed}; + +use super::{ + DockerAttachment, DockerError, DockerOutputReader, OutputChannel, OutputChunk, check_deadline, +}; + +const TAIL: usize = 4096; +const SILENCE: Duration = Duration::from_secs(3600); + +/// Bounded binary API output and capture status; Debug never prints user bytes. +#[derive(Clone)] +#[non_exhaustive] +pub struct LogSnapshot { + /// The final 4 KiB in observed stdout/stderr chunk order, without UTF-8 conversion. + pub tail: Bytes, + /// Old stdout bytes overwritten in its independent ring. + pub stdout_discarded: u64, + /// Old stderr bytes overwritten in its independent ring. + pub stderr_discarded: u64, + /// The attachment reached EOF; container exit remains independently owned. + pub closed: bool, + /// A retained framing/IO/deadline failure; joined shutdown preserves it. + pub failure: Option, +} +impl fmt::Debug for LogSnapshot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("LogSnapshot") + .field("bytes", &self.tail.len()) + .field("stdout_discarded", &self.stdout_discarded) + .field("stderr_discarded", &self.stderr_discarded) + .field("closed", &self.closed) + .field("failure", &self.failure) + .finish() + } +} + +enum Action { + Snapshot, + Begin(ActiveWindow, PendingReceipt), + Finish(WindowKey, PendingReceipt), +} +struct Request { + action: Action, + deadline: Instant, + reply: oneshot::Sender>, +} + +/// Retained log actor; its environment owns the independent container lifetime. +#[non_exhaustive] +pub struct FunctionLogs { + requests: mpsc::Sender, + stop: watch::Sender, + task: Option>>, + settled: Option>, + window_mutating: bool, +} +impl fmt::Debug for FunctionLogs { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FunctionLogs") + .field("stopping", &*self.stop.borrow()) + .field("settled", &self.settled) + .field("window_mutation_uncertain", &self.window_mutating) + .finish_non_exhaustive() + } +} +impl FunctionLogs { + /// Start output capture with a4KiB..1MiB ring for each channel. + /// + /// # Errors + /// Rejects invalid ring/record budgets, unavailable runtime or allocation. + /// + /// ```no_run + /// # fn start(attachment: rustack_lambda_core::docker::DockerAttachment) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let logs = rustack_lambda_core::docker::FunctionLogs::start( + /// attachment, std::time::Duration::from_secs(5), 262_144)?; + /// # Ok(()) } + /// ``` + pub fn start( + attachment: DockerAttachment, + io_budget: Duration, + ring_bytes: usize, + ) -> Result { + let reader = DockerOutputReader::new(attachment.0, io_budget)?; + Self::start_reader(reader, ring_bytes) + } + + pub(super) fn start_reader( + reader: DockerOutputReader, + ring_bytes: usize, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| DockerError::Worker)?; + if !(TAIL..=1024 * 1024).contains(&ring_bytes) { + return Err(DockerError::Configuration); + } + let state = Capture::new(ring_bytes)?; + let (requests, received) = mpsc::channel(8); + let (stop, stopped) = watch::channel(false); + let task = runtime + .spawn(capture(reader, state, received, stopped).with_subscriber(runtime_dispatch())); + Ok(Self { + requests, + stop, + task: Some(task), + settled: None, + window_mutating: false, + }) + } + + /// Read a bounded tail/status without interrupting a partial record. + /// + /// # Errors + /// Returns expired/invalid budget, full mailbox, stopped actor or worker failure. + /// + /// ```no_run + /// # async fn snapshot(logs: &rustack_lambda_core::docker::FunctionLogs) { + /// let snapshot = logs.snapshot(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn snapshot(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + if let Some(result) = &self.settled { + return result.clone(); + } + self.request(Action::Snapshot, deadline).await + } + + /// Open an empty invocation window before sending its runtime event. + /// + /// # Errors + /// Returns expired/invalid budget, overload, overlapping/uncertain mutation, + /// stopped/closed capture or retained IO failure. An accepted mutation's + /// failure or cancellation forbids further window mutations; retire the owner. + /// + /// ```no_run + /// # async fn begin(logs: &mut rustack_lambda_core::docker::FunctionLogs) { + /// let window = logs.begin(Default::default(), tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn begin( + &mut self, + attempt: AttemptId, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + if self.window_mutating { + return Err(DockerError::Stopped); + } + self.window_mutating = true; + let (window, active) = LogWindow::new(attempt); + let (mut receipt, pending) = ReceiptGuard::new(deadline); + self.request(Action::Begin(active, pending), deadline) + .await?; + receipt.consume(deadline)?; + self.window_mutating = false; + Ok(window) + } + + /// Close exactly that window and return its bounded observed tail/status. + /// + /// # Errors + /// Returns budget/admission failure, foreign/stale capability, uncertain + /// mutation or retained capture failure. Caller cancellation requires owner + /// retirement. Environment diagnostic discard counters remain cumulative. + /// + /// ```no_run + /// # async fn finish(logs: &mut rustack_lambda_core::docker::FunctionLogs, window: rustack_lambda_core::docker::LogWindow) { + /// let tail = logs.finish(window, tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn finish( + &mut self, + mut window: LogWindow, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + if self.window_mutating { + return Err(DockerError::Stopped); + } + self.window_mutating = true; + let (mut receipt, pending) = ReceiptGuard::new(deadline); + let snapshot = self + .request(Action::Finish(window.key, pending), deadline) + .await?; + receipt.consume(deadline)?; + window.complete(); + self.window_mutating = false; + Ok(snapshot) + } + + async fn request(&self, action: Action, deadline: Instant) -> Result { + check_deadline(deadline)?; + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + let (reply, response) = oneshot::channel(); + self.requests + .try_send(Request { + action, + deadline, + reply, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Worker, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + + /// Stop and join socket ownership, preserving failure across cancelled waiters. + /// + /// # Errors + /// Returns a retained parser/IO/deadline failure or failed actor join. + /// Container cleanup must be awaited separately by the environment owner. + /// + /// ```no_run + /// # async fn stop(logs: &mut rustack_lambda_core::docker::FunctionLogs) { + /// let final_tail = logs.shutdown().await; + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result { + self.stop.send_replace(true); + if let Some(result) = &self.settled { + return result.clone(); + } + let result = match &mut self.task { + Some(task) => task + .await + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Err(DockerError::Worker), + }; + self.task.take(); + self.settled = Some(result.clone()); + result + } +} +impl Drop for FunctionLogs { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +struct Ring { + bytes: VecDeque, + capacity: usize, + discarded: u64, +} +impl Ring { + fn new(capacity: usize) -> Result { + let mut bytes = VecDeque::new(); + bytes + .try_reserve_exact(capacity) + .map_err(|_| DockerError::Allocation)?; + Ok(Self { + bytes, + capacity, + discarded: 0, + }) + } + fn push(&mut self, bytes: &[u8]) -> Result<(), DockerError> { + let skipped = self + .bytes + .len() + .saturating_add(bytes.len()) + .saturating_sub(self.capacity); + self.discarded = self + .discarded + .saturating_add(u64::try_from(skipped).map_err(|_| DockerError::TooLarge)?); + let remove = skipped.min(self.bytes.len()); + self.bytes.drain(..remove); + let offset = bytes.len().saturating_sub(self.capacity); + self.bytes + .extend(bytes.get(offset..).ok_or(DockerError::Protocol)?); + Ok(()) + } +} +struct Capture { + stdout: Ring, + stderr: Ring, + tail: Ring, + window_tail: Ring, + active: Option, + receipt: Option, + closed: bool, + failure: Option, +} +impl Capture { + fn new(capacity: usize) -> Result { + Ok(Self { + stdout: Ring::new(capacity)?, + stderr: Ring::new(capacity)?, + tail: Ring::new(TAIL)?, + window_tail: Ring::new(TAIL)?, + active: None, + receipt: None, + closed: false, + failure: None, + }) + } + fn push(&mut self, chunk: &OutputChunk) -> Result<(), DockerError> { + match chunk.channel { + OutputChannel::Stdout => self.stdout.push(&chunk.bytes)?, + OutputChannel::Stderr => self.stderr.push(&chunk.bytes)?, + } + self.tail.push(&chunk.bytes)?; + if self.active.is_some() { + self.window_tail.push(&chunk.bytes)?; + } + Ok(()) + } + fn snapshot(&self) -> LogSnapshot { + self.snapshot_of(&self.tail) + } + fn window_snapshot(&self) -> LogSnapshot { + self.snapshot_of(&self.window_tail) + } + fn snapshot_of(&self, ring: &Ring) -> LogSnapshot { + LogSnapshot { + tail: Bytes::from(ring.bytes.iter().copied().collect::>()), + stdout_discarded: self.stdout.discarded, + stderr_discarded: self.stderr.discarded, + closed: self.closed, + failure: self.failure.clone(), + } + } + fn fail_window(&mut self, error: DockerError) { + self.active = None; + self.receipt = None; + if self.failure.is_none() { + self.failure = Some(error); + } + } + fn reconcile(&mut self) -> bool { + if let Some(receipt) = &self.receipt { + match receipt.consumed() { + Ok(true) => { + self.receipt = None; + } + Ok(false) => {} + Err(error) => { + self.fail_window(error); + return true; + } + } + } + if self + .active + .as_ref() + .is_some_and(|window| *window.revoked.borrow() || window.revoked.has_changed().is_err()) + { + self.fail_window(DockerError::Protocol); + true + } else { + false + } + } + fn revocation(&self) -> Option> { + self.active.as_ref().map(|window| window.revoked.clone()) + } + fn reply(&mut self, request: Request) { + self.reconcile(); + if request.reply.is_closed() { + return; + } + let mut committed = false; + let result = if Instant::now() >= request.deadline { + Err(DockerError::Deadline) + } else { + match request.action { + Action::Snapshot => Ok(self.snapshot()), + _ if self.failure.is_some() => { + Err(self.failure.clone().unwrap_or(DockerError::Worker)) + } + _ if self.receipt.is_some() => Err(DockerError::Busy), + Action::Begin(_, _) if self.closed => Err(DockerError::Stopped), + Action::Begin(_, _) if self.active.is_some() => Err(DockerError::Busy), + Action::Begin(window, receipt) => { + self.window_tail.bytes.clear(); + self.window_tail.discarded = 0; + self.active = Some(window); + self.receipt = Some(receipt); + committed = true; + Ok(self.window_snapshot()) + } + Action::Finish(key, receipt) + if self.active.as_ref().map(|window| window.key) == Some(key) => + { + self.active = None; + self.receipt = Some(receipt); + committed = true; + Ok(self.window_snapshot()) + } + Action::Finish(_, _) => Err(DockerError::Protocol), + } + }; + if request.reply.send(result).is_err() && committed { + self.fail_window(DockerError::Protocol); + } + } +} + +async fn capture( + reader: DockerOutputReader, + mut state: Capture, + mut requests: mpsc::Receiver, + mut stop: watch::Receiver, +) -> Result { + let mut reader = Some(reader); + let mut quantum = 0_u8; + loop { + if *stop.borrow() { + break; + } + if state.reconcile() { + reader.take(); + } + if state.closed || state.failure.is_some() { + let receipt = state.receipt.clone(); + let revocation = state.revocation(); + tokio::select! { + biased; + () = super::connection::stopped(&mut stop) => break, + () = changed(receipt, revocation) => { state.reconcile(); }, + request = requests.recv() => match request { Some(request) => state.reply(request), None => break }, + } + continue; + } + let result = match &mut reader { + Some(reader) => read_next(reader, &mut state, &mut requests, &mut stop).await, + None => return Err(DockerError::Worker), + }; + match result { + None => break, + Some(Ok(None)) => state.closed = true, + Some(Err(error)) => state.failure = Some(error), + Some(Ok(Some(chunk))) => { + if let Err(error) = state.push(&chunk) { + state.failure = Some(error); + } + quantum = quantum.saturating_add(1); + if quantum == 64 { + tokio::task::yield_now().await; + quantum = 0; + } + } + } + if state.closed || state.failure.is_some() { + reader.take(); + } + } + drop(reader); + requests.close(); + while let Ok(request) = requests.try_recv() { + if matches!(request.action, Action::Snapshot) { + state.reply(request); + } else { + let _ = request.reply.send(Err(DockerError::Stopped)); + } + } + state.reconcile(); + match state.failure.clone() { + Some(error) => Err(error), + None => Ok(state.snapshot()), + } +} + +async fn read_next( + reader: &mut DockerOutputReader, + state: &mut Capture, + requests: &mut mpsc::Receiver, + stop: &mut watch::Receiver, +) -> Option, DockerError>> { + // Requests retain this read; only stop or terminal capture failure drops it. + let next = reader.next(Instant::now() + SILENCE); + tokio::pin!(next); + loop { + if *stop.borrow() { + return None; + } + state.reconcile(); + if let Some(error) = &state.failure { + return Some(Err(error.clone())); + } + if let Ok(request) = requests.try_recv() { + state.reply(request); + } + if let Some(error) = &state.failure { + return Some(Err(error.clone())); + } + let receipt = state.receipt.clone(); + let revocation = state.revocation(); + tokio::select! { + biased; + () = super::connection::stopped(stop) => return None, + () = changed(receipt, revocation) => { state.reconcile(); }, + result = &mut next => return Some(result), + request = requests.recv() => match request { Some(request) => state.reply(request), None => return None }, + } + } +} + +#[cfg(test)] +#[path = "logs_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/logs/window.rs b/crates/rustack-lambda-core/src/docker/logs/window.rs new file mode 100644 index 00000000..856336a5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs/window.rs @@ -0,0 +1,130 @@ +//! Window revocation and mutation consumption receipts; neither Drop claims IO joins. + +use std::future::pending; + +use rustack_lambda_runtime::protocol::AttemptId; +use tokio::{ + sync::watch, + time::{Instant, sleep_until}, +}; +use uuid::Uuid; + +use super::DockerError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct WindowKey { + id: Uuid, + attempt: AttemptId, +} +impl WindowKey { + pub(super) fn new(attempt: AttemptId) -> Self { + Self { + id: Uuid::new_v4(), + attempt, + } + } +} + +/// Opaque ownership of one invocation's observed log window; consume with finish. +#[derive(Debug)] +#[non_exhaustive] +pub struct LogWindow { + pub(super) key: WindowKey, + revoke: watch::Sender, + complete: bool, +} +impl LogWindow { + pub(super) fn new(attempt: AttemptId) -> (Self, ActiveWindow) { + let key = WindowKey::new(attempt); + let (revoke, revoked) = watch::channel(false); + ( + Self { + key, + revoke, + complete: false, + }, + ActiveWindow { key, revoked }, + ) + } + pub(super) fn complete(&mut self) { + self.complete = true; + } +} +impl Drop for LogWindow { + fn drop(&mut self) { + if !self.complete { + self.revoke.send_replace(true); + } + } +} +pub(super) struct ActiveWindow { + pub(super) key: WindowKey, + pub(super) revoked: watch::Receiver, +} + +#[derive(Clone, Copy)] +enum ReceiptState { + Pending, + Consumed(Instant), + Revoked, +} +pub(super) struct ReceiptGuard { + sender: watch::Sender, + complete: bool, +} +#[derive(Clone)] +pub(super) struct PendingReceipt { + receiver: watch::Receiver, + deadline: Instant, +} +impl ReceiptGuard { + pub(super) fn new(deadline: Instant) -> (Self, PendingReceipt) { + let (sender, receiver) = watch::channel(ReceiptState::Pending); + ( + Self { + sender, + complete: false, + }, + PendingReceipt { receiver, deadline }, + ) + } + pub(super) fn consume(&mut self, deadline: Instant) -> Result<(), DockerError> { + let consumed = Instant::now(); + if consumed >= deadline { + return Err(DockerError::Deadline); + } + self.sender.send_replace(ReceiptState::Consumed(consumed)); + self.complete = true; + Ok(()) + } +} +impl Drop for ReceiptGuard { + fn drop(&mut self) { + if !self.complete { + self.sender.send_replace(ReceiptState::Revoked); + } + } +} +impl PendingReceipt { + pub(super) fn consumed(&self) -> Result { + match *self.receiver.borrow() { + ReceiptState::Consumed(at) if at < self.deadline => Ok(true), + ReceiptState::Consumed(_) => Err(DockerError::Deadline), + ReceiptState::Revoked => Err(DockerError::Protocol), + ReceiptState::Pending if Instant::now() >= self.deadline => Err(DockerError::Deadline), + ReceiptState::Pending if self.receiver.has_changed().is_err() => { + Err(DockerError::Protocol) + } + ReceiptState::Pending => Ok(false), + } + } +} +pub(super) async fn changed( + mut receipt: Option, + mut revocation: Option>, +) { + tokio::select! { biased; + ()=async {match &mut revocation { Some(receiver)=>{let _=receiver.wait_for(|revoked|*revoked).await;},None=>pending().await,}}=>{}, + ()=async {match &mut receipt {Some(receipt)=>tokio::select! {biased;_=receipt.receiver.changed()=>{},()=sleep_until(receipt.deadline)=>{}},None=>pending().await,}}=>{}, + } +} diff --git a/crates/rustack-lambda-core/src/docker/logs_tests.rs b/crates/rustack-lambda-core/src/docker/logs_tests.rs new file mode 100644 index 00000000..e33568e9 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs_tests.rs @@ -0,0 +1,802 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll, Waker}, +}; + +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, DuplexStream, ReadBuf, duplex}, + time::{advance, timeout}, +}; + +use super::*; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} +fn logs(reader: DuplexStream) -> FunctionLogs { + FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + TAIL, + ) + .unwrap() +} +fn record(channel: u8, bytes: &[u8]) -> Vec { + let mut frame = vec![channel, 0, 0, 0]; + frame.extend_from_slice(&u32::try_from(bytes.len()).unwrap().to_be_bytes()); + frame.extend_from_slice(bytes); + frame +} +async fn closed(logs: &FunctionLogs) -> LogSnapshot { + timeout(Duration::from_secs(5), async { + loop { + let snapshot = logs.snapshot(deadline()).await.unwrap(); + if snapshot.closed || snapshot.failure.is_some() { + break snapshot; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap() +} + +#[tokio::test] +async fn test_should_capture_binary_channels_in_observed_order_and_retain_tail_after_eof() { + let (mut writer, reader) = duplex(1024); + let mut logs = logs(reader); + writer.write_all(&record(1, b"stdout\0\xff")).await.unwrap(); + writer.write_all(&record(2, b"stderr\n")).await.unwrap(); + writer.shutdown().await.unwrap(); + let snapshot = closed(&logs).await; + assert_eq!(snapshot.tail.as_ref(), b"stdout\0\xffstderr\n"); + assert_eq!(snapshot.stdout_discarded, 0); + assert_eq!(snapshot.stderr_discarded, 0); + assert!(snapshot.failure.is_none()); + assert!(!format!("{snapshot:?} {logs:?}").contains("stdout\\0")); + assert!(!format!("{snapshot:?}").contains("stderr\\n")); + assert_eq!(logs.snapshot(deadline()).await.unwrap().tail, snapshot.tail); + assert_eq!(logs.shutdown().await.unwrap().tail, snapshot.tail); + assert_eq!(logs.shutdown().await.unwrap().tail, snapshot.tail); +} + +#[tokio::test] +async fn test_should_bound_independent_rings_and_saturate_discard_counts() { + let mut capture = Capture::new(TAIL).unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stdout, + bytes: Bytes::from(vec![1; TAIL + 17]), + }) + .unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stderr, + bytes: Bytes::from(vec![2; TAIL + 9]), + }) + .unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stdout, + bytes: Bytes::from_static(b"final"), + }) + .unwrap(); + assert_eq!(capture.stdout.bytes.len(), TAIL); + assert_eq!(capture.stderr.bytes.len(), TAIL); + assert_eq!(capture.stdout.discarded, 22); + assert_eq!(capture.stderr.discarded, 9); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.tail.len(), TAIL); + assert!( + snapshot + .tail + .get(..TAIL - 5) + .unwrap() + .iter() + .all(|byte| *byte == 2) + ); + assert_eq!(snapshot.tail.get(TAIL - 5..), Some(b"final".as_slice())); + capture.stdout.discarded = u64::MAX - 1; + capture.stdout.push(b"more").unwrap(); + assert_eq!(capture.stdout.discarded, u64::MAX); +} + +#[rstest::rstest] +#[case(0)] +#[case(TAIL - 1)] +#[case(1024 * 1024 + 1)] +#[tokio::test] +async fn test_should_reject_invalid_log_capacity(#[case] capacity: usize) { + let (_, reader) = duplex(1); + assert!(matches!( + FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + capacity + ), + Err(DockerError::Configuration) + )); +} + +#[tokio::test] +async fn test_should_keep_a_fragmented_record_alive_through_cancelled_and_completed_snapshots() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + let frame = record(1, b"secret-frame"); + writer.write_all(frame.get(..4).unwrap()).await.unwrap(); + for _ in 0..16 { + assert!(logs.snapshot(deadline()).await.unwrap().tail.is_empty()); + } + { + let snapshot = logs.snapshot(deadline()); + tokio::pin!(snapshot); + assert!( + snapshot + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + } + writer.write_all(frame.get(4..).unwrap()).await.unwrap(); + writer.shutdown().await.unwrap(); + assert_eq!(closed(&logs).await.tail.as_ref(), b"secret-frame"); + assert!(logs.shutdown().await.is_ok()); +} + +#[tokio::test(start_paused = true)] +async fn test_should_not_extend_the_partial_record_deadline_with_snapshot_traffic() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + writer.write_all(&[1, 0, 0, 0]).await.unwrap(); + for _ in 0..12 { + let _ = logs.snapshot(deadline()).await.unwrap(); + advance(Duration::from_millis(100)).await; + } + let failed = logs.snapshot(deadline()).await.unwrap(); + assert!(matches!(failed.failure, Some(DockerError::Deadline))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_first_byte_silence_independently_of_snapshot_requests() { + let (_writer, reader) = duplex(1); + let mut logs = logs(reader); + logs.snapshot(deadline()).await.unwrap(); + advance(SILENCE + Duration::from_secs(1)).await; + assert!(matches!( + logs.snapshot(deadline()).await.unwrap().failure, + Some(DockerError::Deadline) + )); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); +} + +#[tokio::test] +async fn test_should_retain_shutdown_ownership_after_its_waiter_is_cancelled() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + { + let shutdown = logs.shutdown(); + tokio::pin!(shutdown); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + } + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); + assert!(writer.write_all(b"x").await.is_err()); +} + +#[tokio::test] +async fn test_should_signal_quiet_socket_cleanup_on_drop_outside_an_entered_runtime() { + let (mut writer, reader) = duplex(1); + let logs = logs(reader); + let thread = std::thread::spawn(move || { + assert!(Handle::try_current().is_err()); + drop(logs); + }); + let task = tokio::task::spawn_blocking(move || thread.join().unwrap()); + timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn test_should_serve_snapshots_and_stop_while_output_floods() { + let (mut writer, reader) = duplex(64 * 1024); + let mut logs = logs(reader); + let producer = tokio::spawn(async move { + let frame = record(1, &vec![b'x'; 16 * 1024]); + while writer.write_all(&frame).await.is_ok() { + tokio::task::yield_now().await; + } + }); + timeout(Duration::from_secs(5), async { + loop { + let snapshot = logs.snapshot(deadline()).await.unwrap(); + assert!(snapshot.tail.len() <= TAIL); + if snapshot.stdout_discarded > 64 * 1024 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); + timeout(Duration::from_secs(1), producer) + .await + .unwrap() + .unwrap(); +} + +#[rstest::rstest] +#[case::bad_channel(vec![3, 0, 0, 0, 0, 0, 0, 0])] +#[case::reserved(vec![1, 1, 0, 0, 0, 0, 0, 0])] +#[case::truncated_header(vec![1, 0])] +#[case::truncated_payload(vec![1, 0, 0, 0, 0, 0, 0, 2, 1])] +#[tokio::test] +async fn test_should_preserve_malformed_stream_failure_through_shutdown(#[case] frame: Vec) { + let (mut writer, reader) = duplex(64); + let mut logs = logs(reader); + writer.write_all(&frame).await.unwrap(); + writer.shutdown().await.unwrap(); + assert!(closed(&logs).await.failure.is_some()); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} + +struct Panics; +impl AsyncRead for Panics { + fn poll_read( + self: Pin<&mut Self>, + _: &mut Context<'_>, + _: &mut ReadBuf<'_>, + ) -> Poll> { + panic!("log worker failure") + } +} +#[tokio::test] +async fn test_should_cache_log_actor_join_failure() { + let mut logs = FunctionLogs::start_reader( + DockerOutputReader::new(Panics, Duration::from_secs(1)).unwrap(), + TAIL, + ) + .unwrap(); + while !logs.task.as_ref().unwrap().is_finished() { + tokio::task::yield_now().await; + } + assert!(matches!( + logs.snapshot(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!(logs.shutdown().await, Err(DockerError::Worker))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Worker))); +} + +#[tokio::test] +async fn test_should_bound_snapshot_admission_and_discard_abandoned_requests() { + let (_writer, reader) = duplex(1); + let mut logs = logs(reader); + { + let mut requests = Vec::new(); + for _ in 0..8 { + let mut request = Box::pin(logs.snapshot(deadline())); + assert!( + request + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + requests.push(request); + } + assert!(matches!( + logs.snapshot(deadline()).await, + Err(DockerError::Busy) + )); + drop(requests); + } + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); +} + +#[tokio::test] +async fn test_should_use_output_only_attach_without_replay_and_transfer_driver_ownership() { + let mut daemon = super::super::tests::Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let attach = tokio::spawn(async move { + port.attach_logs(&"function".parse().unwrap(), deadline()) + .await + }); + let request = daemon.next().await; + assert!(request.head.starts_with( + "POST /v1.53/containers/function/attach?stdin=false&stdout=true&stderr=true&stream=true&\ + logs=false HTTP/1.1\r\n" + )); + let mut reply = + b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n".to_vec(); + reply.extend_from_slice(&record(2, b"wire-secret-output")); + request.raw(&reply).await; + let mut logs = + FunctionLogs::start(attach.await.unwrap().unwrap(), Duration::from_secs(1), TAIL).unwrap(); + wire.shutdown().await.unwrap(); + assert_eq!(closed(&logs).await.tail.as_ref(), b"wire-secret-output"); + assert!(logs.shutdown().await.is_ok()); + daemon.close().await; +} + +async fn observed(logs: &FunctionLogs, suffix: &[u8]) { + timeout(Duration::from_secs(3), async { + while !logs + .snapshot(deadline()) + .await + .unwrap() + .tail + .ends_with(suffix) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn probe(sender: &mpsc::Sender) -> LogSnapshot { + let (reply, response) = oneshot::channel(); + sender + .try_send(Request { + action: Action::Snapshot, + deadline: deadline(), + reply, + }) + .unwrap_or_else(|_| panic!("test probe admission")); + timeout(Duration::from_secs(3), response) + .await + .unwrap() + .unwrap() + .unwrap() +} +#[tokio::test] +async fn test_should_keep_diagnostics_separate_and_never_seed_a_silent_invocation_with_prior_logs() +{ + let (mut writer, reader) = duplex(8192); + let mut logs = logs(reader); + writer.write_all(&record(1, b"idle-before")).await.unwrap(); + observed(&logs, b"idle-before").await; + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&record(1, b"first\0\xff")).await.unwrap(); + writer.write_all(&record(2, b"stderr")).await.unwrap(); + observed(&logs, b"stderr").await; + let first = logs.finish(first, deadline()).await.unwrap(); + assert_eq!(first.tail.as_ref(), b"first\0\xffstderr"); + writer.write_all(&record(2, b"idle-after")).await.unwrap(); + observed(&logs, b"idle-after").await; + let silent = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!( + logs.finish(silent, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + let third = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&record(1, b"third")).await.unwrap(); + observed(&logs, b"third").await; + assert_eq!( + logs.finish(third, deadline()).await.unwrap().tail.as_ref(), + b"third" + ); + assert!( + logs.snapshot(deadline()) + .await + .unwrap() + .tail + .starts_with(b"idle-before") + ); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_bound_binary_window_tail_and_allow_matching_finish_after_clean_eof() { + let (mut writer, reader) = duplex(16384); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer + .write_all(&record(1, &vec![0xff; TAIL + 17])) + .await + .unwrap(); + writer.write_all(&record(2, b"end\0")).await.unwrap(); + writer.shutdown().await.unwrap(); + let diagnostic = closed(&logs).await; + let finished = logs.finish(window, deadline()).await.unwrap(); + assert_eq!(finished.tail.len(), TAIL); + assert_eq!(finished.tail, diagnostic.tail); + assert!(finished.tail.ends_with(b"end\0")); + assert_eq!(finished.stdout_discarded, 17); + assert!(finished.closed); + assert!(!format!("{finished:?}").contains("end\\0")); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_revoke_capture_and_close_its_quiet_socket_when_an_open_capability_is_dropped() +{ + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + drop(window); + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(matches!( + closed(&logs).await.failure, + Some(DockerError::Protocol) + )); + assert!(logs.begin(AttemptId::default(), deadline()).await.is_err()); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_reject_cross_owner_capabilities_and_revoke_the_originating_window() { + let (_first_writer, first_reader) = duplex(8); + let (_second_writer, second_reader) = duplex(8); + let mut first = logs(first_reader); + let mut second = logs(second_reader); + let attempt = AttemptId::default(); + let foreign = first.begin(attempt, deadline()).await.unwrap(); + let own = second.begin(attempt, deadline()).await.unwrap(); + assert!(matches!( + second.finish(foreign, deadline()).await, + Err(DockerError::Protocol) + )); + assert!(matches!( + closed(&first).await.failure, + Some(DockerError::Protocol) + )); + assert!(matches!( + second.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + drop(own); + assert!(closed(&second).await.failure.is_some()); + assert!(first.shutdown().await.is_err()); + assert!(second.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_reject_a_stale_finish_without_closing_the_current_window() { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let old = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let key = old.key; + logs.finish(old, deadline()).await.unwrap(); + let current = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let (_guard, receipt) = ReceiptGuard::new(deadline()); + assert!(matches!( + logs.request(Action::Finish(key, receipt), deadline()).await, + Err(DockerError::Protocol) + )); + assert!( + logs.finish(current, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_poison_window_mutation_after_an_overlapping_begin() { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Busy) + )); + assert!(matches!( + logs.finish(first, deadline()).await, + Err(DockerError::Stopped) + )); + assert!(closed(&logs).await.failure.is_some()); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_preserve_partial_record_io_across_begin_and_finish_commands() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + let frame = record(1, b"fragmented"); + writer.write_all(frame.get(..4).unwrap()).await.unwrap(); + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!( + logs.finish(first, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + let second = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(frame.get(4..).unwrap()).await.unwrap(); + writer.shutdown().await.unwrap(); + closed(&logs).await; + assert_eq!( + logs.finish(second, deadline()).await.unwrap().tail.as_ref(), + b"fragmented" + ); + logs.shutdown().await.unwrap(); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test] +async fn test_should_retire_a_mutation_whose_ready_reply_is_abandoned(#[case] finishing: bool) { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let sender = logs.requests.clone(); + let window = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + if let Some(window) = window { + let mut finish = Box::pin(logs.finish(window, deadline())); + assert!( + finish + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + drop(finish); + } else { + let mut begin = Box::pin(logs.begin(AttemptId::default(), deadline())); + assert!( + begin + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + drop(begin); + } + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(closed(&logs).await.failure.is_some()); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + assert!(logs.shutdown().await.is_err()); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test(start_paused = true)] +async fn test_should_expire_a_ready_mutation_receipt_while_its_public_future_remains_unpolled( + #[case] finishing: bool, +) { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let sender = logs.requests.clone(); + let until = Instant::now() + Duration::from_secs(1); + let window = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + if let Some(window) = window { + let mut finish = Box::pin(logs.finish(window, until)); + assert!( + finish + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + advance(Duration::from_secs(1)).await; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(probe(&sender).await.failure.is_some()); + assert!(matches!(finish.await, Err(DockerError::Deadline))); + } else { + let mut begin = Box::pin(logs.begin(AttemptId::default(), until)); + assert!( + begin + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + advance(Duration::from_secs(1)).await; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(probe(&sender).await.failure.is_some()); + assert!(matches!(begin.await, Err(DockerError::Deadline))); + } + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_retain_capture_failure_when_finishing_an_active_window() { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&[3, 0, 0, 0, 0, 0, 0, 0]).await.unwrap(); + assert!(closed(&logs).await.failure.is_some()); + assert!(matches!( + logs.finish(window, deadline()).await, + Err(DockerError::Protocol) + )); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} + +#[tokio::test(start_paused = true)] +async fn test_should_preserve_an_on_time_consumption_receipt_processed_after_its_deadline() { + let mut state = Capture::new(TAIL).unwrap(); + let (window, active) = LogWindow::new(AttemptId::default()); + let until = Instant::now() + Duration::from_secs(1); + let (mut guard, receipt) = ReceiptGuard::new(until); + let (reply, response) = oneshot::channel(); + state.reply(Request { + action: Action::Begin(active, receipt), + deadline: until, + reply, + }); + response.await.unwrap().unwrap(); + guard.consume(until).unwrap(); + advance(Duration::from_secs(2)).await; + assert!(!state.reconcile()); + assert!(state.failure.is_none()); + assert!(state.active.is_some()); + drop(window); + assert!(state.reconcile()); +} +#[rstest::rstest] +#[case::expired(false)] +#[case::abandoned(true)] +#[tokio::test] +async fn test_should_reject_unissued_window_requests_without_mutating_capture( + #[case] abandoned: bool, +) { + let mut state = Capture::new(TAIL).unwrap(); + let (window, active) = LogWindow::new(AttemptId::default()); + let until = if abandoned { + deadline() + } else { + Instant::now() + }; + let (_guard, receipt) = ReceiptGuard::new(until); + let (reply, response) = oneshot::channel(); + let response = if abandoned { + drop(response); + None + } else { + Some(response) + }; + state.reply(Request { + action: Action::Begin(active, receipt), + deadline: until, + reply, + }); + if let Some(response) = response { + assert!(matches!( + response.await.unwrap(), + Err(DockerError::Deadline) + )); + } + assert!(state.active.is_none()); + assert!(state.receipt.is_none()); + assert!(state.failure.is_none()); + drop(window); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test] +async fn test_should_reject_queued_mutations_when_priority_shutdown_wins(#[case] finishing: bool) { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let (window, active) = LogWindow::new(AttemptId::default()); + let own = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + let (_guard, receipt) = ReceiptGuard::new(deadline()); + let action = if let Some(own) = &own { + Action::Finish(own.key, receipt) + } else { + Action::Begin(active, receipt) + }; + let (reply, response) = oneshot::channel(); + logs.requests + .try_send(Request { + action, + deadline: deadline(), + reply, + }) + .unwrap_or_else(|_| panic!("test mutation admission")); + logs.shutdown().await.unwrap(); + assert!(matches!(response.await.unwrap(), Err(DockerError::Stopped))); + drop((window, own)); +} +#[tokio::test] +async fn test_should_serve_bounded_window_commands_without_starving_flooded_output_or_shutdown() { + let (mut writer, reader) = duplex(64 * 1024); + let mut logs = logs(reader); + let producer = tokio::spawn(async move { + let frame = record(1, &vec![b'x'; 16384]); + while writer.write_all(&frame).await.is_ok() { + tokio::task::yield_now().await; + } + }); + for _ in 0..16 { + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let tail = logs.finish(window, deadline()).await.unwrap(); + assert!(tail.tail.len() <= TAIL); + } + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .unwrap(); + timeout(Duration::from_secs(1), producer) + .await + .unwrap() + .unwrap(); +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs new file mode 100644 index 00000000..ed1450b5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -0,0 +1,1173 @@ +//! Bounded local Docker wire service and strict attach framing. +//! +//! This layer owns HTTP connections, not container policy or lifetime. Callers +//! retain preallocated identities and reservations through reconciliation. + +mod binding; +mod code; +mod code_volume; +mod config; +mod connection; +mod environment; +mod executor; +mod framing; +mod image; +mod lease; +mod logs; +mod mutation; +mod network; +mod ownership; +mod pin; +mod plan; +mod pool; +mod probe; +mod pull; +mod recovery; +mod sdk; +mod startup; +mod types; +mod volumes; +mod zip; + +use std::{ + collections::BTreeMap, + fmt, + io::{self, Write}, + mem, + path::PathBuf, + sync::Arc, + time::Duration, +}; + +/// Generated Docker model contracts, without the upstream network client. +pub use bollard_stubs::models; +use bytes::Bytes; +pub use code::DockerCodeArchive; +pub use code_volume::{ + CodePreparation, CodeRetirement, CodeVolumeConfig, CodeVolumeIdentity, CodeVolumeManager, + CodeVolumePort, CodeVolumeStatus, PinnedCode, +}; +pub use config::{ + DockerGatewayEndpoint, DockerRuntimeSettings, RegistryAuthError, RegistryAuthFile, +}; +use connection::exchange; +pub use environment::{ + EnvironmentCapacity, EnvironmentConfig, EnvironmentError, EnvironmentHandle, + EnvironmentManager, EnvironmentPhase, EnvironmentPort, EnvironmentResponse, + EnvironmentSettings, EnvironmentStatus, +}; +pub use executor::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, +}; +pub use framing::{DockerOutputReader, MAX_DOCKER_RECORD, OutputChannel, OutputChunk}; +use http::{Method, StatusCode}; +use hyper::upgrade::Upgraded; +use hyper_util::rt::TokioIo; +pub use image::ImageResolver; +pub use lease::{ImageLeaseConfig, ImageLeaseManager, ImageLeasePort, ImageLeaseStatus}; +pub use logs::{FunctionLogs, LogSnapshot, LogWindow}; +use models::{ + ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, + ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, +}; +pub use network::QualifiedNetwork; +pub use ownership::{DirectoryLease, DirectoryLeaseError}; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +pub use pin::{ImagePin, ImagePinScope}; +pub use plan::{ + ContainerPolicy, DockerCpuAllocation, EnvironmentPlan, NetworkSelection, RelayBudgets, +}; +pub use pool::{EnvironmentPool, PoolConfig, PoolInvocation, PoolPort, PoolSettings, PoolStatus}; +pub use probe::{ + EndpointProbe, EndpointProbeConfig, EnvironmentProbe, EnvironmentProbeConfig, + EnvironmentProbeError, ProbeFailure, ProbeSettings, probe_endpoint, probe_environment, +}; +pub use pull::{ImagePullPolicy, RegistryCredential, RegistryCredentials, RegistryHost}; +pub use recovery::{ + CodeRecovery, ContainerRecovery, ImagePinRecovery, NamespaceRecovery, reconcile_containers, + reconcile_image_pins, reconcile_namespace, +}; +use rustack_core::json::{JsonLimits, from_slice}; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +pub use rustack_lambda_runtime::probe::{GatewayInstanceId, GatewayReply}; +pub use sdk::DockerSdkEnvironment; +use serde::{Serialize, de::DeserializeOwned}; +pub use startup::{ + DockerCapabilities, DockerInstallation, DockerRuntime, DockerRuntimeStatus, + DockerStartupConfig, DockerStartupError, +}; +use tokio::{ + io::{ReadHalf, WriteHalf, split}, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; +pub use types::{ApiVersion, DockerError, DockerKey, DockerSocket}; +use uuid::Uuid; +pub use zip::DockerZipRuntime; + +use self::binding::DaemonBinding; +use crate::{ + artifact::{Architecture, ImageReference}, + executor::ExecutorError, + invocation::CleanupTask, +}; + +const MAX_BODY_BYTES: usize = 1024 * 1024; + +#[derive(Debug, Clone, Copy)] +enum Operation { + Version, + Info, + Network, + Image, + TagPin, + InspectPin, + RemovePin, + Create, + Start, + Inspect, + Kill, + Wait, + Remove, + List, + ListImages, + Pull, + Attach, + UploadCode, + CreateVolume, + InspectVolume, + ListVolumes, + RemoveVolume, +} +impl Operation { + fn cleanup(self) -> bool { + matches!( + self, + Self::Inspect + | Self::Kill + | Self::Remove + | Self::List + | Self::ListImages + | Self::InspectPin + | Self::RemovePin + | Self::InspectVolume + | Self::ListVolumes + | Self::RemoveVolume + ) + } +} + +struct RawRequest { + archive: Option, + pull: Option, + binding: Option>, + mutation: Option, + operation: Operation, + method: Method, + target: String, + body: Bytes, + expected: StatusCode, +} +impl fmt::Debug for RawRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DockerRequest") + .field("operation", &self.operation) + .field("bytes", &self.body.len()) + .finish_non_exhaustive() + } +} +enum RawReply { + Pulled, + Body { status: StatusCode, bytes: Bytes }, + Upgrade(TokioIo), +} +impl fmt::Debug for RawReply { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Pulled => formatter.write_str("DockerPull()"), + Self::Body { status, bytes } => formatter + .debug_struct("DockerReply") + .field("status", status) + .field("bytes", &bytes.len()) + .finish(), + Self::Upgrade(_) => formatter.write_str("DockerUpgrade()"), + } + } +} +#[cfg(test)] +#[derive(Debug, Clone)] +struct CompletionGate { + entered: watch::Sender, + release: watch::Receiver, +} + +#[derive(Debug)] +struct Envelope { + #[cfg(test)] + upload_completion: Option, + request: RawRequest, + deadline: Instant, + reply: oneshot::Sender>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ResponseOwner { + Observer, + Owner, +} + +#[derive(Debug, Clone, TypedBuilder)] +struct RawPort { + #[cfg(test)] + #[builder(default)] + upload_completion: Option, + response_owner: ResponseOwner, + binding: Option>, + business: mpsc::Sender, + cleanup: mpsc::Sender, + waits: mpsc::Sender, + pulls: mpsc::Sender, + stop: watch::Sender, +} +impl RawPort { + async fn request( + &self, + request: RawRequest, + deadline: Instant, + ) -> Result { + if self.response_owner == ResponseOwner::Owner { + return self.request_owned(request, deadline).await; + } + let response = self.enqueue(request, deadline)?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + async fn request_owned( + &self, + request: RawRequest, + deadline: Instant, + ) -> Result { + let response = self.enqueue(request, deadline)?; + let result = response.await.map_err(|_| DockerError::Worker)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + fn enqueue( + &self, + mut request: RawRequest, + deadline: Instant, + ) -> Result>, DockerError> { + request.binding.clone_from(&self.binding); + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + check_deadline(deadline)?; + let maximum = match (&request.archive, request.operation) { + (Some(archive), Operation::UploadCode) + if archive.bytes().as_ptr() == request.body.as_ptr() + && archive.len() == request.body.len() => + { + code::MAX_CODE_ARCHIVE_BYTES + } + (None, Operation::UploadCode) | (Some(_), _) => { + return Err(DockerError::Configuration); + } + (None, _) => MAX_BODY_BYTES, + }; + if request.target.len() > 4096 || request.body.len() > maximum { + return Err(DockerError::TooLarge); + } + let sender = if matches!(request.operation, Operation::Pull) { + &self.pulls + } else if matches!(request.operation, Operation::Wait) { + &self.waits + } else if request.operation.cleanup() { + &self.cleanup + } else { + &self.business + }; + let (reply, response) = oneshot::channel(); + #[cfg(test)] + let upload_completion = if matches!(request.operation, Operation::UploadCode) { + self.upload_completion.clone() + } else { + None + }; + sender + .try_send(Envelope { + #[cfg(test)] + upload_completion, + request, + deadline, + reply, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Stopped, + })?; + Ok(response) + } +} + +/// A strict upgraded attach stream whose HTTP driver has already joined. +#[non_exhaustive] +pub struct DockerAttachment(TokioIo); +impl fmt::Debug for DockerAttachment { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerAttachment()") + } +} + +/// Strict stdout/stderr reader for a split upgraded Docker socket. +pub type AttachmentReader = DockerOutputReader>>; +/// Raw container stdin; the owning relay writer supplies framing/IO deadlines. +pub type AttachmentWriter = WriteHalf>; + +impl DockerAttachment { + /// Transfer both socket halves into retained environment IO pumps. + /// + /// # Errors + /// Rejects a zero/over60s record budget. Dropping either returned owner does + /// not prove the external container has exited. + /// + /// ```no_run + /// # fn split(attach: rustack_lambda_core::docker::DockerAttachment) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let (reader, writer) = attach.split(std::time::Duration::from_secs(5))?; + /// # Ok(()) } + /// ``` + pub fn split( + self, + budget: Duration, + ) -> Result<(AttachmentReader, AttachmentWriter), DockerError> { + let (reader, writer) = split(self.0); + Ok((DockerOutputReader::new(reader, budget)?, writer)) + } +} + +/// Fixed ownership-label filter; authorization to reclaim remains above wire IO. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct OwnerFilter { + owner: Uuid, + run: Option, + environment: Option, +} +impl OwnerFilter { + /// Validate nonnil UUID scope before issuing a label-filtered list. + /// + /// # Errors + /// Rejects nil identities or an environment without a run scope. + /// + /// ``` + /// let filter = rustack_lambda_core::docker::OwnerFilter::new(uuid::Uuid::new_v4(), None, None)?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new( + owner: Uuid, + run: Option, + environment: Option, + ) -> Result { + if owner.is_nil() + || run.is_some_and(|id| id.is_nil()) + || environment.is_some_and(|id| id.is_nil()) + || (environment.is_some() && run.is_none()) + { + return Err(DockerError::Configuration); + } + Ok(Self { + owner, + run, + environment, + }) + } + fn encoded(self) -> Result { + self.query(Some(64)) + } + fn query(self, limit: Option) -> Result { + let filters = self.encoded_labels()?; + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("all", "true"); + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + Ok(query.append_pair("filters", &filters).finish()) + } + fn encoded_labels(self) -> Result { + let mut labels = vec![format!("rustack.lambda.ownerId={}", self.owner)]; + if let Some(run) = self.run { + labels.push(format!("rustack.lambda.runId={run}")); + } + if let Some(environment) = self.environment { + labels.push(format!("rustack.lambda.environmentId={environment}")); + } + let filters = BTreeMap::from([("label", labels)]); + serde_json::to_string(&filters).map_err(|_| DockerError::Schema) + } +} + +/// Cloneable typed request port; it does not own service shutdown. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct DockerPort { + raw: RawPort, + version: ApiVersion, +} + +macro_rules! container_method { + ($name:ident, $method:expr, $suffix:literal, $status:expr, $operation:ident, $result:ty, $decode:ident) => { + #[doc = concat!("Issue the scoped container ", stringify!($operation), " operation.")] + /// + /// # Errors + /// Returns bounded admission, deadline, transport, status or schema failure. + /// + #[doc = concat!("```no_run\n# async fn example(port: &rustack_lambda_core::docker::DockerPort, key: &rustack_lambda_core::docker::DockerKey) {\nlet result = port.", stringify!($name), "(key, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await;\n# }\n```")] + pub async fn $name(&self, key: &DockerKey, deadline: Instant) -> Result<$result, DockerError> { + let path = format!("/containers/{}{}", escaped(key.as_str()), $suffix); + let reply = self.call(Operation::$operation, $method, path, Bytes::new(), $status, deadline).await?; + $decode(reply, deadline) + } + }; +} + +impl DockerPort { + /// Only retained internal owners may await accepted IO past observation expiry. + fn owned(mut self) -> Self { + self.raw.response_owner = ResponseOwner::Owner; + self + } + + /// Negotiated API version, immutable after successful connection startup. + /// + /// ```no_run + /// # fn inspect(port: &rustack_lambda_core::docker::DockerPort) { + /// let version = port.api_version(); + /// # } + /// ``` + #[must_use] + pub fn api_version(&self) -> ApiVersion { + self.version + } + + /// Inspect bounded daemon capabilities; validate execution policy above IO. + /// + /// # Errors + /// Returns admission, deadline, transport or bounded metadata/schema failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort) { + /// let info = port.info(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn info(&self, deadline: Instant) -> Result { + decode( + self.call( + Operation::Info, + Method::GET, + "/info".into(), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Inspect an already validated image reference without pulling it. + /// + /// # Errors + /// Returns missing-image status, deadline, transport or metadata failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, image: &rustack_lambda_core::artifact::ImageReference) { + /// let info = port.inspect_image(image, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_image( + &self, + image: &ImageReference, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::Image, + Method::GET, + format!("/images/{}/json", escaped(image.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Create under a preallocated name retained by the environment owner. + /// + /// # Errors + /// Returns bounded request/schema, status, deadline or transport failure. + /// A failed/dropped response still requires name/label reconciliation. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, name: &rustack_lambda_core::docker::DockerKey, body: &rustack_lambda_core::docker::models::ContainerCreateBody) { + /// let created = port.create_container(name, body, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_container( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + deadline: Instant, + ) -> Result { + self.create_container_inner(name, body, None, deadline, None) + .await + } + + /// Create from an immutable image ID using an explicit captured platform. + /// + /// # Errors + /// Returns bounded request, status, deadline or transport failure; the owner + /// must reconcile uncertain creation under the preallocated name/labels. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, key: &rustack_lambda_core::docker::DockerKey, body: &rustack_lambda_core::docker::models::ContainerCreateBody) { + /// let result = port.create_container_on_platform(key,body, + /// rustack_lambda_core::artifact::Architecture::Arm64, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_container_on_platform( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + architecture: Architecture, + deadline: Instant, + ) -> Result { + self.create_container_inner(name, body, Some(architecture), deadline, None) + .await + } + + async fn create_container_inner( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + architecture: Option, + deadline: Instant, + mutation: Option, + ) -> Result { + let query = { + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("name", name.as_str()); + if let Some(architecture) = architecture { + query.append_pair("platform", architecture.docker_platform()); + } + query.finish() + }; + decode( + self.raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation, + operation: Operation::Create, + method: Method::POST, + target: format!("/v{}/containers/create?{query}", self.version), + body: encode(body)?, + expected: StatusCode::CREATED, + }, + deadline, + ) + .await?, + deadline, + ) + } + + container_method!( + start_container, + Method::POST, + "/start", + StatusCode::NO_CONTENT, + Start, + (), + unit + ); + container_method!( + inspect_container, + Method::GET, + "/json", + StatusCode::OK, + Inspect, + ContainerInspectResponse, + decode + ); + container_method!( + kill_container, + Method::POST, + "/kill?signal=SIGKILL", + StatusCode::NO_CONTENT, + Kill, + (), + unit + ); + container_method!( + wait_container, + Method::POST, + "/wait?condition=not-running", + StatusCode::OK, + Wait, + ContainerWaitResponse, + decode + ); + container_method!( + remove_container, + Method::DELETE, + "?force=true&v=false", + StatusCode::NO_CONTENT, + Remove, + (), + unit + ); + + /// List only a fixed ownership-label scope for reconciliation. + /// + /// # Errors + /// Returns admission, deadline, status, transport or metadata-budget failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, filter: rustack_lambda_core::docker::OwnerFilter) { + /// let found = port.list_containers(filter, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn list_containers( + &self, + filter: OwnerFilter, + deadline: Instant, + ) -> Result, DockerError> { + decode( + self.call( + Operation::List, + Method::GET, + format!("/containers/json?{}", filter.encoded()?), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Establish binary attach before starting the relay; no log replay. + /// + /// # Errors + /// Returns invalid upgrade, status, deadline or joined transport failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, relay: &rustack_lambda_core::docker::DockerKey) { + /// let attached = port.attach(relay, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn attach( + &self, + relay: &DockerKey, + deadline: Instant, + ) -> Result { + self.attach_inner(relay, true, deadline).await + } + + /// Attach only live function output before start, without stdin or replay. + /// + /// # Errors + /// Returns admission, upgrade, status, deadline or joined transport failure. + /// + /// ```no_run + /// # async fn logs(port: &rustack_lambda_core::docker::DockerPort, function: &rustack_lambda_core::docker::DockerKey) { + /// let attached = port.attach_logs(function, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn attach_logs( + &self, + function: &DockerKey, + deadline: Instant, + ) -> Result { + self.attach_inner(function, false, deadline).await + } + + async fn attach_inner( + &self, + key: &DockerKey, + stdin: bool, + deadline: Instant, + ) -> Result { + let reply = self + .call( + Operation::Attach, + Method::POST, + format!( + "/containers/{}/attach?stdin={stdin}&stdout=true&stderr=true&stream=true&\ + logs=false", + escaped(key.as_str()) + ), + Bytes::new(), + StatusCode::SWITCHING_PROTOCOLS, + deadline, + ) + .await?; + match reply { + RawReply::Upgrade(io) => Ok(DockerAttachment(io)), + RawReply::Body { .. } | RawReply::Pulled => Err(DockerError::Protocol), + } + } + + async fn call( + &self, + operation: Operation, + method: Method, + path: String, + body: Bytes, + expected: StatusCode, + deadline: Instant, + ) -> Result { + let target = format!("/v{}{path}", self.version); + self.raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation: None, + operation, + method, + target, + body, + expected, + }, + deadline, + ) + .await + } +} + +/// Owner of bounded request scheduling and all in-flight management drivers. +#[derive(Debug)] +#[non_exhaustive] +pub struct DockerWire { + port: DockerPort, + owner: Owner, +} + +impl DockerWire { + pub(super) fn physically_settled(&self) -> bool { + self.owner.primary.is_some() + && self.owner.task.is_none() + && self.owner.emergency.is_finished() + } + + /// Verify the explicit socket and negotiate a bounded API response. + /// + /// # Errors + /// Returns unavailable socket, malformed/incompatible API, IO or supervision + /// failure. Dropped startup waiters retain cleanup through the owner task. + /// + /// ```no_run + /// # async fn example() -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let socket = "unix:///var/run/docker.sock".parse()?; + /// let mut wire = rustack_lambda_core::docker::DockerWire::connect(socket, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// wire.shutdown().await?; + /// # Ok(()) } + /// ``` + pub async fn connect(socket: DockerSocket, deadline: Instant) -> Result { + tokio::runtime::Handle::try_current().map_err(|_| DockerError::Worker)?; + let path = socket.verified(deadline).await?; + let (mut owner, raw) = Owner::start(path); + let result = async { + let response = raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation: None, + operation: Operation::Version, + method: Method::GET, + target: "/version".into(), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline, + ) + .await?; + let version: SystemVersion = decode(response, deadline)?; + ApiVersion::negotiate( + version.api_version.as_deref().ok_or(DockerError::Version)?, + version + .min_api_version + .as_deref() + .ok_or(DockerError::Version)?, + ) + } + .await; + match result { + Ok(version) => Ok(Self { + port: DockerPort { raw, version }, + owner, + }), + Err(error) => { + owner.shutdown().await?; + Err(error) + } + } + } + + /// Clone a typed port; the original wire keeps shutdown responsibility. + /// + /// ```no_run + /// # fn port(wire: &rustack_lambda_core::docker::DockerWire) { + /// let port = wire.port(); + /// # } + /// ``` + #[must_use] + pub fn port(&self) -> DockerPort { + self.port.clone() + } + + /// Stop admission and join every retained management request/driver. + /// + /// # Errors + /// Caches task supervision failures. Exported stream owners and external + /// containers must be joined/removed by their environment owner separately. + /// + /// ```no_run + /// # async fn shutdown(wire: &mut rustack_lambda_core::docker::DockerWire) { + /// let settled = wire.shutdown().await; + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), DockerError> { + self.owner.shutdown().await + } +} + +#[derive(Debug)] +struct Owner { + emergency: Arc, + stop: watch::Sender, + task: Option>>, + primary: Option>, + settlement: Option>, +} +impl Owner { + fn start(socket: PathBuf) -> (Self, RawPort) { + let (business, requests) = mpsc::channel(32); + let (cleanup, priority) = mpsc::channel(8); + let (waits, pending_waits) = mpsc::channel(8); + let (pulls, pending_pulls) = mpsc::channel(8); + let (stop, _) = watch::channel(false); + let emergency = Arc::new(CleanupTask::default()); + let actor = Actor::builder() + .runtime(Handle::current()) + .emergency(emergency.clone()) + .socket(socket) + .business(requests) + .cleanup(priority) + .waits(pending_waits) + .pulls(pending_pulls) + .control(stop.clone()) + .build(); + let task = tokio::spawn(actor.run().with_subscriber(runtime_dispatch())); + ( + Self { + emergency, + stop: stop.clone(), + task: Some(task), + primary: None, + settlement: None, + }, + RawPort::builder() + .response_owner(ResponseOwner::Observer) + .binding(None) + .business(business) + .cleanup(cleanup) + .waits(waits) + .pulls(pulls) + .stop(stop) + .build(), + ) + } + async fn shutdown(&mut self) -> Result<(), DockerError> { + self.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = if let Some(result) = &self.primary { + result.clone() + } else { + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.primary = Some(result.clone()); + result + }; + let emergency = self.emergency.wait().await.map_err(|_| DockerError::Worker); + let result = result.and(emergency); + self.settlement = Some(result.clone()); + result + } +} +impl Drop for Owner { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[derive(Debug, TypedBuilder)] +struct Actor { + runtime: Handle, + emergency: Arc, + socket: PathBuf, + business: mpsc::Receiver, + cleanup: mpsc::Receiver, + waits: mpsc::Receiver, + pulls: mpsc::Receiver, + control: watch::Sender, + #[builder(default)] + jobs: JoinSet<(Lane, bool)>, + #[builder(default)] + counts: Counts, +} + +#[derive(Debug, Clone, Copy)] +enum Lane { + Business, + Cleanup, + Wait, + Pull, +} +#[derive(Debug, Default)] +struct Counts { + business: usize, + cleanup: usize, + waits: usize, + pulls: usize, +} +impl Counts { + fn get_mut(&mut self, lane: Lane) -> &mut usize { + match lane { + Lane::Business => &mut self.business, + Lane::Cleanup => &mut self.cleanup, + Lane::Wait => &mut self.waits, + Lane::Pull => &mut self.pulls, + } + } +} +impl Actor { + async fn run(mut self) -> Result<(), DockerError> { + let mut stop = self.control.subscribe(); + let mut outcome = loop { + tokio::select! { + biased; + () = connection::stopped(&mut stop) => break Ok(()), + job = self.jobs.join_next(), if !self.jobs.is_empty() => match job { + Some(Ok((lane, false))) => { + let count = self.counts.get_mut(lane); + if let Some(value) = count.checked_sub(1) { *count = value; } else { break Err(DockerError::Worker); } + } + _ => break Err(DockerError::Worker), + }, + envelope = self.cleanup.recv(), if self.counts.cleanup < 2 => match envelope { Some(envelope) => self.start(envelope, Lane::Cleanup), None => break Err(DockerError::Stopped) }, + envelope = self.waits.recv(), if self.counts.waits < 32 => match envelope { Some(envelope) => self.start(envelope, Lane::Wait), None => break Err(DockerError::Stopped) }, + envelope = self.pulls.recv(), if self.counts.pulls < 4 => match envelope { Some(envelope) => self.start(envelope, Lane::Pull), None => break Err(DockerError::Stopped) }, + envelope = self.business.recv(), if self.counts.business < 6 => match envelope { Some(envelope) => self.start(envelope, Lane::Business), None => break Err(DockerError::Stopped) }, + } + }; + self.control.send_replace(true); + self.business.close(); + self.cleanup.close(); + self.waits.close(); + self.pulls.close(); + while let Ok(envelope) = self.business.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Ok(envelope) = self.cleanup.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Ok(envelope) = self.waits.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Ok(envelope) = self.pulls.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Some(result) = self.jobs.join_next().await { + if !matches!(result, Ok((_, false))) { + outcome = Err(DockerError::Worker); + } + } + outcome + } + fn start(&mut self, envelope: Envelope, lane: Lane) { + if *self.control.borrow() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + return; + } + if envelope.reply.is_closed() { + return; + } + if Instant::now() >= envelope.deadline { + let _ = envelope.reply.send(Err(DockerError::Deadline)); + return; + } + let socket = self.socket.clone(); + let stopped = self.control.subscribe(); + *self.counts.get_mut(lane) += 1; + self.jobs.spawn( + async move { + let result = exchange(&socket, envelope.request, envelope.deadline, stopped).await; + #[cfg(test)] + if let Some(mut gate) = envelope.upload_completion { + gate.entered.send_replace(true); + let _ = gate.release.wait_for(|released| *released).await; + } + let failed = matches!(&result, Err(DockerError::Worker)); + let _ = envelope.reply.send(result); + (lane, failed) + } + .with_subscriber(runtime_dispatch()), + ); + } +} + +impl Drop for Actor { + fn drop(&mut self) { + self.control.send_replace(true); + if self.jobs.is_empty() { + return; + } + let mut jobs = mem::take(&mut self.jobs); + self.emergency.start_on(&self.runtime, move || async move { + let mut failed = false; + while let Some(result) = jobs.join_next().await { + if !matches!(result, Ok((_, false))) { + failed = true; + } + } + if failed { + Err(ExecutorError::CleanupFailed( + "Docker wire cleanup failed".into(), + )) + } else { + Ok(()) + } + }); + } +} + +fn escaped(value: &str) -> String { + utf8_percent_encode(value, NON_ALPHANUMERIC).to_string() +} +fn check_deadline(deadline: Instant) -> Result<(), DockerError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(600) { + return Err(DockerError::Configuration); + } + Ok(()) +} +fn decode(reply: RawReply, deadline: Instant) -> Result { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let result = match reply { + RawReply::Body { bytes, .. } => from_slice( + &bytes, + JsonLimits::new(MAX_BODY_BYTES, 32, 4096, 16 * 1024, 256)?, + ) + .map_err(DockerError::from), + RawReply::Upgrade(_) | RawReply::Pulled => Err(DockerError::Protocol), + }; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result +} +fn unit(reply: RawReply, deadline: Instant) -> Result<(), DockerError> { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + match reply { + RawReply::Body { bytes, .. } if bytes.is_empty() => Ok(()), + _ => Err(DockerError::Protocol), + } +} + +#[derive(Default)] +struct Encoded { + bytes: Vec, + failure: Option, +} +impl fmt::Debug for Encoded { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Encoded") + .field("bytes", &self.bytes.len()) + .field("failure", &self.failure) + .finish() + } +} +impl Write for Encoded { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let length = self + .bytes + .len() + .checked_add(bytes.len()) + .filter(|length| *length <= MAX_BODY_BYTES); + if length.is_none() { + self.failure = Some(DockerError::TooLarge); + return Err(io::Error::other("Docker byte budget exceeded")); + } + if self.bytes.try_reserve(bytes.len()).is_err() { + self.failure = Some(DockerError::Allocation); + return Err(io::Error::other("Docker allocation failed")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn encode(value: &impl Serialize) -> Result { + let mut output = Encoded::default(); + if serde_json::to_writer(&mut output, value).is_err() { + return Err(output.failure.unwrap_or(DockerError::Schema)); + } + Ok(Bytes::from(output.bytes)) +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +pub(crate) use lease::tests as image_fixture; diff --git a/crates/rustack-lambda-core/src/docker/mutation.rs b/crates/rustack-lambda-core/src/docker/mutation.rs new file mode 100644 index 00000000..bece8280 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/mutation.rs @@ -0,0 +1,31 @@ +//! A retained receipt distinguishes definitely unissued from uncertain mutation IO. + +use std::sync::{ + Arc, + atomic::{AtomicU8, Ordering}, +}; + +#[derive(Debug, Clone)] +pub(super) struct MutationReceipt(Arc); +pub(super) struct MutationGuard(MutationReceipt); +impl MutationGuard { + pub(super) fn new() -> Self { + Self(MutationReceipt(Arc::new(AtomicU8::new(0)))) + } + pub(super) fn receipt(&self) -> MutationReceipt { + self.0.clone() + } + pub(super) fn issued(&mut self) { + self.0.0.store(1, Ordering::Release); + } +} +impl Drop for MutationGuard { + fn drop(&mut self) { + self.0.0.fetch_or(2, Ordering::Release); + } +} +impl MutationReceipt { + pub(super) fn definitely_unissued(&self) -> bool { + self.0.load(Ordering::Acquire) == 2 + } +} diff --git a/crates/rustack-lambda-core/src/docker/network.rs b/crates/rustack-lambda-core/src/docker/network.rs new file mode 100644 index 00000000..90185e61 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/network.rs @@ -0,0 +1,146 @@ +//! Read-only network qualification pins a bridge ID before allocation. + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; + +use super::{ + ContainerPolicy, DockerError, DockerKey, DockerPort, NetworkSelection, Operation, + check_deadline, decode, escaped, models::NetworkInspect, +}; + +/// Validated disconnected or full-ID local bridge selection; not startup readiness. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct QualifiedNetwork { + requested: NetworkSelection, + selected: NetworkSelection, +} +impl QualifiedNetwork { + /// Bind the inspected network ID into a matching operator policy. + /// + /// # Errors + /// Rejects a policy for a different originally requested/qualified network. + /// + /// ```no_run + /// # fn bind(network:&rustack_lambda_core::docker::QualifiedNetwork, policy:&rustack_lambda_core::docker::ContainerPolicy) { + /// let fixed=network.apply(policy); + /// # } + /// ``` + pub fn apply(&self, policy: &ContainerPolicy) -> Result { + if policy.network != self.requested && policy.network != self.selected { + return Err(DockerError::Configuration); + } + let mut policy = policy.clone(); + policy.network = self.selected.clone(); + Ok(policy) + } + /// Whether the selected namespace deliberately has no external network. + /// + /// ```no_run + /// # fn check(network:&rustack_lambda_core::docker::QualifiedNetwork) { + /// let disconnected=network.disconnected(); + /// # } + /// ``` + #[must_use] + pub fn disconnected(&self) -> bool { + self.selected.value() == "none" + } +} +impl NetworkSelection { + /// Inspect and pin an exact operator-owned local user bridge, or select none. + /// + /// No create/delete or host/default-network fallback occurs. A qualified + /// bridge can still be deleted by the operator; allocation then fails. + /// + /// # Errors + /// Rejects expired budgets, wire failure, incomplete/builtin/nonlocal network + /// metadata, partial IDs or conflicting returned identity. + /// + /// ```no_run + /// # async fn qualify(port:&rustack_lambda_core::docker::DockerPort)->Result<(),rustack_lambda_core::docker::DockerError>{ + /// let selection:rustack_lambda_core::docker::NetworkSelection="worker-network".parse()?; + /// let network=selection.qualify(port,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn qualify( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + let selected = if self.value() == "none" { + self.clone() + } else { + let inspected = port + .inspect_network(&self.value().parse()?, deadline) + .await?; + validate(self, &inspected)? + }; + check_deadline(deadline)?; + Ok(QualifiedNetwork { + requested: self.clone(), + selected, + }) + } +} +impl DockerPort { + /// Inspect bounded network metadata without mutating its lifetime. + /// + /// # Errors + /// Returns wire admission, status, deadline or bounded metadata failure. + /// + /// ```no_run + /// # async fn inspect(port:&rustack_lambda_core::docker::DockerPort,key:&rustack_lambda_core::docker::DockerKey){ + /// let network=port.inspect_network(key,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_network( + &self, + key: &DockerKey, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::Network, + Method::GET, + format!("/networks/{}", escaped(key.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } +} +fn validate( + requested: &NetworkSelection, + network: &NetworkInspect, +) -> Result { + let name = network.name.as_deref().ok_or(DockerError::Configuration)?; + name.parse::()?; + let id = network.id.as_deref().ok_or(DockerError::Configuration)?; + let default = network + .options + .as_ref() + .and_then(|options| options.get("com.docker.network.bridge.default_bridge")); + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || !matches!(requested.value(),value if value==name||value==id) + || matches!(name, "host" | "bridge" | "none" | "default") + || network.driver.as_deref() != Some("bridge") + || network.scope.as_deref() != Some("local") + || network.ingress != Some(false) + || network.config_only != Some(false) + || default.is_some_and(|value| value != "false") + { + return Err(DockerError::Configuration); + } + id.parse() +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/network/tests.rs b/crates/rustack-lambda-core/src/docker/network/tests.rs new file mode 100644 index 00000000..57c2c80b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/network/tests.rs @@ -0,0 +1,86 @@ +use serde_json::{Value, json}; +use tokio::time::{Duration, Instant}; + +use super::*; +use crate::docker::{plan::tests::policy, tests::Daemon}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} +fn fixture() -> Value { + json!({"Name":"probe-network","Id":"a".repeat(64),"Driver":"bridge","Scope":"local","Ingress":false,"ConfigOnly":false,"Options":{}}) +} +#[rstest::rstest] +#[case::driver("Driver",json!("host"))] +#[case::scope("Scope",json!("swarm"))] +#[case::ingress("Ingress",json!(true))] +#[case::config_only("ConfigOnly",json!(true))] +#[case::missing_ingress("Ingress", Value::Null)] +#[case::missing_scope("Scope", Value::Null)] +#[case::wrong_name("Name",json!("another-network"))] +#[case::builtin("Name",json!("bridge"))] +#[case::bad_id("Id",json!("a".repeat(63)))] +#[case::uppercase_id("Id",json!("A".repeat(64)))] +#[case::default_bridge("Options",json!({"com.docker.network.bridge.default_bridge":"true"}))] +#[case::unknown_default("Options",json!({"com.docker.network.bridge.default_bridge":"unknown"}))] +#[test] +fn test_should_reject_unqualified_network_metadata(#[case] field: &str, #[case] value: Value) { + let mut network = fixture(); + network[field] = value; + let network = serde_json::from_value(network).unwrap(); + assert!(validate(&"probe-network".parse().unwrap(), &network).is_err()); +} +#[test] +fn test_should_pin_only_exact_names_or_full_ids_and_bind_matching_policy() { + let network = serde_json::from_value(fixture()).unwrap(); + let requested: NetworkSelection = "probe-network".parse().unwrap(); + let selected = validate(&requested, &network).unwrap(); + assert_eq!(selected.value(), "a".repeat(64)); + assert!(validate(&"aaaa".parse().unwrap(), &network).is_err()); + assert!(validate(&"a".repeat(64).parse().unwrap(), &network).is_ok()); + let qualified = QualifiedNetwork { + requested, + selected, + }; + let mut configured = policy(); + assert!(qualified.apply(&configured).is_err()); + configured.network = "probe-network".parse().unwrap(); + let fixed = qualified.apply(&configured).unwrap(); + assert_eq!(fixed.network.value(), "a".repeat(64)); + assert!(qualified.apply(&fixed).is_ok()); + assert!(!qualified.disconnected()); +} +#[tokio::test] +async fn test_should_issue_only_bounded_network_inspect_and_no_request_for_none() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let task = tokio::spawn(async move { + "probe-network" + .parse::() + .unwrap() + .qualify(&port, deadline()) + .await + }); + let request = daemon.next().await; + assert!( + request + .head + .starts_with("GET /v1.53/networks/probe%2Dnetwork ") + ); + request + .reply(200, &serde_json::to_vec(&fixture()).unwrap()) + .await; + assert!(!task.await.unwrap().unwrap().disconnected()); + let none = "none" + .parse::() + .unwrap() + .qualify(&wire.port(), deadline()) + .await + .unwrap(); + assert!(none.disconnected()); + assert!(none.apply(&policy()).is_ok()); + assert!(daemon.requests.try_recv().is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/ownership.rs b/crates/rustack-lambda-core/src/docker/ownership.rs new file mode 100644 index 00000000..ac44be41 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership.rs @@ -0,0 +1,843 @@ +//! Persistent, descriptor-relative ownership of a Docker resource namespace. +//! +//! Standard files are required for OS advisory locks. All synchronous IO runs +//! on bounded blocking workers; lease clones perform no IO and retain the lock. +#![allow(clippy::disallowed_types, clippy::disallowed_methods)] + +use std::{ + ffi::OsString, + fmt, + fs::{File, Metadata, TryLockError, canonicalize, symlink_metadata}, + future::Future, + io::{self, Read, Write}, + os::unix::fs::MetadataExt, + path::{Component, Path, PathBuf}, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use nix::{ + dir::Dir, + errno::Errno, + fcntl::{OFlag, open, openat, renameat}, + sys::stat::{Mode, mkdirat}, + unistd::geteuid, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + runtime::Handle, + sync::{AcquireError, OwnedSemaphorePermit, Semaphore, SemaphorePermit}, + task::{JoinHandle, spawn_blocking}, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +pub(super) use self::{images::ImageRecord, journal::ContainerRecord}; + +pub(super) mod codes; +mod daemon; +mod images; +mod journal; +mod snapshot; + +const NAMESPACE: &str = ".rustack-lambda-docker"; +const LOCK: &str = "lease.lock"; +const OWNER: &str = "owner.v1"; +const PENDING: &str = "owner.pending"; +const MAGIC: &[u8; 8] = b"RSDKOWN2"; +const CODE_MAGIC: &[u8; 8] = b"RSDKOWN3"; +const LEGACY_MAGIC: &[u8; 8] = b"RSDKOWN1"; +const OWNER_BYTES: usize = 56; +static WORKERS: Semaphore = Semaphore::const_new(4); + +/// Directory acquisition failures; diagnostics never include operator paths. +#[derive(thiserror::Error)] +#[non_exhaustive] +pub enum DirectoryLeaseError { + /// Invalid operator directory or acquisition budget. + #[error("invalid Docker ownership configuration")] + Configuration, + /// Another descriptor/process owns this data directory. + #[error("Docker data directory is already owned")] + InUse, + /// This immutable identity predates journal-before-effect provenance. + #[error("Docker ownership predates journal provenance; choose a new controlled data directory")] + Legacy, + /// Unsafe file kind/permissions/links or damaged authoritative identity. + #[error("invalid Docker ownership state")] + InvalidState, + /// The absolute acquisition deadline elapsed. + #[error("Docker ownership deadline exceeded")] + Deadline, + /// The requesting acquisition was abandoned. + #[error("Docker ownership acquisition cancelled")] + Cancelled, + /// No runtime or the blocking worker failed. + #[error("Docker ownership worker failed")] + Worker, + /// A filesystem operation failed, preserving its source without displaying it. + #[error("Docker ownership IO failed")] + Io(#[source] io::Error), +} +#[derive(Debug)] +struct CodeProducerState { + gate: Arc, + poisoned: AtomicBool, +} +impl CodeProducerState { + fn new() -> Self { + Self { + gate: Arc::new(Semaphore::new(1)), + poisoned: AtomicBool::new(false), + } + } +} +/// Exclusive producing lifetime; cancellation without an empty joined ledger poisons acquisition. +#[derive(Debug)] +pub(in crate::docker) struct CodeProducerGuard { + state: Arc, + _permit: OwnedSemaphorePermit, + clean: AtomicBool, +} +impl CodeProducerGuard { + pub(in crate::docker) fn settle(&self) { + self.clean.store(true, Ordering::Release); + } +} +impl Drop for CodeProducerGuard { + fn drop(&mut self) { + if !self.clean.load(Ordering::Acquire) { + self.state.poisoned.store(true, Ordering::Release); + } + } +} + +impl fmt::Debug for DirectoryLeaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} +impl From for DirectoryLeaseError { + fn from(source: io::Error) -> Self { + Self::Io(source) + } +} +impl From for DirectoryLeaseError { + fn from(source: Errno) -> Self { + match source { + Errno::ELOOP | Errno::ENOTDIR => Self::InvalidState, + _ => Self::Io(source.into()), + } + } +} + +/// Shared proof of an exclusive, durably identified Docker data directory. +#[derive(Clone)] +#[non_exhaustive] +pub struct DirectoryLease(Arc); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Provenance { + Legacy, + Journaled, + CodeJournaled, +} + +#[derive(TypedBuilder)] +struct Lease { + owner: Uuid, + run: Uuid, + transaction: Arc, + code_producer: Arc, + provenance: Provenance, + guard: LeaseGuard, +} +enum LeaseGuard { + Held { + directory: File, + _lock: File, + }, + #[cfg(test)] + Testing { + directory: File, + _root: tempfile::TempDir, + }, +} +impl fmt::Debug for DirectoryLease { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DirectoryLease()") + } +} +impl DirectoryLease { + pub(super) fn require_journaled(&self) -> Result<(), DirectoryLeaseError> { + match self.0.provenance { + Provenance::Journaled | Provenance::CodeJournaled => Ok(()), + Provenance::Legacy => Err(DirectoryLeaseError::Legacy), + } + } + + pub(super) fn code_producer(&self) -> Result { + let state = &self.0.code_producer; + if self.0.provenance != Provenance::CodeJournaled || state.poisoned.load(Ordering::Acquire) + { + return Err(DirectoryLeaseError::InvalidState); + } + let permit = state + .gate + .clone() + .try_acquire_owned() + .map_err(|_| DirectoryLeaseError::InUse)?; + if state.poisoned.load(Ordering::Acquire) { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(CodeProducerGuard { + state: state.clone(), + _permit: permit, + clean: AtomicBool::new(false), + }) + } + + pub(super) fn code_journaled(&self) -> bool { + self.0.provenance == Provenance::CodeJournaled + } + + /// Acquire a fresh code-capable namespace; existing Image roots are refused. + /// + /// Never upgrades or rewrites an existing owner. General acquisition can + /// reopen this generation to recover its recorded effects. + /// + /// # Errors + /// Returns acquisition errors, including `InvalidState` for older provenance. + /// + /// ```no_run + /// # async fn example() -> Result<(), rustack_lambda_core::docker::DirectoryLeaseError> { + /// let lease = rustack_lambda_core::docker::DirectoryLease::acquire_code( + /// "/tmp/rustack-code-data".into(), + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn acquire_code( + directory: PathBuf, + deadline: Instant, + ) -> Result { + let directory = DataDirectory::try_from(directory)?; + run(&WORKERS, deadline, move |control| { + acquire_profile(&directory, control, Provenance::CodeJournaled) + }) + .await + } + + pub(crate) fn same_instance(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + #[cfg(test)] + pub(super) fn synthetic(&self) -> bool { + matches!(self.0.guard, LeaseGuard::Testing { .. }) + } + /// Acquire a private namespace, retaining exclusion until the last clone drops. + /// + /// # Errors + /// Rejects invalid paths/budgets, competing owners, unsafe/corrupt state, + /// cancellation/deadline, filesystem failures and unavailable/failed workers. + /// Acquisition does not establish previous-run reconciliation or readiness. + /// + /// ```no_run + /// # async fn acquire() -> Result<(), rustack_lambda_core::docker::DirectoryLeaseError> { + /// use rustack_lambda_core::docker::DirectoryLease; + /// let lease = DirectoryLease::acquire( + /// "/tmp/rustack-data".into(), + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5), + /// ).await?; + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(lease); + /// # Ok(()) } + /// ``` + pub async fn acquire( + directory: PathBuf, + deadline: Instant, + ) -> Result { + let directory = DataDirectory::try_from(directory)?; + run(&WORKERS, deadline, move |control| { + acquire(&directory, control) + }) + .await + } + + /// Persistent UUID used for resource labels and private image aliases. + /// + /// ```no_run + /// # fn owner(lease: &rustack_lambda_core::docker::DirectoryLease) { + /// let owner = lease.owner_id(); + /// assert!(!owner.is_nil()); + /// # } + /// ``` + #[must_use] + pub fn owner_id(&self) -> Uuid { + self.0.owner + } + + /// Fresh UUID distinguishing this acquisition from prior runs. + /// + /// ```no_run + /// # fn run(lease: &rustack_lambda_core::docker::DirectoryLease) { + /// let run = lease.run_id(); + /// assert!(!run.is_nil()); + /// # } + /// ``` + #[must_use] + pub fn run_id(&self) -> Uuid { + self.0.run + } + + #[cfg(test)] + pub(super) fn testing(owner: Uuid, run: Uuid) -> Self { + let root = tempfile::tempdir().unwrap(); + let directory = File::open(root.path()).unwrap(); + Self(Arc::new( + Lease::builder() + .owner(owner) + .run(run) + .transaction(Arc::new(Semaphore::new(1))) + .code_producer(Arc::new(CodeProducerState::new())) + .provenance(Provenance::Journaled) + .guard(LeaseGuard::Testing { + directory, + _root: root, + }) + .build(), + )) + } +} + +pub(super) struct DataDirectory(PathBuf); +impl TryFrom for DataDirectory { + type Error = DirectoryLeaseError; + + fn try_from(value: PathBuf) -> Result { + let encoded = value.as_os_str().as_encoded_bytes(); + if encoded.len() > 4096 + || encoded.contains(&0) + || value.components().count() > 128 + || !value.is_absolute() + || value + .components() + .any(|part| matches!(part, Component::ParentDir)) + { + return Err(DirectoryLeaseError::Configuration); + } + Ok(Self(value.components().collect())) + } +} + +struct Control { + deadline: Instant, + cancelled: Arc, +} +impl Control { + fn check(&self) -> Result<(), DirectoryLeaseError> { + if self.cancelled.load(Ordering::Acquire) { + Err(DirectoryLeaseError::Cancelled) + } else if Instant::now() >= self.deadline { + Err(DirectoryLeaseError::Deadline) + } else { + Ok(()) + } + } +} +struct CancelOnDrop(Arc); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +async fn run( + pool: &'static Semaphore, + deadline: Instant, + work: impl FnOnce(&Control) -> Result + Send + 'static, +) -> Result { + Handle::try_current().map_err(|_| DirectoryLeaseError::Worker)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or(DirectoryLeaseError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DirectoryLeaseError::Configuration); + } + let permit = timeout_at(deadline, pool.acquire()) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + let cancelled = Arc::new(AtomicBool::new(false)); + let _cancel = CancelOnDrop(cancelled.clone()); + let task = spawn_work( + permit, + Control { + deadline, + cancelled, + }, + work, + ); + let (value, _permit) = timeout_at(deadline, task) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + if Instant::now() >= deadline { + return Err(DirectoryLeaseError::Deadline); + } + value +} + +fn spawn_work( + permit: SemaphorePermit<'static>, + control: Control, + work: impl FnOnce(&Control) -> Result + Send + 'static, +) -> JoinHandle<(Result, SemaphorePermit<'static>)> { + spawn_blocking(move || { + let result = (|| { + control.check()?; + let value = work(&control)?; + control.check()?; + Ok(value) + })(); + (result, permit) + }) +} + +fn acquire( + directory: &DataDirectory, + control: &Control, +) -> Result { + acquire_profile(directory, control, Provenance::Journaled) +} + +fn acquire_profile( + directory: &DataDirectory, + control: &Control, + requested: Provenance, +) -> Result { + control.check()?; + let root = open_root(&directory.0, control)?; + match mkdirat(&root, NAMESPACE, Mode::from_bits_truncate(0o700)) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(error.into()), + } + control.check()?; + let namespace = File::from(openat( + &root, + NAMESPACE, + OFlag::O_RDONLY | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC, + Mode::empty(), + )?); + let metadata = namespace.metadata()?; + if !metadata.is_dir() || !private(&metadata) { + return Err(DirectoryLeaseError::InvalidState); + } + let lock = managed_file(&namespace, LOCK, OFlag::O_RDWR | OFlag::O_CREAT, 0)?; + match lock.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => return Err(DirectoryLeaseError::InUse), + Err(TryLockError::Error(error)) => return Err(error.into()), + } + control.check()?; + let (owner, provenance) = if let Some(owner) = read_owner(&namespace)? { + owner + } else { + pristine_namespace(&namespace, control)?; + (publish_owner(&namespace, control, requested)?, requested) + }; + if requested == Provenance::CodeJournaled && provenance != requested { + return Err(DirectoryLeaseError::InvalidState); + } + codes::check_profile(&namespace, owner, provenance)?; + namespace.sync_all()?; + root.sync_all()?; + control.check()?; + Ok(DirectoryLease(Arc::new( + Lease::builder() + .owner(owner) + .run(Uuid::new_v4()) + .transaction(Arc::new(Semaphore::new(1))) + .code_producer(Arc::new(CodeProducerState::new())) + .provenance(provenance) + .guard(LeaseGuard::Held { + directory: namespace, + _lock: lock, + }) + .build(), + ))) +} + +fn open_root(path: &Path, control: &Control) -> Result { + open_root_with_sync(path, control, File::sync_all) +} + +fn open_root_with_sync( + path: &Path, + control: &Control, + sync: impl FnMut(&File) -> io::Result<()>, +) -> Result { + let (anchor, missing) = root_anchor(path, control)?; + let mut root = File::from(open(&anchor, directory_flags(), Mode::empty())?); + trusted_directory(&root.metadata()?)?; + for component in missing.into_iter().rev() { + control.check()?; + match mkdirat( + &root, + component.as_os_str(), + Mode::from_bits_truncate(0o700), + ) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(error.into()), + } + let child = File::from(openat( + &root, + component.as_os_str(), + directory_flags(), + Mode::empty(), + )?); + trusted_directory(&child.metadata()?)?; + root = child; + } + let resolved = DataDirectory::try_from(canonicalize(path)?)?; + trusted_chain(&resolved.0, control)?; + let observed = File::from(open(&resolved.0, directory_flags(), Mode::empty())?); + let original = root.metadata()?; + let current = observed.metadata()?; + if original.dev() != current.dev() || original.ino() != current.ino() { + return Err(DirectoryLeaseError::InvalidState); + } + sync_ancestors(&resolved.0, control, sync)?; + Ok(root) +} + +fn root_anchor( + path: &Path, + control: &Control, +) -> Result<(PathBuf, Vec), DirectoryLeaseError> { + let mut missing = Vec::new(); + let mut ancestor = path; + loop { + control.check()?; + match symlink_metadata(ancestor) { + Ok(metadata) => { + if ancestor == path && metadata.file_type().is_symlink() { + return Err(DirectoryLeaseError::InvalidState); + } + break; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + missing.push( + ancestor + .file_name() + .ok_or(DirectoryLeaseError::Configuration)? + .to_owned(), + ); + ancestor = ancestor + .parent() + .ok_or(DirectoryLeaseError::Configuration)?; + } + Err(error) => return Err(error.into()), + } + } + trusted_chain(ancestor, control)?; + let resolved = DataDirectory::try_from(canonicalize(ancestor)?)?; + trusted_chain(&resolved.0, control)?; + Ok((resolved.0, missing)) +} + +fn directory_flags() -> OFlag { + OFlag::O_RDONLY | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC +} + +fn trusted_directory(metadata: &Metadata) -> Result<(), DirectoryLeaseError> { + if !metadata.is_dir() + || !trusted_uid(metadata) + || (metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0) + { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(()) +} + +fn trusted_uid(metadata: &Metadata) -> bool { + metadata.uid() == geteuid().as_raw() || metadata.uid() == 0 +} + +fn trusted_chain(path: &Path, control: &Control) -> Result<(), DirectoryLeaseError> { + for ancestor in path.ancestors() { + control.check()?; + let metadata = symlink_metadata(ancestor)?; + if metadata.file_type().is_symlink() { + if !trusted_uid(&metadata) { + return Err(DirectoryLeaseError::InvalidState); + } + } else { + trusted_directory(&metadata)?; + } + } + Ok(()) +} + +fn sync_ancestors( + path: &Path, + control: &Control, + mut sync: impl FnMut(&File) -> io::Result<()>, +) -> Result<(), DirectoryLeaseError> { + let mut device = None; + for ancestor in path.ancestors() { + control.check()?; + let directory = File::from(open(ancestor, directory_flags(), Mode::empty())?); + let metadata = directory.metadata()?; + trusted_directory(&metadata)?; + if device.is_some_and(|expected| expected != metadata.dev()) { + break; + } + device = Some(metadata.dev()); + sync(&directory)?; + } + Ok(()) +} + +fn private(metadata: &Metadata) -> bool { + metadata.uid() == geteuid().as_raw() && metadata.mode() & 0o7077 == 0 +} + +fn managed_file( + directory: &File, + name: &str, + flags: OFlag, + maximum: u64, +) -> Result { + let file = File::from(openat( + directory, + name, + flags | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NONBLOCK | OFlag::O_NOCTTY, + Mode::from_bits_truncate(0o600), + )?); + let metadata = file.metadata()?; + if !metadata.is_file() + || !private(&metadata) + || metadata.nlink() != 1 + || metadata.len() > maximum + { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(file) +} + +fn read_owner(directory: &File) -> Result, DirectoryLeaseError> { + let mut file = match managed_file(directory, OWNER, OFlag::O_RDONLY, OWNER_BYTES as u64) { + Ok(file) => file, + Err(DirectoryLeaseError::Io(error)) if error.kind() == io::ErrorKind::NotFound => { + return Ok(None); + } + Err(error) => return Err(error), + }; + if file.metadata()?.len() != OWNER_BYTES as u64 { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = [0; OWNER_BYTES]; + file.read_exact(&mut bytes)?; + let payload = bytes.get(..24).ok_or(DirectoryLeaseError::InvalidState)?; + let checksum = Sha256::digest(payload); + let expected: &[u8] = checksum.as_ref(); + let provenance = match bytes.get(..8) { + Some(magic) if magic == MAGIC => Provenance::Journaled, + Some(magic) if magic == CODE_MAGIC => Provenance::CodeJournaled, + Some(magic) if magic == LEGACY_MAGIC => Provenance::Legacy, + _ => return Err(DirectoryLeaseError::InvalidState), + }; + if bytes.get(24..) != Some(expected) { + return Err(DirectoryLeaseError::InvalidState); + } + let owner = Uuid::from_slice(bytes.get(8..24).ok_or(DirectoryLeaseError::InvalidState)?) + .map_err(|_| DirectoryLeaseError::InvalidState)?; + if owner.is_nil() { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(Some((owner, provenance))) +} + +fn pristine_namespace(directory: &File, control: &Control) -> Result<(), DirectoryLeaseError> { + let descriptor = openat(directory, ".", directory_flags(), Mode::empty())?; + let mut entries = Dir::from_fd(descriptor)?; + for (count, entry) in entries.iter().enumerate() { + control.check()?; + if count >= 4 { + return Err(DirectoryLeaseError::InvalidState); + } + let entry = entry?; + match entry.file_name().to_bytes() { + b"." | b".." | b"lease.lock" => {} + b"owner.pending" => { + managed_file(directory, PENDING, OFlag::O_RDONLY, OWNER_BYTES as u64)?; + } + _ => return Err(DirectoryLeaseError::InvalidState), + } + } + Ok(()) +} + +fn publish_owner( + directory: &File, + control: &Control, + provenance: Provenance, +) -> Result { + control.check()?; + let owner = Uuid::new_v4(); + let mut bytes = [0; OWNER_BYTES]; + bytes + .get_mut(..8) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(if provenance == Provenance::CodeJournaled { + CODE_MAGIC + } else { + MAGIC + }); + bytes + .get_mut(8..24) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(owner.as_bytes()); + let checksum = Sha256::digest(bytes.get(..24).ok_or(DirectoryLeaseError::InvalidState)?); + bytes + .get_mut(24..) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(checksum.as_ref()); + let mut pending = managed_file( + directory, + PENDING, + OFlag::O_RDWR | OFlag::O_CREAT, + OWNER_BYTES as u64, + )?; + control.check()?; + pending.set_len(0)?; + pending.write_all(&bytes)?; + pending.sync_all()?; + control.check()?; + renameat(directory, PENDING, directory, OWNER)?; + directory.sync_all()?; + Ok(owner) +} + +#[cfg(test)] +#[path = "ownership_tests.rs"] +mod tests; + +/// Exact acquisition job retained by production startup through observer loss. +pub(super) struct DirectoryAcquisition { + code: bool, + cancelled: Arc, + deadline: Instant, + state: AcquisitionState, +} +enum AcquisitionState { + Queued { + directory: DataDirectory, + permit: + Pin, AcquireError>> + Send>>, + }, + Running( + JoinHandle<( + Result, + SemaphorePermit<'static>, + )>, + ), + Consumed, +} +impl fmt::Debug for DirectoryAcquisition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DirectoryAcquisition()") + } +} +impl DirectoryAcquisition { + pub(super) fn begin( + directory: PathBuf, + deadline: Instant, + ) -> Result { + Self::begin_profile(directory, deadline, false) + } + pub(super) fn begin_code( + directory: PathBuf, + deadline: Instant, + ) -> Result { + Self::begin_profile(directory, deadline, true) + } + fn begin_profile( + directory: PathBuf, + deadline: Instant, + code: bool, + ) -> Result { + let directory = DataDirectory::try_from(directory)?; + Handle::try_current().map_err(|_| DirectoryLeaseError::Worker)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or(DirectoryLeaseError::Deadline)?; + if remaining.is_zero() || remaining > Duration::from_secs(60) { + return Err(DirectoryLeaseError::Configuration); + } + Ok(Self { + code, + cancelled: Arc::new(AtomicBool::new(false)), + deadline, + state: AcquisitionState::Queued { + directory, + permit: Box::pin(WORKERS.acquire()), + }, + }) + } + pub(super) fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + pub(super) async fn wait( + &mut self, + deadline: Instant, + ) -> Result, DirectoryLeaseError> { + if matches!(self.state, AcquisitionState::Queued { .. }) + && self.cancelled.load(Ordering::Acquire) + { + self.state = AcquisitionState::Consumed; + return Ok(Err(DirectoryLeaseError::Cancelled)); + } + if let AcquisitionState::Queued { directory, permit } = &mut self.state { + let permit = timeout_at(deadline.min(self.deadline), permit) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + let directory = DataDirectory(directory.0.clone()); + let control = Control { + deadline: self.deadline, + cancelled: self.cancelled.clone(), + }; + let code = self.code; + self.state = AcquisitionState::Running(spawn_work(permit, control, move |control| { + if code { + acquire_profile(&directory, control, Provenance::CodeJournaled) + } else { + acquire(&directory, control) + } + })); + } + let result = match &mut self.state { + AcquisitionState::Running(task) => timeout_at(deadline, task) + .await + .map_err(|_| DirectoryLeaseError::Deadline)?, + _ => return Ok(Err(DirectoryLeaseError::Worker)), + }; + // The exact blocking task is terminal, even on JoinError. Never poll it twice. + self.state = AcquisitionState::Consumed; + Ok(match result { + Ok((result, _permit)) => result, + Err(_) => Err(DirectoryLeaseError::Worker), + }) + } +} +impl Drop for DirectoryAcquisition { + fn drop(&mut self) { + self.cancel(); + } +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/codes.rs b/crates/rustack-lambda-core/src/docker/ownership/codes.rs new file mode 100644 index 00000000..258e14cd --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/codes.rs @@ -0,0 +1,383 @@ +//! Bounded code journal, checked producer transitions and monotonic retirement. + +use std::{collections::BTreeSet, fs::File, io}; + +use nix::fcntl::OFlag; +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + DirectoryLease, DirectoryLeaseError, Provenance, managed_file, + snapshot::{Codec, read}, +}; +use crate::docker::{DockerError, DockerKey, mutation::MutationReceipt}; + +mod identity; + +const MAGIC: &[u8; 8] = b"RSDKCOD1"; +const RECORD_BYTES: usize = 194; +const MAX_RECORDS: usize = 32; +const MAX_BYTES: usize = 25 + MAX_RECORDS * RECORD_BYTES + 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::docker) enum VolumeState { + Planned, + Uncertain, + Known, + Retired, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::docker) enum LoaderState { + Planned, + Uncertain, + Known([u8; 32]), + Retired, +} +#[derive(Debug, Clone, Copy)] +pub(in crate::docker) enum CodeEffect { + Volume, + Loader, +} + +#[derive(Clone, PartialEq, Eq, TypedBuilder)] +pub(in crate::docker) struct CodeRecord { + pub(in crate::docker) run: Uuid, + pub(in crate::docker) environment: Uuid, + resource: Uuid, + revision: Uuid, + zip: [u8; 32], + base: [u8; 32], + fingerprint: [u8; 32], + pub(in crate::docker) volume: VolumeState, + pub(in crate::docker) loader: LoaderState, +} +impl std::fmt::Debug for CodeRecord { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("CodeRecord()") + } +} +pub(in crate::docker) fn volume_key(environment: Uuid) -> Result { + format!("rustack-code-{environment}").parse() +} + +impl CodeRecord { + fn valid(&self) -> bool { + ![self.run, self.environment, self.resource, self.revision] + .iter() + .any(Uuid::is_nil) + && match self.volume { + VolumeState::Planned | VolumeState::Uncertain => { + self.loader == LoaderState::Planned + } + VolumeState::Known => true, + VolumeState::Retired => self.loader == LoaderState::Retired, + } + } + pub(in crate::docker) fn same_identity(&self, next: &Self) -> bool { + let mut identity = self.clone(); + identity.volume = next.volume; + identity.loader = next.loader; + identity == *next + } + fn advances(&self, next: &Self) -> bool { + self.same_identity(next) + && next.valid() + && ((self.volume == next.volume + && matches!( + (self.loader, next.loader), + ( + LoaderState::Planned | LoaderState::Known(_), + LoaderState::Retired + ) | (LoaderState::Planned, LoaderState::Uncertain) + | (LoaderState::Uncertain, LoaderState::Known(_)) + )) + || (self.loader == next.loader + && matches!( + (self.volume, next.volume), + (VolumeState::Planned, VolumeState::Uncertain) + | (VolumeState::Uncertain, VolumeState::Known) + | (VolumeState::Known, VolumeState::Retired) + )) + || (self.volume == VolumeState::Planned + && self.loader == LoaderState::Planned + && next.volume == VolumeState::Retired + && next.loader == LoaderState::Retired)) + } +} + +pub(super) fn check_profile( + directory: &File, + owner: Uuid, + provenance: Provenance, +) -> Result<(), DirectoryLeaseError> { + if provenance == Provenance::CodeJournaled { + read::(directory, owner)?; + } else { + for name in [CodeCodec::PUBLISHED, CodeCodec::STAGING] { + match managed_file(directory, name, OFlag::O_RDONLY, MAX_BYTES as u64) { + Err(DirectoryLeaseError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {} + _ => return Err(DirectoryLeaseError::InvalidState), + } + } + } + Ok(()) +} +impl DirectoryLease { + pub(in crate::docker) async fn code_records( + &self, + deadline: Instant, + ) -> Result, DirectoryLeaseError> { + let provenance = self.0.provenance; + self.transaction(deadline, move |directory, owner, _control| { + if provenance == Provenance::CodeJournaled { + read::(directory, owner) + } else { + check_profile(directory, owner, provenance)?; + Ok(Vec::new()) + } + }) + .await + } + pub(in crate::docker) async fn prepare_code( + &self, + record: CodeRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if !self.code_journaled() + || record.run != self.run_id() + || record.volume != VolumeState::Planned + || record.loader != LoaderState::Planned + || !record.valid() + { + return Err(DirectoryLeaseError::InvalidState); + } + let run = self.run_id(); + self.snapshot::(deadline, move |records| { + if records.iter().any(|old| old.run != run) { + return Err(DirectoryLeaseError::InvalidState); + } + if let Some(old) = records + .iter() + .find(|old| old.environment == record.environment) + { + return if *old == record { + Ok(()) + } else { + Err(DirectoryLeaseError::InvalidState) + }; + } + if records.len() >= MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + records.push(record); + Ok(()) + }) + .await + } + pub(in crate::docker) async fn abandon_code( + &self, + record: CodeRecord, + effect: CodeEffect, + receipt: &MutationReceipt, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if !self.code_journaled() || record.run != self.run_id() || !receipt.definitely_unissued() { + return Err(DirectoryLeaseError::InvalidState); + } + let mut next = record.clone(); + match (effect, record.volume, record.loader) { + (CodeEffect::Volume, VolumeState::Uncertain, LoaderState::Planned) => { + next.volume = VolumeState::Retired; + next.loader = LoaderState::Retired; + } + (CodeEffect::Loader, VolumeState::Known, LoaderState::Uncertain) => { + next.loader = LoaderState::Retired; + } + _ => return Err(DirectoryLeaseError::InvalidState), + } + self.replace_code(record, next, deadline).await + } + pub(in crate::docker) async fn advance_code( + &self, + old: CodeRecord, + next: CodeRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + let arming = (old.volume == VolumeState::Planned && next.volume == VolumeState::Uncertain) + || (old.loader == LoaderState::Planned && next.loader == LoaderState::Uncertain); + if !self.code_journaled() || !old.advances(&next) || (arming && old.run != self.run_id()) { + return Err(DirectoryLeaseError::InvalidState); + } + self.replace_code(old, next, deadline).await + } + async fn replace_code( + &self, + old: CodeRecord, + next: CodeRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + self.snapshot::(deadline, move |records| { + let saved = records + .iter_mut() + .find(|record| record.environment == old.environment) + .ok_or(DirectoryLeaseError::InvalidState)?; + if *saved == next { + return Ok(()); + } + if *saved != old { + return Err(DirectoryLeaseError::InvalidState); + } + *saved = next; + Ok(()) + }) + .await + } + pub(in crate::docker) async fn settle_code( + &self, + record: CodeRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if !self.code_journaled() + || record.volume != VolumeState::Retired + || record.loader != LoaderState::Retired + { + return Err(DirectoryLeaseError::InvalidState); + } + self.snapshot::(deadline, move |records| { + if let Some(position) = records + .iter() + .position(|saved| saved.environment == record.environment) + { + if records.get(position) != Some(&record) { + return Err(DirectoryLeaseError::InvalidState); + } + records.remove(position); + } + Ok(()) + }) + .await + } +} + +pub(super) struct CodeCodec; +impl Codec for CodeCodec { + type Record = CodeRecord; + const PUBLISHED: &'static str = "code.v1"; + const STAGING: &'static str = "code.pending"; + const MAX_BYTES: usize = MAX_BYTES; + fn encode(owner: Uuid, records: &[CodeRecord]) -> Result, DirectoryLeaseError> { + if owner.is_nil() || records.len() > MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = Vec::with_capacity(MAX_BYTES); + bytes.extend_from_slice(MAGIC); + bytes.extend_from_slice(owner.as_bytes()); + bytes.push(u8::try_from(records.len()).map_err(|_| DirectoryLeaseError::InvalidState)?); + for record in records { + for id in [ + record.run, + record.environment, + record.resource, + record.revision, + ] { + bytes.extend_from_slice(id.as_bytes()); + } + for digest in [record.zip, record.base, record.fingerprint] { + bytes.extend_from_slice(&digest); + } + bytes.push(match record.volume { + VolumeState::Planned => 0, + VolumeState::Uncertain => 1, + VolumeState::Known => 2, + VolumeState::Retired => 3, + }); + bytes.push(match record.loader { + LoaderState::Planned => 0, + LoaderState::Uncertain => 1, + LoaderState::Known(_) => 2, + LoaderState::Retired => 3, + }); + bytes.extend_from_slice(&match record.loader { + LoaderState::Known(id) => id, + _ => [0; 32], + }); + } + bytes.extend_from_slice(&Sha256::digest(&bytes)); + Self::decode(&bytes, owner)?; + Ok(bytes) + } + fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + let invalid = || DirectoryLeaseError::InvalidState; + let count = usize::from(*bytes.get(24).ok_or_else(invalid)?); + let body_len = count + .checked_mul(RECORD_BYTES) + .and_then(|n| n.checked_add(25)) + .ok_or_else(invalid)?; + let expected_len = body_len.checked_add(32).ok_or_else(invalid)?; + if owner.is_nil() + || count > MAX_RECORDS + || bytes.len() != expected_len + || bytes.get(..8) != Some(MAGIC.as_slice()) + || bytes.get(8..24) != Some(owner.as_bytes().as_slice()) + || bytes.get(body_len..) + != Some(Sha256::digest(bytes.get(..body_len).ok_or_else(invalid)?).as_slice()) + { + return Err(invalid()); + } + let mut records = Vec::with_capacity(count); + let mut identities = BTreeSet::new(); + for row in bytes + .get(25..body_len) + .ok_or_else(invalid)? + .as_chunks::() + .0 + { + let uuid = |start| { + Uuid::from_slice(row.get(start..start + 16).ok_or_else(invalid)?) + .map_err(|_| invalid()) + }; + let digest = |start| -> Result<[u8; 32], DirectoryLeaseError> { + row.get(start..start + 32) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid()) + }; + let volume = match row.get(160) { + Some(0) => VolumeState::Planned, + Some(1) => VolumeState::Uncertain, + Some(2) => VolumeState::Known, + Some(3) => VolumeState::Retired, + _ => return Err(invalid()), + }; + let id = digest(162)?; + let loader = match row.get(161) { + Some(0) if id == [0; 32] => LoaderState::Planned, + Some(1) if id == [0; 32] => LoaderState::Uncertain, + Some(2) => LoaderState::Known(id), + Some(3) if id == [0; 32] => LoaderState::Retired, + _ => return Err(invalid()), + }; + let record = CodeRecord::builder() + .run(uuid(0)?) + .environment(uuid(16)?) + .resource(uuid(32)?) + .revision(uuid(48)?) + .zip(digest(64)?) + .base(digest(96)?) + .fingerprint(digest(128)?) + .volume(volume) + .loader(loader) + .build(); + if !record.valid() || !identities.insert(record.environment) { + return Err(invalid()); + } + records.push(record); + } + Ok(records) + } +} + +#[cfg(test)] +pub(in crate::docker) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership/codes/identity.rs b/crates/rustack-lambda-core/src/docker/ownership/codes/identity.rs new file mode 100644 index 00000000..bd8be09a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/codes/identity.rs @@ -0,0 +1,119 @@ +//! Exact recorded identity checks before any code-resource deletion. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use super::{CodeRecord, LoaderState}; +use crate::docker::{ + DockerError, DockerKey, + models::{ContainerInspectResponse, ContainerStateStatusEnum, Volume, VolumeScopeEnum}, +}; + +impl CodeRecord { + pub(in crate::docker) fn volume_key(&self) -> Result { + super::volume_key(self.environment) + } + pub(in crate::docker) fn loader_key(&self) -> Result { + match self.loader { + LoaderState::Known(id) => hex::encode(id).parse(), + _ => format!("rustack-code-loader-{}", self.environment).parse(), + } + } + pub(in crate::docker) fn labels(&self, owner: Uuid, loader: bool) -> BTreeMap { + [ + ("ownerId", owner.to_string()), + ("runId", self.run.to_string()), + ("environmentId", self.environment.to_string()), + ("functionResourceId", self.resource.to_string()), + ("revisionId", self.revision.to_string()), + ("codeSha256", hex::encode(self.zip)), + ("baseFingerprint", hex::encode(self.fingerprint)), + ("role", if loader { "code-loader" } else { "code" }.into()), + ] + .into_iter() + .map(|(key, value)| (format!("rustack.lambda.{key}"), value)) + .collect() + } + pub(in crate::docker) fn owned_volume( + &self, + owner: Uuid, + volume: &Volume, + ) -> Result<(), DockerError> { + let labels = self.labels(owner, false); + if volume.name != self.volume_key()?.as_str() + || volume.driver != "local" + || volume.scope != Some(VolumeScopeEnum::LOCAL) + || !volume.options.is_empty() + || volume.labels.len() != labels.len() + || !labels + .iter() + .all(|(key, value)| volume.labels.get(key) == Some(value)) + { + return Err(DockerError::Quarantined); + } + Ok(()) + } + pub(in crate::docker) fn owned_loader( + &self, + owner: Uuid, + inspected: &ContainerInspectResponse, + ) -> Result<[u8; 32], DockerError> { + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let labels = config.labels.as_ref().ok_or(DockerError::Quarantined)?; + let expected = self.labels(owner, true); + let image = format!("sha256:{}", hex::encode(self.base)); + let id = inspected.id.as_deref().ok_or(DockerError::Protocol)?; + let mut decoded = [0; 32]; + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DockerError::Quarantined); + } + hex::decode_to_slice(id, &mut decoded).map_err(|_| DockerError::Quarantined)?; + let state = inspected.state.as_ref().ok_or(DockerError::Protocol)?; + let mounts = inspected + .mounts + .as_deref() + .ok_or(DockerError::Quarantined)?; + let [mount] = mounts else { + return Err(DockerError::Quarantined); + }; + if matches!(self.loader, LoaderState::Known(known) if known != decoded) + || inspected.name.as_deref() + != Some(format!("/rustack-code-loader-{}", self.environment).as_str()) + || config.image.as_deref() != Some(image.as_str()) + || inspected.image.as_deref() != Some(image.as_str()) + || labels.len() > 64 + || labels.iter().any(|(key, value)| { + key.is_empty() + || key.len() > 128 + || value.len() > 256 + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) + || value.chars().any(char::is_control) + || (key.starts_with("rustack.lambda.") && expected.get(key) != Some(value)) + }) + || !expected + .iter() + .all(|(key, value)| labels.get(key) == Some(value)) + || state.running != Some(false) + || state.status != Some(ContainerStateStatusEnum::CREATED) + || !matches!( + state.started_at.as_deref(), + None | Some("" | "0001-01-01T00:00:00Z" | "0001-01-01T00:00:00.000000000Z") + ) + || mount.typ.as_deref() != Some("volume") + || mount.driver.as_deref() != Some("local") + || mount.name.as_deref() != Some(self.volume_key()?.as_str()) + || mount.destination.as_deref() != Some("/var/task") + || mount.rw != Some(true) + { + return Err(DockerError::Quarantined); + } + Ok(decoded) + } +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/codes/tests.rs b/crates/rustack-lambda-core/src/docker/ownership/codes/tests.rs new file mode 100644 index 00000000..38d141d7 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/codes/tests.rs @@ -0,0 +1,334 @@ +use std::{ + fs::Permissions, + os::unix::fs::{PermissionsExt, symlink}, + time::Duration, +}; + +use rstest::rstest; +use tokio::{ + fs::{read, set_permissions, write}, + time::Instant, +}; +use uuid::Uuid; + +use super::*; +use crate::docker::{ + binding::DaemonIdentity, + ownership::{LEGACY_MAGIC, MAGIC, NAMESPACE, OWNER}, + tests::DAEMON_ID, +}; + +pub(in crate::docker) fn record(run: Uuid, volume: VolumeState, loader: LoaderState) -> CodeRecord { + CodeRecord::builder() + .run(run) + .environment(Uuid::new_v4()) + .resource(Uuid::new_v4()) + .revision(Uuid::new_v4()) + .zip([0x11; 32]) + .base([0xaa; 32]) + .fingerprint([0xbb; 32]) + .volume(volume) + .loader(loader) + .build() +} +pub(in crate::docker) async fn publish(directory: &DirectoryLease, records: Vec) { + assert!(directory.code_journaled()); + directory + .snapshot::(deadline(), move |saved| { + *saved = records; + Ok(()) + }) + .await + .unwrap(); +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn signed(bytes: &mut [u8]) { + let end = bytes.len() - 32; + let sum = Sha256::digest(&bytes[..end]); + bytes[end..].copy_from_slice(&sum); +} + +#[rstest] +#[case(0)] +#[case(8)] +#[case(24)] +#[case(25)] +#[case(41)] +#[case(57)] +#[case(73)] +#[case(185)] +#[case(186)] +#[case(187)] +fn test_should_reject_corrupt_canonical_fields_even_with_recomputed_checksum( + #[case] offset: usize, +) { + // Make the former 0xff owner-byte no-op a deterministic regression case. + let owner = Uuid::from_u128(u128::MAX); + let row = record(Uuid::new_v4(), VolumeState::Known, LoaderState::Planned); + let mut bytes = CodeCodec::encode(owner, &[row]).unwrap(); + if matches!(offset, 25 | 41 | 57 | 73) { + bytes[offset..offset + 16].fill(0); + } else if offset == 8 { + bytes[offset] ^= 0xff; + } else { + bytes[offset] = 0xff; + } + signed(&mut bytes); + assert!(matches!( + CodeCodec::decode(&bytes, owner), + Err(DirectoryLeaseError::InvalidState) + )); +} +#[rstest] +#[case(0)] +#[case(1)] +#[case(10)] +#[case(24)] +#[case(25)] +#[case(218)] +#[case(250)] +fn test_should_reject_truncation_and_appended_data(#[case] length: usize) { + let owner = Uuid::new_v4(); + let row = record( + Uuid::new_v4(), + VolumeState::Known, + LoaderState::Known([0xcc; 32]), + ); + let bytes = CodeCodec::encode(owner, &[row]).unwrap(); + assert_eq!(bytes.len(), 251); + assert!(CodeCodec::decode(&bytes[..length], owner).is_err()); + let mut extra = bytes.clone(); + extra.push(0); + assert!(CodeCodec::decode(&extra, owner).is_err()); +} +#[test] +fn test_should_bound_records_and_reject_duplicate_environments_and_wrong_owner() { + let owner = Uuid::new_v4(); + let rows = (0..32) + .map(|_| record(Uuid::new_v4(), VolumeState::Known, LoaderState::Uncertain)) + .collect::>(); + let bytes = CodeCodec::encode(owner, &rows).unwrap(); + assert_eq!(bytes.len(), 6265); + assert_eq!(CodeCodec::decode(&bytes, owner).unwrap(), rows); + assert!(CodeCodec::decode(&bytes, Uuid::new_v4()).is_err()); + let mut too_many = rows.clone(); + too_many.push(rows[0].clone()); + assert!(CodeCodec::encode(owner, &too_many).is_err()); + let mut duplicate = rows[0].clone(); + duplicate.run = Uuid::new_v4(); + assert!(CodeCodec::encode(owner, &[rows[0].clone(), duplicate]).is_err()); + assert!(CodeCodec::encode(Uuid::nil(), &[]).is_err()); + let mut damaged = bytes; + damaged[100] ^= 1; + assert!(CodeCodec::decode(&damaged, owner).is_err()); +} +#[rstest] +#[case(VolumeState::Planned, LoaderState::Uncertain)] +#[case(VolumeState::Planned, LoaderState::Retired)] +#[case(VolumeState::Uncertain, LoaderState::Known([1;32]))] +#[case(VolumeState::Retired, LoaderState::Planned)] +#[case(VolumeState::Retired, LoaderState::Known([1;32]))] +fn test_should_reject_impossible_effect_states( + #[case] volume: VolumeState, + #[case] loader: LoaderState, +) { + assert!(CodeCodec::encode(Uuid::new_v4(), &[record(Uuid::new_v4(), volume, loader)]).is_err()); +} +#[tokio::test] +async fn test_should_create_explicit_code_provenance_and_reopen_without_rewriting_owner() { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + lease.require_journaled().unwrap(); + assert!(lease.code_journaled()); + let owner = lease.owner_id(); + let run = lease.run_id(); + let path = root.path().join(NAMESPACE); + let before = read(path.join(OWNER)).await.unwrap(); + assert_eq!(&before[..8], b"RSDKOWN3"); + let row = record(run, VolumeState::Known, LoaderState::Known([0xcc; 32])); + publish(&lease, vec![row.clone()]).await; + drop(lease); + let lease = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + assert_eq!(owner, lease.owner_id()); + assert_ne!(run, lease.run_id()); + assert_eq!(lease.code_records(deadline()).await.unwrap(), [row]); + assert_eq!(read(path.join(OWNER)).await.unwrap(), before); +} +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_refuse_old_roots_without_automatic_upgrade(#[case] legacy: bool) { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + drop(lease); + let path = root.path().join(NAMESPACE).join(OWNER); + let mut bytes = read(&path).await.unwrap(); + if legacy { + bytes[..8].copy_from_slice(LEGACY_MAGIC); + signed(&mut bytes); + write(&path, &bytes).await.unwrap(); + } + assert!(matches!( + DirectoryLease::acquire_code(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(read(path).await.unwrap(), bytes); + assert_eq!(&bytes[..8], if legacy { LEGACY_MAGIC } else { MAGIC }); +} +#[rstest] +#[case("code.v1")] +#[case("code.pending")] +#[tokio::test] +async fn test_should_refuse_any_code_file_in_image_namespace_on_acquire_and_binding( + #[case] name: &str, +) { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let identity = DaemonIdentity::try_from(DAEMON_ID).unwrap(); + lease.bind_daemon(identity, deadline()).await.unwrap(); + let path = root.path().join(NAMESPACE).join(name); + write(&path, []).await.unwrap(); + set_permissions(&path, Permissions::from_mode(0o600)) + .await + .unwrap(); + assert!(lease.bind_daemon(identity, deadline()).await.is_err()); + assert!(lease.code_records(deadline()).await.is_err()); + drop(lease); + assert!( + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .is_err() + ); +} +#[rstest] +#[case("code.v1")] +#[case("code.pending")] +#[tokio::test] +async fn test_should_refuse_unsafe_code_files_and_preserve_outside_content(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + let outside = root.path().join("outside"); + write(&outside, b"preserve").await.unwrap(); + let path = root.path().join(NAMESPACE).join(name); + symlink(&outside, &path).unwrap(); + assert!(lease.code_records(deadline()).await.is_err()); + tokio::fs::remove_file(&path).await.unwrap(); + write(&path, vec![0; MAX_BYTES + 1]).await.unwrap(); + set_permissions(&path, Permissions::from_mode(0o600)) + .await + .unwrap(); + assert!(lease.code_records(deadline()).await.is_err()); + assert_eq!(read(outside).await.unwrap(), b"preserve"); +} +#[tokio::test] +async fn test_should_tolerate_private_torn_staging_but_refuse_missing_daemon_binding_over_code() { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + let row = record(lease.run_id(), VolumeState::Known, LoaderState::Planned); + publish(&lease, vec![row.clone()]).await; + let path = root.path().join(NAMESPACE).join("code.pending"); + write(&path, b"torn").await.unwrap(); + set_permissions(&path, Permissions::from_mode(0o600)) + .await + .unwrap(); + assert_eq!(lease.code_records(deadline()).await.unwrap(), [row]); + assert!( + lease + .bind_daemon(DaemonIdentity::try_from(DAEMON_ID).unwrap(), deadline()) + .await + .is_err() + ); + assert!(!root.path().join(NAMESPACE).join("daemon.v1").exists()); +} +#[tokio::test] +async fn test_should_compare_and_swap_monotonic_recovery_state_and_reject_early_erasure() { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + let row = record(lease.run_id(), VolumeState::Known, LoaderState::Uncertain); + publish(&lease, vec![row.clone()]).await; + assert!(lease.settle_code(row.clone(), deadline()).await.is_err()); + let mut next = row.clone(); + next.loader = LoaderState::Known([0xcc; 32]); + lease + .advance_code(row.clone(), next.clone(), deadline()) + .await + .unwrap(); + lease + .advance_code(row.clone(), next.clone(), deadline()) + .await + .unwrap(); + let mut conflict = next.clone(); + conflict.loader = LoaderState::Known([0xdd; 32]); + assert!(lease.advance_code(row, conflict, deadline()).await.is_err()); + let mut retired = next.clone(); + retired.loader = LoaderState::Retired; + lease + .advance_code(next, retired.clone(), deadline()) + .await + .unwrap(); + let mut done = retired.clone(); + done.volume = VolumeState::Retired; + lease + .advance_code(retired, done.clone(), deadline()) + .await + .unwrap(); + lease.settle_code(done.clone(), deadline()).await.unwrap(); + lease.settle_code(done, deadline()).await.unwrap(); + assert_eq!(lease.code_records(deadline()).await.unwrap(), []); +} + +#[rstest] +#[case("code.v1", false)] +#[case("code.pending", false)] +#[case("code.v1", true)] +#[case("code.pending", true)] +#[tokio::test] +async fn test_should_reject_public_mode_and_hardlinked_code_files( + #[case] name: &str, + #[case] hardlink: bool, +) { + let root = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + let path = root.path().join(NAMESPACE).join(name); + if hardlink { + let outside = root.path().join("outside"); + write(&outside, []).await.unwrap(); + set_permissions(&outside, Permissions::from_mode(0o600)) + .await + .unwrap(); + tokio::fs::hard_link(&outside, &path).await.unwrap(); + } else { + write(&path, []).await.unwrap(); + set_permissions(&path, Permissions::from_mode(0o644)) + .await + .unwrap(); + } + assert!(matches!( + lease.code_records(deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + drop(lease); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/daemon.rs b/crates/rustack-lambda-core/src/docker/ownership/daemon.rs new file mode 100644 index 00000000..d6ef9cc3 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/daemon.rs @@ -0,0 +1,91 @@ +//! Immutable owner/engine association, published before any journaled effect. + +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + DirectoryLease, DirectoryLeaseError, + codes::{CodeCodec, check_profile}, + images::ImageCodec, + journal::ContainerCodec, + snapshot::{Codec, read, write}, +}; +use crate::docker::binding::DaemonIdentity; + +const MAGIC: &[u8; 8] = b"RSDKDMD1"; +const BYTES: usize = 72; + +impl DirectoryLease { + pub(in crate::docker) async fn bind_daemon( + &self, + identity: DaemonIdentity, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + let provenance = self.0.provenance; + self.transaction(deadline, move |directory, owner, control| { + check_profile(directory, owner, provenance)?; + let existing = read::(directory, owner)?; + control.check()?; + match existing.as_slice() { + [saved] if *saved == identity => Ok(()), + [] => { + if !read::(directory, owner)?.is_empty() + || !read::(directory, owner)?.is_empty() + || !read::(directory, owner)?.is_empty() + { + return Err(DirectoryLeaseError::InvalidState); + } + write::(directory, owner, &[identity], control) + } + _ => Err(DirectoryLeaseError::InvalidState), + } + }) + .await + } +} + +#[derive(Debug)] +struct DaemonCodec; +impl Codec for DaemonCodec { + type Record = DaemonIdentity; + const PUBLISHED: &'static str = "daemon.v1"; + const STAGING: &'static str = "daemon.pending"; + const MAX_BYTES: usize = BYTES; + + fn encode(owner: Uuid, records: &[DaemonIdentity]) -> Result, DirectoryLeaseError> { + let [identity] = records else { + return Err(DirectoryLeaseError::InvalidState); + }; + if owner.is_nil() { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = Vec::with_capacity(BYTES); + bytes.extend_from_slice(MAGIC); + bytes.extend_from_slice(owner.as_bytes()); + bytes.extend_from_slice(identity.uuid().as_bytes()); + bytes.extend_from_slice(&Sha256::digest(&bytes)); + Ok(bytes) + } + + fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + let invalid = || DirectoryLeaseError::InvalidState; + if owner.is_nil() + || bytes.len() != BYTES + || bytes.get(..8) != Some(MAGIC.as_slice()) + || bytes.get(8..24) != Some(owner.as_bytes().as_slice()) + || bytes.get(40..) + != Some(Sha256::digest(bytes.get(..40).ok_or_else(invalid)?).as_slice()) + { + return Err(invalid()); + } + let uuid = + Uuid::from_slice(bytes.get(24..40).ok_or_else(invalid)?).map_err(|_| invalid())?; + Ok(vec![ + DaemonIdentity::from_uuid(uuid).map_err(|_| invalid())?, + ]) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership/daemon/tests.rs b/crates/rustack-lambda-core/src/docker/ownership/daemon/tests.rs new file mode 100644 index 00000000..5d033d65 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/daemon/tests.rs @@ -0,0 +1,207 @@ +use std::{ + fs::Permissions, + os::unix::fs::{PermissionsExt, symlink}, + sync::{Arc, mpsc::channel as blocking_channel}, + time::Duration, +}; + +use rstest::rstest; +use sha2::{Digest, Sha256}; +use tokio::{ + fs::{read, remove_file, set_permissions, write}, + spawn, + sync::oneshot::channel as oneshot_channel, + time::{Instant, timeout}, +}; +use uuid::Uuid; + +use super::{BYTES, DaemonCodec}; +use crate::docker::{ + DirectoryLease, DirectoryLeaseError, EnvironmentPlan, ImagePinScope, + binding::DaemonIdentity, + ownership::{ + ContainerRecord, ImageRecord, NAMESPACE, + snapshot::{Codec, write as write_snapshot}, + }, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + tests::DAEMON_ID, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn identity() -> DaemonIdentity { + DaemonIdentity::try_from(DAEMON_ID).unwrap() +} + +#[rstest] +#[case(0)] +#[case(1)] +#[case(2)] +#[case(3)] +#[case(4)] +#[case(5)] +#[case(6)] +fn test_should_reject_damaged_daemon_binding(#[case] corruption: u8) { + let owner = Uuid::new_v4(); + let mut bytes = DaemonCodec::encode(owner, &[identity()]).unwrap(); + assert_eq!(bytes.len(), BYTES); + assert_eq!(DaemonCodec::decode(&bytes, owner).unwrap(), [identity()]); + match corruption { + 0 => bytes[0] ^= 1, + 1 => bytes[8] ^= 1, + 2 => bytes[24..40].fill(0), + 3 => { + bytes.pop(); + } + 4 => bytes.push(0), + 5 => bytes[40] ^= 1, + _ => bytes[8..24].fill(0), + } + if matches!(corruption, 0 | 1 | 2 | 6) { + let sum = Sha256::digest(&bytes[..40]); + bytes[40..].copy_from_slice(&sum); + } + assert!(matches!( + DaemonCodec::decode(&bytes, owner), + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[tokio::test] +async fn test_should_persist_immutable_identity_across_reacquisition_and_torn_staging() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let owner = directory.owner_id(); + directory.bind_daemon(identity(), deadline()).await.unwrap(); + let path = root.path().join(NAMESPACE); + let before = read(path.join("daemon.v1")).await.unwrap(); + write(path.join("daemon.pending"), b"torn").await.unwrap(); + set_permissions(path.join("daemon.pending"), Permissions::from_mode(0o600)) + .await + .unwrap(); + directory.bind_daemon(identity(), deadline()).await.unwrap(); + drop(directory); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + assert_eq!(directory.owner_id(), owner); + directory.bind_daemon(identity(), deadline()).await.unwrap(); + let other = DaemonIdentity::from_uuid(Uuid::new_v4()).unwrap(); + assert!(matches!( + directory.bind_daemon(other, deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(read(path.join("daemon.v1")).await.unwrap(), before); +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_missing_binding_over_any_journal(#[case] container: bool) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + if container { + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + directory + .prepare_container( + ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(), + deadline(), + ) + .await + .unwrap(); + } else { + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let record = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + directory.prepare_image(record, deadline()).await.unwrap(); + } + assert!(matches!( + directory.bind_daemon(identity(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!(!root.path().join(NAMESPACE).join("daemon.v1").exists()); +} + +#[rstest] +#[case("daemon.v1")] +#[case("daemon.pending")] +#[tokio::test] +async fn test_should_refuse_unsafe_or_oversized_binding_files(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let outside = root.path().join("outside"); + write(&outside, b"preserve").await.unwrap(); + let path = root.path().join(NAMESPACE).join(name); + symlink(&outside, &path).unwrap(); + assert!(directory.bind_daemon(identity(), deadline()).await.is_err()); + remove_file(&path).await.unwrap(); + write(&path, vec![0; BYTES + 1]).await.unwrap(); + set_permissions(&path, Permissions::from_mode(0o600)) + .await + .unwrap(); + assert!(directory.bind_daemon(identity(), deadline()).await.is_err()); + assert_eq!(read(outside).await.unwrap(), b"preserve"); +} + +#[tokio::test] +async fn test_should_retain_published_binding_and_serial_lease_after_observer_loss() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let retained = directory.clone(); + let (entered, observed) = oneshot_channel(); + let (release, barrier) = blocking_channel(); + let task = spawn(async move { + retained + .transaction(deadline(), move |directory, owner, control| { + write_snapshot::(directory, owner, &[identity()], control)?; + entered.send(()).unwrap(); + barrier.recv().unwrap(); + Ok(()) + }) + .await + }); + observed.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let copy = directory.clone(); + let other = DaemonIdentity::from_uuid(Uuid::new_v4()).unwrap(); + let mut retry = spawn(async move { copy.bind_daemon(other, deadline()).await }); + assert!( + timeout(Duration::from_millis(30), &mut retry) + .await + .is_err() + ); + drop(directory); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + release.send(()).unwrap(); + assert!(matches!( + retry.await.unwrap(), + Err(DirectoryLeaseError::InvalidState) + )); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + next.bind_daemon(identity(), deadline()).await.unwrap(); +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/images.rs b/crates/rustack-lambda-core/src/docker/ownership/images.rs new file mode 100644 index 00000000..611dcf85 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/images.rs @@ -0,0 +1,240 @@ +//! Persistent private image-tag intent; unknown absence never proves settlement. + +use std::{collections::BTreeSet, fmt}; + +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{DirectoryLease, DirectoryLeaseError, snapshot::Codec}; +use crate::{ + artifact::Sha256Digest, + docker::{DockerError, ImagePin}, +}; + +const PUBLISHED: &str = "pins.v1"; +const STAGING: &str = "pins.pending"; +const MAGIC: &[u8; 8] = b"RSDKPIN1"; +const RECORD_BYTES: usize = 65; +const MAX_RECORDS: usize = 8192; +const MAX_BYTES: usize = 26 + MAX_RECORDS * RECORD_BYTES + 32; + +#[derive(Clone, PartialEq, Eq)] +pub(in crate::docker) struct ImageRecord { + pub(in crate::docker) run: Uuid, + pin: Uuid, + image: [u8; 32], + pub(in crate::docker) committed: bool, +} +impl fmt::Debug for ImageRecord { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ImageRecord()") + } +} +impl ImageRecord { + pub(in crate::docker) fn new( + pin: &ImagePin, + image: &Sha256Digest, + ) -> Result { + let mut digest = [0; 32]; + hex::decode_to_slice( + image + .as_str() + .strip_prefix("sha256:") + .ok_or(DockerError::Configuration)?, + &mut digest, + ) + .map_err(|_| DockerError::Configuration)?; + Ok(Self { + run: pin.run, + pin: pin.id, + image: digest, + committed: false, + }) + } + pub(in crate::docker) fn pin( + &self, + directory: &DirectoryLease, + ) -> Result { + ImagePin::historical(directory.clone(), self.run, self.pin) + } + pub(in crate::docker) fn expected(&self) -> Result { + format!("sha256:{}", hex::encode(self.image)) + .parse() + .map_err(DockerError::from) + } + fn same_key(&self, other: &Self) -> bool { + self.run == other.run && self.pin == other.pin + } + fn identity(&self, other: &Self) -> bool { + self.same_key(other) && self.image == other.image + } +} +impl DirectoryLease { + pub(in crate::docker) async fn image_records( + &self, + deadline: Instant, + ) -> Result, DirectoryLeaseError> { + self.snapshot::(deadline, |records| Ok(records.clone())) + .await + } + pub(in crate::docker) async fn prepare_image( + &self, + record: ImageRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if record.run != self.run_id() { + return Err(DirectoryLeaseError::InvalidState); + } + self.snapshot::(deadline, move |records| { + if let Some(old) = records.iter().find(|old| old.same_key(&record)) { + if !old.identity(&record) { + return Err(DirectoryLeaseError::InvalidState); + } + } else { + if records.len() >= MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + records.push(record); + } + Ok(()) + }) + .await + } + pub(in crate::docker) async fn commit_image( + &self, + record: ImageRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + self.snapshot::(deadline, move |records| { + let old = records + .iter_mut() + .find(|old| old.same_key(&record)) + .ok_or(DirectoryLeaseError::InvalidState)?; + if !old.identity(&record) { + return Err(DirectoryLeaseError::InvalidState); + } + old.committed = true; + Ok(()) + }) + .await + } + pub(in crate::docker) async fn settle_image( + &self, + record: ImageRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + self.snapshot::(deadline, move |records| { + if let Some(position) = records.iter().position(|old| old.same_key(&record)) { + if !records + .get(position) + .is_some_and(|old| old.identity(&record)) + { + return Err(DirectoryLeaseError::InvalidState); + } + records.remove(position); + } + Ok(()) + }) + .await + } +} +pub(super) struct ImageCodec; +impl Codec for ImageCodec { + type Record = ImageRecord; + const PUBLISHED: &'static str = PUBLISHED; + const STAGING: &'static str = STAGING; + const MAX_BYTES: usize = MAX_BYTES; + fn encode(owner: Uuid, records: &[ImageRecord]) -> Result, DirectoryLeaseError> { + encode(owner, records) + } + fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + decode(bytes, owner) + } +} +fn encode(owner: Uuid, records: &[ImageRecord]) -> Result, DirectoryLeaseError> { + if owner.is_nil() || records.len() > MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + let capacity = records + .len() + .checked_mul(RECORD_BYTES) + .and_then(|bytes| bytes.checked_add(58)) + .ok_or(DirectoryLeaseError::InvalidState)?; + let mut bytes = Vec::with_capacity(capacity); + bytes.extend_from_slice(MAGIC); + bytes.extend_from_slice(owner.as_bytes()); + bytes.extend_from_slice( + &u16::try_from(records.len()) + .map_err(|_| DirectoryLeaseError::InvalidState)? + .to_be_bytes(), + ); + for record in records { + bytes.extend_from_slice(record.run.as_bytes()); + bytes.extend_from_slice(record.pin.as_bytes()); + bytes.extend_from_slice(&record.image); + bytes.push(u8::from(record.committed)); + } + bytes.extend_from_slice(&Sha256::digest(&bytes)); + decode(&bytes, owner)?; + Ok(bytes) +} +fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + let invalid = || DirectoryLeaseError::InvalidState; + let count = usize::from(u16::from_be_bytes( + bytes + .get(24..26) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?, + )); + if owner.is_nil() || count > MAX_RECORDS { + return Err(invalid()); + } + let body_len = count + .checked_mul(RECORD_BYTES) + .and_then(|bytes| bytes.checked_add(26)) + .ok_or_else(invalid)?; + let expected_len = body_len.checked_add(32).ok_or_else(invalid)?; + if bytes.len() != expected_len + || bytes.get(..8) != Some(MAGIC.as_slice()) + || bytes.get(8..24) != Some(owner.as_bytes().as_slice()) + || bytes.get(body_len..) + != Some(Sha256::digest(bytes.get(..body_len).ok_or_else(invalid)?).as_slice()) + { + return Err(invalid()); + } + let mut records = Vec::with_capacity(count); + let mut identities = BTreeSet::new(); + for row in bytes + .get(26..body_len) + .ok_or_else(invalid)? + .as_chunks::() + .0 + { + let run = Uuid::from_slice(row.get(..16).ok_or_else(invalid)?).map_err(|_| invalid())?; + let pin = Uuid::from_slice(row.get(16..32).ok_or_else(invalid)?).map_err(|_| invalid())?; + if run.is_nil() || pin.is_nil() || !identities.insert((run, pin)) { + return Err(invalid()); + } + let committed = match row.get(64) { + Some(0) => false, + Some(1) => true, + _ => return Err(invalid()), + }; + records.push(ImageRecord { + run, + pin, + image: row + .get(32..64) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?, + committed, + }); + } + Ok(records) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership/images/tests.rs b/crates/rustack-lambda-core/src/docker/ownership/images/tests.rs new file mode 100644 index 00000000..4247061b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/images/tests.rs @@ -0,0 +1,267 @@ +use std::{ + os::unix::fs::{PermissionsExt, symlink}, + sync::Arc, + time::Duration, +}; + +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ImageRecord, MAX_BYTES, PUBLISHED, STAGING, decode, encode}; +use crate::docker::{ + DirectoryLease, DirectoryLeaseError, EnvironmentPlan, ImagePinScope, + ownership::{ContainerRecord, NAMESPACE}, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn record(run: Uuid) -> ImageRecord { + ImageRecord { + run, + pin: Uuid::new_v4(), + image: [0xab; 32], + committed: false, + } +} +async fn lease(root: &tempfile::TempDir) -> DirectoryLease { + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap() +} +fn resign(bytes: &mut [u8]) { + let length = bytes.len() - 32; + let checksum = Sha256::digest(&bytes[..length]); + bytes[length..].copy_from_slice(&checksum); +} +#[test] +fn test_should_roundtrip_full_capacity_and_reject_semantic_corruption() { + let owner = Uuid::new_v4(); + let run = Uuid::new_v4(); + let records: Vec<_> = (1..=8192) + .map(|id| ImageRecord { + run, + pin: Uuid::from_u128(id), + image: [1; 32], + committed: id % 2 == 0, + }) + .collect(); + let bytes = encode(owner, &records).unwrap(); + assert_eq!(bytes.len(), MAX_BYTES); + assert_eq!(decode(&bytes, owner).unwrap(), records); + assert!(encode(owner, &[records.clone(), vec![record(run)]].concat()).is_err()); + assert!(decode(&bytes, Uuid::new_v4()).is_err()); + let single = encode(owner, &[record(run)]).unwrap(); + for length in 0..single.len() { + assert!(decode(&single[..length], owner).is_err()); + } + for position in 0..single.len() { + let mut bad = single.clone(); + bad[position] ^= 1; + assert!(decode(&bad, owner).is_err()); + } + for position in [26, 42] { + let mut bad = single.clone(); + bad[position..position + 16].fill(0); + resign(&mut bad); + assert!(decode(&bad, owner).is_err()); + } + let mut nil_owner = encode(owner, &[]).unwrap(); + nil_owner.get_mut(8..24).unwrap().fill(0); + resign(&mut nil_owner); + assert!(decode(&nil_owner, Uuid::nil()).is_err()); + let mut bad = single; + bad[90] = 2; + resign(&mut bad); + assert!(decode(&bad, owner).is_err()); + let entry = record(run); + assert!(encode(owner, &[entry.clone(), entry]).is_err()); +} +#[tokio::test] +async fn test_should_persist_unknown_commitment_and_erasure_under_new_run() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let entry = record(first.run_id()); + first + .prepare_image(entry.clone(), deadline()) + .await + .unwrap(); + drop(first); + let second = lease(&root).await; + assert_ne!(entry.run, second.run_id()); + assert_eq!( + second.image_records(deadline()).await.unwrap(), + vec![entry.clone()] + ); + let pin = entry.pin(&second).unwrap(); + assert_eq!( + pin.reference().as_str(), + format!( + "rustack-internal/lambda-{}/{}:{}", + second.owner_id().simple(), + entry.run.simple(), + entry.pin.simple() + ) + ); + assert_eq!( + entry.expected().unwrap().as_str(), + format!("sha256:{}", "ab".repeat(32)) + ); + second + .commit_image(entry.clone(), deadline()) + .await + .unwrap(); + assert!(second.image_records(deadline()).await.unwrap()[0].committed); + drop(pin); + drop(second); + let third = lease(&root).await; + assert!(third.image_records(deadline()).await.unwrap()[0].committed); + third.settle_image(entry.clone(), deadline()).await.unwrap(); + third.settle_image(entry, deadline()).await.unwrap(); + assert_eq!(third.image_records(deadline()).await.unwrap(), []); +} +#[tokio::test] +async fn test_should_reject_cross_run_and_changed_target_without_erasing_authority() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_image(entry.clone(), deadline()) + .await + .unwrap(); + let mut changed = entry.clone(); + changed.image = [7; 32]; + assert!( + lease + .prepare_image(changed.clone(), deadline()) + .await + .is_err() + ); + assert!( + lease + .commit_image(changed.clone(), deadline()) + .await + .is_err() + ); + assert!(lease.settle_image(changed, deadline()).await.is_err()); + assert!( + lease + .prepare_image(record(Uuid::new_v4()), deadline()) + .await + .is_err() + ); + assert_eq!(lease.image_records(deadline()).await.unwrap(), vec![entry]); +} +#[rstest::rstest] +#[case(PUBLISHED)] +#[case(STAGING)] +#[tokio::test] +async fn test_should_reject_unsafe_oversized_or_corrupt_pin_files(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let target = root.path().join(NAMESPACE).join(name); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, &target).unwrap(); + assert!(lease.image_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::hard_link(&outside, &target).await.unwrap(); + assert!(lease.image_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::write(&target, b"partial").await.unwrap(); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)) + .await + .unwrap(); + assert!(lease.image_records(deadline()).await.is_err()); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o600)) + .await + .unwrap(); + if name == PUBLISHED { + assert!(lease.image_records(deadline()).await.is_err()); + } else { + assert_eq!(lease.image_records(deadline()).await.unwrap(), []); + } + tokio::fs::write(&target, vec![0; MAX_BYTES + 1]) + .await + .unwrap(); + assert!(lease.image_records(deadline()).await.is_err()); + assert_eq!(tokio::fs::read(outside).await.unwrap(), b"keep"); +} +#[tokio::test] +async fn test_should_share_transaction_exclusion_and_reject_overlong_budget_before_waiting() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let held = lease.0.transaction.acquire().await.unwrap(); + assert!(matches!( + tokio::time::timeout( + Duration::from_millis(50), + lease.image_records(Instant::now() + Duration::from_secs(61)) + ) + .await, + Ok(Err(DirectoryLeaseError::Configuration)) + )); + let copy = lease.clone(); + let mut task = + tokio::spawn(async move { copy.prepare_image(record(copy.run_id()), deadline()).await }); + assert!( + tokio::time::timeout(Duration::from_millis(30), &mut task) + .await + .is_err() + ); + drop(held); + task.await.unwrap().unwrap(); + assert_eq!(lease.image_records(deadline()).await.unwrap().len(), 1); +} +#[tokio::test] +async fn test_should_retain_record_when_erasure_cannot_publish() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_image(entry.clone(), deadline()) + .await + .unwrap(); + let staging = root.path().join(NAMESPACE).join(STAGING); + symlink(root.path().join("outside"), &staging).unwrap(); + assert!(lease.settle_image(entry.clone(), deadline()).await.is_err()); + let published = tokio::fs::read(root.path().join(NAMESPACE).join(PUBLISHED)) + .await + .unwrap(); + assert_eq!( + decode(&published, lease.owner_id()).unwrap(), + vec![entry.clone()] + ); + tokio::fs::remove_file(staging).await.unwrap(); + lease.settle_image(entry, deadline()).await.unwrap(); +} +#[tokio::test] +async fn test_should_keep_container_encoding_unchanged_and_allow_independent_records() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let plan = EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let container = ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(); + lease + .prepare_container(container, deadline()) + .await + .unwrap(); + let path = root.path().join(NAMESPACE).join("containers.v1"); + let before = tokio::fs::read(&path).await.unwrap(); + let pin = ImagePinScope::new(lease.clone()).allocate().unwrap(); + let entry = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + lease.prepare_image(entry, deadline()).await.unwrap(); + assert_eq!(tokio::fs::read(path).await.unwrap(), before); + assert_eq!(lease.container_records(deadline()).await.unwrap().len(), 1); + assert_eq!(lease.image_records(deadline()).await.unwrap().len(), 1); +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/journal.rs b/crates/rustack-lambda-core/src/docker/ownership/journal.rs new file mode 100644 index 00000000..3436b254 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/journal.rs @@ -0,0 +1,317 @@ +//! Bounded, atomically replaced container identities; no deployment secrets. + +use std::fmt; + +use sha2::{Digest, Sha256}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{DirectoryLease, DirectoryLeaseError, snapshot::Codec}; +use crate::docker::{ + DockerError, DockerKey, EnvironmentPlan, models::ContainerInspectResponse, plan::ContainerRole, +}; + +const PUBLISHED: &str = "containers.v1"; +const STAGING: &str = "containers.pending"; +const HEADER: &[u8; 8] = b"RSDKCTR1"; +const RECORD_BYTES: usize = 114; +const MAX_RECORDS: usize = 64; +const MAX_BYTES: usize = 25 + MAX_RECORDS * RECORD_BYTES + 32; + +#[derive(Clone, PartialEq, Eq)] +pub(in crate::docker) struct ContainerRecord { + pub(in crate::docker) run: Uuid, + pub(in crate::docker) environment: Uuid, + resource: Uuid, + function: bool, + image: [u8; 32], + pub(in crate::docker) known: Option<[u8; 32]>, +} +impl fmt::Debug for ContainerRecord { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ContainerRecord()") + } +} +impl ContainerRecord { + pub(in crate::docker) fn from_plan( + plan: &EnvironmentPlan, + role: ContainerRole, + ) -> Result { + let image = plan + .image_id(role) + .strip_prefix("sha256:") + .ok_or(DockerError::Configuration)?; + let mut digest = [0; 32]; + hex::decode_to_slice(image, &mut digest).map_err(|_| DockerError::Configuration)?; + Ok(Self { + run: plan.directory.run_id(), + environment: plan + .environment_id() + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + resource: plan + .revision() + .resource_id + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + function: matches!(role, ContainerRole::Function), + image: digest, + known: None, + }) + } + pub(in crate::docker) fn function(&self) -> bool { + self.function + } + fn role(&self) -> &'static str { + if self.function { "function" } else { "relay" } + } + pub(in crate::docker) fn key(&self) -> Result { + format!("rustack-{}-{}", self.role(), self.environment).parse() + } + pub(in crate::docker) fn known_key(&self) -> Result, DockerError> { + self.known.map(|id| hex::encode(id).parse()).transpose() + } + pub(in crate::docker) fn owned( + &self, + owner: Uuid, + inspected: &ContainerInspectResponse, + ) -> Result { + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let labels = config.labels.as_ref().ok_or(DockerError::Quarantined)?; + let id = inspected.id.as_deref().ok_or(DockerError::Protocol)?; + let mut decoded = [0; 32]; + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DockerError::Quarantined); + } + hex::decode_to_slice(id, &mut decoded).map_err(|_| DockerError::Quarantined)?; + if self.known.is_some_and(|expected| expected != decoded) + || inspected.name.as_deref() != Some(format!("/{}", self.key()?.as_str()).as_str()) + || config.image.as_deref() + != Some(format!("sha256:{}", hex::encode(self.image)).as_str()) + || ![ + ("rustack.lambda.ownerId", owner.to_string()), + ("rustack.lambda.runId", self.run.to_string()), + ("rustack.lambda.environmentId", self.environment.to_string()), + ( + "rustack.lambda.functionResourceId", + self.resource.to_string(), + ), + ("rustack.lambda.role", self.role().to_owned()), + ] + .iter() + .all(|(key, value)| labels.get(*key) == Some(value)) + { + return Err(DockerError::Quarantined); + } + id.parse() + } + fn same_key(&self, other: &Self) -> bool { + self.environment == other.environment && self.function == other.function + } + fn identity(&self, other: &Self) -> bool { + self.run == other.run + && self.environment == other.environment + && self.resource == other.resource + && self.function == other.function + && self.image == other.image + } +} +impl DirectoryLease { + async fn journal( + &self, + deadline: Instant, + work: impl FnOnce(&mut Vec) -> Result + Send + 'static, + ) -> Result { + self.snapshot::(deadline, work).await + } + pub(in crate::docker) async fn container_records( + &self, + deadline: Instant, + ) -> Result, DirectoryLeaseError> { + self.journal(deadline, |records| Ok(records.clone())).await + } + pub(in crate::docker) async fn prepare_container( + &self, + record: ContainerRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if record.run != self.run_id() { + return Err(DirectoryLeaseError::InvalidState); + } + self.journal(deadline, move |records| { + if let Some(previous) = records.iter().find(|old| old.same_key(&record)) { + if !previous.identity(&record) { + return Err(DirectoryLeaseError::InvalidState); + } + } else { + if records.len() >= MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + records.push(record); + } + Ok(()) + }) + .await + } + pub(in crate::docker) async fn know_container( + &self, + record: ContainerRecord, + id: DockerKey, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + let mut decoded = [0; 32]; + if id.as_str().len() != 64 + || !id + .as_str() + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DirectoryLeaseError::InvalidState); + } + hex::decode_to_slice(id.as_str(), &mut decoded) + .map_err(|_| DirectoryLeaseError::InvalidState)?; + self.journal(deadline, move |records| { + let previous = records + .iter_mut() + .find(|old| old.same_key(&record)) + .ok_or(DirectoryLeaseError::InvalidState)?; + if !previous.identity(&record) || previous.known.is_some_and(|old| old != decoded) { + return Err(DirectoryLeaseError::InvalidState); + } + previous.known = Some(decoded); + Ok(()) + }) + .await + } + pub(in crate::docker) async fn settle_container( + &self, + record: ContainerRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + self.journal(deadline, move |records| { + if let Some(position) = records.iter().position(|old| old.same_key(&record)) { + if !records + .get(position) + .is_some_and(|old| old.identity(&record)) + { + return Err(DirectoryLeaseError::InvalidState); + } + records.remove(position); + } + Ok(()) + }) + .await + } +} + +fn encode(owner: Uuid, records: &[ContainerRecord]) -> Result, DirectoryLeaseError> { + if records.len() > MAX_RECORDS || owner.is_nil() { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = Vec::with_capacity(MAX_BYTES); + bytes.extend_from_slice(HEADER); + bytes.extend_from_slice(owner.as_bytes()); + bytes.push(u8::try_from(records.len()).map_err(|_| DirectoryLeaseError::InvalidState)?); + for record in records { + bytes.extend_from_slice(record.run.as_bytes()); + bytes.extend_from_slice(record.environment.as_bytes()); + bytes.extend_from_slice(record.resource.as_bytes()); + bytes.push(u8::from(!record.function)); + bytes.extend_from_slice(&record.image); + bytes.push(u8::from(record.known.is_some())); + bytes.extend_from_slice(&record.known.unwrap_or([0; 32])); + } + let checksum = Sha256::digest(&bytes); + bytes.extend_from_slice(&checksum); + decode(&bytes, owner)?; + Ok(bytes) +} +fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + let invalid = || DirectoryLeaseError::InvalidState; + let count = usize::from(*bytes.get(24).ok_or_else(invalid)?); + let body_len = 25 + count * RECORD_BYTES; + if count > MAX_RECORDS + || bytes.len() != body_len + 32 + || bytes.get(..8) != Some(HEADER.as_slice()) + || bytes.get(8..24) != Some(owner.as_bytes().as_slice()) + || bytes.get(body_len..) + != Some(Sha256::digest(bytes.get(..body_len).ok_or_else(invalid)?).as_slice()) + { + return Err(invalid()); + } + let mut records: Vec = Vec::with_capacity(count); + for row in bytes + .get(25..body_len) + .ok_or_else(invalid)? + .as_chunks::() + .0 + { + let uuid = |start| -> Result { + let value = Uuid::from_slice(row.get(start..start + 16).ok_or_else(invalid)?) + .map_err(|_| invalid())?; + if value.is_nil() { + Err(invalid()) + } else { + Ok(value) + } + }; + let function = match row.get(48) { + Some(0) => true, + Some(1) => false, + _ => return Err(invalid()), + }; + let id: [u8; 32] = row + .get(82..114) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?; + let known = match row.get(81) { + Some(0) if id == [0; 32] => None, + Some(1) => Some(id), + _ => return Err(invalid()), + }; + let record = ContainerRecord { + run: uuid(0)?, + environment: uuid(16)?, + resource: uuid(32)?, + function, + image: row + .get(49..81) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?, + known, + }; + if records.iter().any(|old| { + old.same_key(&record) + || (old.environment == record.environment + && (old.run != record.run || old.resource != record.resource)) + }) { + return Err(invalid()); + } + records.push(record); + } + Ok(records) +} +pub(super) struct ContainerCodec; +impl Codec for ContainerCodec { + type Record = ContainerRecord; + const PUBLISHED: &'static str = PUBLISHED; + const STAGING: &'static str = STAGING; + const MAX_BYTES: usize = MAX_BYTES; + fn encode(owner: Uuid, records: &[ContainerRecord]) -> Result, DirectoryLeaseError> { + encode(owner, records) + } + fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + decode(bytes, owner) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs b/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs new file mode 100644 index 00000000..c5d1bffc --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs @@ -0,0 +1,328 @@ +use std::{ + os::unix::fs::{MetadataExt, PermissionsExt, symlink}, + path::PathBuf, + time::Duration, +}; + +use super::*; +use crate::docker::ownership::NAMESPACE; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn record(run: Uuid) -> ContainerRecord { + ContainerRecord { + run, + environment: Uuid::new_v4(), + resource: Uuid::new_v4(), + function: true, + image: [2; 32], + known: None, + } +} +async fn lease(root: &tempfile::TempDir) -> DirectoryLease { + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap() +} +fn path(root: &tempfile::TempDir, file: &str) -> PathBuf { + root.path().join(NAMESPACE).join(file) +} + +#[tokio::test] +async fn test_should_persist_unknown_known_and_settled_across_reacquisition() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let entry = record(first.run_id()); + first + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + let bytes = tokio::fs::read(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!(bytes.len(), 171); + assert_eq!( + decode(&bytes, first.owner_id()).unwrap(), + vec![entry.clone()] + ); + drop(first); + let second = lease(&root).await; + assert_ne!(second.run_id(), entry.run); + assert_eq!( + second.container_records(deadline()).await.unwrap(), + vec![entry.clone()] + ); + second + .know_container(entry.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + drop(second); + let third = lease(&root).await; + assert_eq!( + third.container_records(deadline()).await.unwrap()[0].known, + Some([0xaa; 32]) + ); + third + .settle_container(entry.clone(), deadline()) + .await + .unwrap(); + third.settle_container(entry, deadline()).await.unwrap(); + drop(third); + assert_eq!( + lease(&root) + .await + .container_records(deadline()) + .await + .unwrap(), + [] + ); +} +#[tokio::test] +async fn test_should_serialize_concurrent_snapshots_without_losing_records() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let mut jobs = tokio::task::JoinSet::new(); + for _ in 0..32 { + let lease = lease.clone(); + jobs.spawn(async move { + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + }); + } + while let Some(result) = jobs.join_next().await { + result.unwrap().unwrap(); + } + assert_eq!(lease.container_records(deadline()).await.unwrap().len(), 32); + let metadata = tokio::fs::metadata(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, 0o600); + assert_eq!(metadata.nlink(), 1); +} +#[tokio::test] +async fn test_should_bound_records_and_reject_cross_run_or_changed_identity() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let first = record(lease.run_id()); + lease + .prepare_container(first.clone(), deadline()) + .await + .unwrap(); + let mut changed = first.clone(); + changed.image = [9; 32]; + assert!( + lease + .prepare_container(changed.clone(), deadline()) + .await + .is_err() + ); + assert!(lease.settle_container(changed, deadline()).await.is_err()); + assert!( + lease + .prepare_container(record(Uuid::new_v4()), deadline()) + .await + .is_err() + ); + lease + .know_container(first.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + assert!( + lease + .know_container(first, "b".repeat(64).parse().unwrap(), deadline()) + .await + .is_err() + ); + for _ in 1..64 { + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + .unwrap(); + } + assert_eq!(lease.container_records(deadline()).await.unwrap().len(), 64); + assert!( + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + .is_err() + ); +} +#[test] +fn test_should_reject_truncated_corrupted_foreign_duplicate_and_invalid_encoding() { + let owner = Uuid::new_v4(); + let entry = record(Uuid::new_v4()); + let bytes = encode(owner, std::slice::from_ref(&entry)).unwrap(); + for length in 0..bytes.len() { + assert!(decode(&bytes[..length], owner).is_err()); + } + for position in 0..bytes.len() { + let mut bad = bytes.clone(); + bad[position] ^= 1; + assert!(decode(&bad, owner).is_err()); + } + assert!(decode(&bytes, Uuid::new_v4()).is_err()); + assert!(encode(owner, &[entry.clone(), entry.clone()]).is_err()); + let mut bad = entry.clone(); + bad.run = Uuid::nil(); + assert!(encode(owner, &[bad]).is_err()); + let mut relay = entry.clone(); + relay.function = false; + relay.resource = Uuid::new_v4(); + assert!(encode(owner, &[entry, relay]).is_err()); + assert!(encode(owner, &vec![record(Uuid::new_v4()); 65]).is_err()); +} +#[rstest::rstest] +#[case(PUBLISHED)] +#[case(STAGING)] +#[tokio::test] +async fn test_should_reject_unsafe_snapshot_inodes(#[case] file: &str) { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let target = path(&root, file); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, &target).unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::hard_link(&outside, &target).await.unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::write(&target, b"").await.unwrap(); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)) + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o600)) + .await + .unwrap(); + tokio::fs::write(&target, vec![0; MAX_BYTES + 1]) + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + assert_eq!(tokio::fs::read(outside).await.unwrap(), b"keep"); +} +#[tokio::test] +async fn test_should_ignore_only_bounded_safe_interrupted_staging() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + tokio::fs::write(path(&root, STAGING), b"partial") + .await + .unwrap(); + tokio::fs::set_permissions(path(&root, STAGING), std::fs::Permissions::from_mode(0o600)) + .await + .unwrap(); + assert_eq!( + lease.container_records(deadline()).await.unwrap(), + vec![entry] + ); + tokio::fs::write(path(&root, PUBLISHED), b"partial") + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); +} +#[tokio::test] +async fn test_should_fail_before_publication_with_expired_budget() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + assert!(matches!( + lease + .prepare_container(record(lease.run_id()), Instant::now()) + .await, + Err(DirectoryLeaseError::Deadline) + )); + assert!(!tokio::fs::try_exists(path(&root, PUBLISHED)).await.unwrap()); + assert!( + lease + .container_records(Instant::now() + Duration::from_secs(61)) + .await + .is_err() + ); +} + +#[tokio::test] +async fn test_should_keep_transaction_and_lease_owned_after_waiter_abandonment() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let (entered, observed) = tokio::sync::oneshot::channel(); + let (release, barrier) = std::sync::mpsc::channel(); + let entry = record(lease.run_id()); + let worker_lease = lease.clone(); + let task = tokio::spawn(async move { + worker_lease + .journal(deadline(), move |records| { + records.push(entry); + entered.send(()).unwrap(); + barrier.recv().unwrap(); + Ok(()) + }) + .await + }); + observed.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let check = lease.clone(); + let mut read = tokio::spawn(async move { check.container_records(deadline()).await }); + assert!( + tokio::time::timeout(Duration::from_millis(30), &mut read) + .await + .is_err() + ); + drop(lease); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + release.send(()).unwrap(); + // The cancelled worker must finish/dispose its transaction before this read. + assert_eq!(read.await.unwrap().unwrap(), []); + let _next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_should_keep_authoritative_record_when_erase_cannot_publish() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, path(&root, STAGING)).unwrap(); + assert!( + lease + .settle_container(entry.clone(), deadline()) + .await + .is_err() + ); + let bytes = tokio::fs::read(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!( + decode(&bytes, lease.owner_id()).unwrap(), + vec![entry.clone()] + ); + tokio::fs::remove_file(path(&root, STAGING)).await.unwrap(); + lease.settle_container(entry, deadline()).await.unwrap(); + assert_eq!(lease.container_records(deadline()).await.unwrap(), []); +} + +#[tokio::test] +async fn test_should_reject_overlong_budget_before_waiting_for_transaction() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let _held = lease.0.transaction.acquire().await.unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(50), + lease.container_records(Instant::now() + Duration::from_secs(61)), + ) + .await; + assert!(matches!( + result, + Ok(Err(DirectoryLeaseError::Configuration)) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/ownership/snapshot.rs b/crates/rustack-lambda-core/src/docker/ownership/snapshot.rs new file mode 100644 index 00000000..0295eea2 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/snapshot.rs @@ -0,0 +1,121 @@ +//! Shared bounded file transactions; individual codecs own their wire formats. + +use std::{ + fs::File, + io::{self, Read, Write}, + time::Duration, +}; + +use nix::fcntl::{OFlag, renameat}; +use tokio::time::{Instant, timeout_at}; +use uuid::Uuid; + +use super::{Control, DirectoryLease, DirectoryLeaseError, LeaseGuard, WORKERS, managed_file, run}; + +pub(super) trait Codec { + type Record: Clone + PartialEq + Send + 'static; + const PUBLISHED: &'static str; + const STAGING: &'static str; + const MAX_BYTES: usize; + fn encode(owner: Uuid, records: &[Self::Record]) -> Result, DirectoryLeaseError>; + fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError>; +} +impl DirectoryLease { + fn namespace(&self) -> &File { + match &self.0.guard { + LeaseGuard::Held { directory, .. } => directory, + #[cfg(test)] + LeaseGuard::Testing { directory, .. } => directory, + } + } + pub(super) async fn snapshot( + &self, + deadline: Instant, + work: impl FnOnce(&mut Vec) -> Result + Send + 'static, + ) -> Result { + self.transaction(deadline, move |directory, owner, control| { + let mut records = read::(directory, owner)?; + control.check()?; + let original = records.clone(); + let result = work(&mut records)?; + if records != original { + write::(directory, owner, &records, control)?; + } + Ok(result) + }) + .await + } + pub(super) async fn transaction( + &self, + deadline: Instant, + work: impl FnOnce(&File, Uuid, &Control) -> Result + Send + 'static, + ) -> Result { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or(DirectoryLeaseError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DirectoryLeaseError::Configuration); + } + let permit = timeout_at(deadline, self.0.transaction.clone().acquire_owned()) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + let lease = self.clone(); + run(&WORKERS, deadline, move |control| { + let _transaction = permit; + work(lease.namespace(), lease.owner_id(), control) + }) + .await + } +} +pub(super) fn read( + directory: &File, + owner: Uuid, +) -> Result, DirectoryLeaseError> { + optional::(directory, C::STAGING)?; + let Some(mut file) = optional::(directory, C::PUBLISHED)? else { + directory.sync_all()?; + return Ok(Vec::new()); + }; + let mut bytes = Vec::with_capacity(C::MAX_BYTES); + Read::by_ref(&mut file) + .take((C::MAX_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + let records = C::decode(&bytes, owner)?; + file.sync_all()?; + directory.sync_all()?; + Ok(records) +} +fn optional(directory: &File, name: &str) -> Result, DirectoryLeaseError> { + match managed_file(directory, name, OFlag::O_RDONLY, C::MAX_BYTES as u64) { + Ok(file) => Ok(Some(file)), + Err(DirectoryLeaseError::Io(error)) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} +pub(super) fn write( + directory: &File, + owner: Uuid, + records: &[C::Record], + control: &Control, +) -> Result<(), DirectoryLeaseError> { + let bytes = C::encode(owner, records)?; + if bytes.len() > C::MAX_BYTES { + return Err(DirectoryLeaseError::InvalidState); + } + let mut file = managed_file( + directory, + C::STAGING, + OFlag::O_RDWR | OFlag::O_CREAT, + C::MAX_BYTES as u64, + )?; + control.check()?; + file.set_len(0)?; + file.write_all(&bytes)?; + file.sync_all()?; + control.check()?; + renameat(directory, C::STAGING, directory, C::PUBLISHED)?; + directory.sync_all()?; + Ok(()) +} diff --git a/crates/rustack-lambda-core/src/docker/ownership_tests.rs b/crates/rustack-lambda-core/src/docker/ownership_tests.rs new file mode 100644 index 00000000..8550a35e --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership_tests.rs @@ -0,0 +1,768 @@ +use std::{ + error::Error, + os::unix::fs::{PermissionsExt, symlink}, + process::Stdio, + sync::mpsc, +}; + +use nix::unistd::mkfifo; +use tempfile::TempDir; +use tokio::{ + fs, + process::Command, + sync::oneshot, + time::{sleep, timeout}, +}; + +use super::*; +use crate::docker::ImagePinScope; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn lease(root: &TempDir) -> DirectoryLease { + DirectoryLease::acquire(root.path().to_path_buf(), deadline()) + .await + .unwrap() +} +fn managed(root: &TempDir, name: &str) -> PathBuf { + root.path().join(NAMESPACE).join(name) +} + +#[tokio::test] +async fn test_should_persist_owner_and_refresh_run_after_last_clone_closes() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let (owner, run) = (first.owner_id(), first.run_id()); + assert!(!owner.is_nil() && !run.is_nil()); + let clone = first.clone(); + drop(first); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(clone); + let second = lease(&root).await; + assert_eq!(second.owner_id(), owner); + assert_ne!(second.run_id(), run); + let bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + assert_eq!(bytes.len(), OWNER_BYTES); + assert_eq!(bytes.get(..8), Some(MAGIC.as_slice())); + assert!( + tokio::fs::try_exists(managed(&root, PENDING)) + .await + .is_ok_and(|exists| !exists) + ); +} + +#[tokio::test] +async fn test_should_hold_the_directory_through_allocated_pins() { + let root = tempfile::tempdir().unwrap(); + let owner = lease(&root).await; + let id = owner.owner_id(); + let scope = ImagePinScope::new(owner); + let pin = scope.allocate().unwrap(); + let copy = pin.clone(); + drop(scope); + drop(pin); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(copy); + assert_eq!(lease(&root).await.owner_id(), id); +} + +#[rstest::rstest] +#[case(PathBuf::new())] +#[case(PathBuf::from("relative"))] +#[case(PathBuf::from("/tmp/../outside"))] +#[case(PathBuf::from("/tmp/\0bad"))] +#[case(PathBuf::from(format!("/{}", "a".repeat(4096))))] +#[tokio::test] +async fn test_should_reject_invalid_directories_without_io(#[case] path: PathBuf) { + assert!(matches!( + DirectoryLease::acquire(path, deadline()).await, + Err(DirectoryLeaseError::Configuration) + )); +} + +#[tokio::test] +async fn test_should_create_missing_roots_and_keep_descriptor_files_private() { + let parent = tempfile::tempdir().unwrap(); + let root = parent.path().join("new/data"); + let owner = DirectoryLease::acquire(root.clone(), deadline()) + .await + .unwrap(); + for (path, expected) in [ + (root.join(NAMESPACE), 0o700), + (root.join(NAMESPACE).join(LOCK), 0o600), + (root.join(NAMESPACE).join(OWNER), 0o600), + ] { + let metadata = tokio::fs::metadata(path).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, expected); + assert_eq!(metadata.uid(), geteuid().as_raw()); + } + assert_eq!(format!("{owner:?}"), "DirectoryLease()"); +} + +#[rstest::rstest] +#[case("")] +#[case("/")] +#[case("/.")] +#[case("//./")] +#[tokio::test] +async fn test_should_reject_a_symlink_root_without_touching_the_destination(#[case] suffix: &str) { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let link = root.path().join("link"); + symlink(outside.path(), &link).unwrap(); + let spelling = PathBuf::from(format!("{}{suffix}", link.display())); + assert!(matches!( + DirectoryLease::acquire(spelling, deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!( + !tokio::fs::try_exists(outside.path().join(NAMESPACE)) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn test_should_reject_untrusted_rename_permissions_and_accept_a_sticky_ancestor() { + let root = tempfile::tempdir().unwrap(); + let parent = root.path().join("shared"); + tokio::fs::create_dir(&parent).await.unwrap(); + tokio::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o777)) + .await + .unwrap(); + let path = parent.join("new/data"); + assert!(matches!( + DirectoryLease::acquire(path.clone(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!(!tokio::fs::try_exists(parent.join("new")).await.unwrap()); + tokio::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)) + .await + .unwrap(); + let owner = DirectoryLease::acquire(path.clone(), deadline()) + .await + .unwrap(); + let id = owner.owner_id(); + drop(owner); + assert_eq!( + DirectoryLease::acquire(path, deadline()) + .await + .unwrap() + .owner_id(), + id + ); +} + +#[tokio::test] +async fn test_should_repeat_the_ancestor_sync_barrier_after_interrupted_root_creation() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("new/parent/data"); + let test = tokio::task::spawn_blocking(move || { + let control = Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + }; + let mut attempts = 0; + let first = open_root_with_sync(&path, &control, |directory| { + attempts += 1; + if attempts == 2 { + return Err(io::Error::other("interrupted parent sync")); + } + directory.sync_all() + }); + assert!(matches!(first, Err(DirectoryLeaseError::Io(_)))); + assert!(path.is_dir()); + let parent = symlink_metadata(path.parent().unwrap()).unwrap(); + let mut observed = Vec::new(); + let reopened = open_root_with_sync(&path, &control, |directory| { + let metadata = directory.metadata()?; + observed.push((metadata.dev(), metadata.ino())); + directory.sync_all() + }) + .unwrap(); + assert!( + observed.contains(&(parent.dev(), parent.ino())), + "retry must not skip the already-existing parent" + ); + drop(reopened); + let owner = acquire(&DataDirectory::try_from(path).unwrap(), &control).unwrap(); + assert!(!owner.owner_id().is_nil()); + }); + timeout(Duration::from_secs(10), test) + .await + .unwrap() + .unwrap(); +} + +#[rstest::rstest] +#[case::namespace(NAMESPACE)] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_managed_symlinks_without_modifying_their_targets(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let sentinel = outside.path().join("sentinel"); + tokio::fs::write(&sentinel, b"sentinel").await.unwrap(); + if name == NAMESPACE { + symlink(outside.path(), root.path().join(NAMESPACE)).unwrap(); + } else { + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, name)).await.unwrap(); + symlink(&sentinel, managed(&root, name)).unwrap(); + } + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(&sentinel).await.unwrap(), b"sentinel"); +} + +#[rstest::rstest] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_hardlinked_managed_files(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let before = tokio::fs::read(managed(&root, name)).await.unwrap(); + let link = root.path().join("hardlink"); + tokio::fs::hard_link(managed(&root, name), &link) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(link).await.unwrap(), before); +} + +#[rstest::rstest] +#[case::namespace(NAMESPACE)] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_group_or_other_access(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let path = if name == NAMESPACE { + root.path().join(name) + } else { + managed(&root, name) + }; + tokio::fs::set_permissions( + &path, + std::fs::Permissions::from_mode(if name == NAMESPACE { 0o755 } else { 0o644 }), + ) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[rstest::rstest] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_fifos_without_waiting_for_a_peer(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, name)).await.unwrap(); + mkfifo(&managed(&root, name), Mode::from_bits_truncate(0o600)).unwrap(); + let result = timeout( + Duration::from_secs(2), + DirectoryLease::acquire(root.path().into(), deadline()), + ) + .await + .unwrap(); + assert!(matches!(result, Err(DirectoryLeaseError::InvalidState))); +} + +#[rstest::rstest] +#[case::empty(Vec::new())] +#[case::partial(vec![0; 24])] +#[case::oversized(vec![0; OWNER_BYTES + 1])] +#[case::magic(vec![0; OWNER_BYTES])] +#[tokio::test] +async fn test_should_reject_damaged_published_identity_without_regenerating_it( + #[case] damaged: Vec, +) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::write(managed(&root, OWNER), &damaged) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!( + tokio::fs::read(managed(&root, OWNER)).await.unwrap(), + damaged + ); +} + +#[tokio::test] +async fn test_should_reject_a_nonnil_identity_bit_flip_and_a_checksummed_nil_identity() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let mut bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + *bytes.get_mut(8).unwrap() ^= 1; + tokio::fs::write(managed(&root, OWNER), &bytes) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + bytes.get_mut(8..24).unwrap().fill(0); + let checksum = Sha256::digest(bytes.get(..24).unwrap()); + bytes + .get_mut(24..) + .unwrap() + .copy_from_slice(checksum.as_ref()); + tokio::fs::write(managed(&root, OWNER), &bytes) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[tokio::test] +async fn test_should_recover_only_a_bounded_pending_initialization() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + tokio::fs::write(managed(&root, PENDING), b"partial") + .await + .unwrap(); + tokio::fs::set_permissions( + managed(&root, PENDING), + std::fs::Permissions::from_mode(0o600), + ) + .await + .unwrap(); + let recovered = lease(&root).await; + assert!(!recovered.owner_id().is_nil()); + assert!( + !tokio::fs::try_exists(managed(&root, PENDING)) + .await + .unwrap() + ); + drop(recovered); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + tokio::fs::write(managed(&root, PENDING), vec![0; OWNER_BYTES + 1]) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[tokio::test] +async fn test_should_reject_unsafe_pending_initialization_without_touching_its_target() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + let sentinel = root.path().join("sentinel"); + tokio::fs::write(&sentinel, b"outside").await.unwrap(); + symlink(&sentinel, managed(&root, PENDING)).unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(sentinel).await.unwrap(), b"outside"); +} + +#[tokio::test] +async fn test_should_bound_deadlines_and_keep_diagnostics_redacted() { + let root = tempfile::tempdir().unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), Instant::now()).await, + Err(DirectoryLeaseError::Deadline) + )); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), Instant::now() + Duration::from_secs(61)).await, + Err(DirectoryLeaseError::Configuration) + )); + let error = DirectoryLeaseError::Io(io::Error::other("secret operator path")); + assert!(!format!("{error:?} {error}").contains("secret")); + assert!(error.source().is_some()); +} + +#[rstest::rstest] +#[case::aborted(false)] +#[case::deadline(true)] +#[tokio::test] +async fn test_should_retain_worker_capacity_and_lock_after_the_waiter_ends(#[case] expires: bool) { + let root = tempfile::tempdir().unwrap(); + let path = root.path().to_path_buf(); + let pool = Box::leak(Box::new(Semaphore::new(1))); + let (ready, acquired) = oneshot::channel(); + let (release, held) = mpsc::channel(); + let budget = if expires { + Duration::from_secs(2) + } else { + Duration::from_secs(10) + }; + let task = tokio::spawn(run(pool, Instant::now() + budget, move |control| { + let lease = acquire(&DataDirectory::try_from(path)?, control)?; + ready.send(()).unwrap(); + held.recv_timeout(Duration::from_secs(10)).unwrap(); + Ok(lease) + })); + acquired.await.unwrap(); + if expires { + assert!(matches!( + timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(), + Err(DirectoryLeaseError::Deadline) + )); + } else { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + } + assert_eq!(pool.available_permits(), 0); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + release.send(()).unwrap(); + timeout(Duration::from_secs(5), async { + loop { + match DirectoryLease::acquire(root.path().into(), deadline()).await { + Ok(lease) => { + drop(lease); + break; + } + Err(DirectoryLeaseError::InUse) => sleep(Duration::from_millis(5)).await, + other => panic!("unexpected reacquisition result: {other:?}"), + } + } + }) + .await + .unwrap(); + assert_eq!(pool.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_release_worker_capacity_after_a_panicking_worker() { + let pool = Box::leak(Box::new(Semaphore::new(1))); + let result: Result<(), _> = run(pool, deadline(), |_| panic!("worker failure")).await; + assert!(matches!(result, Err(DirectoryLeaseError::Worker))); + assert_eq!(pool.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_retain_capacity_with_a_completed_but_unconsumed_lease_result() { + let root = tempfile::tempdir().unwrap(); + let held = lease(&root).await; + let pool: &'static Semaphore = Box::leak(Box::new(Semaphore::new(1))); + let task = spawn_work( + pool.try_acquire().unwrap(), + Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + }, + |_| Ok(held), + ); + timeout(Duration::from_secs(5), async { + while !task.is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(pool.available_permits(), 0); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(task); + assert_eq!(pool.available_permits(), 1); + drop(lease(&root).await); +} + +#[tokio::test] +async fn test_should_expire_worker_admission_without_starting_io() { + let pool: &'static Semaphore = Box::leak(Box::new(Semaphore::new(1))); + let _held = pool.acquire().await.unwrap(); + let started = Arc::new(AtomicBool::new(false)); + let observed = started.clone(); + let result = run( + pool, + Instant::now() + Duration::from_millis(20), + move |_| { + started.store(true, Ordering::Release); + Ok(()) + }, + ) + .await; + assert!(matches!(result, Err(DirectoryLeaseError::Deadline))); + assert!(!observed.load(Ordering::Acquire)); +} + +#[tokio::test] +async fn test_should_enforce_exclusion_and_stable_identity_across_processes() { + const PATH: &str = "RUSTACK_DIRECTORY_LEASE_CHILD_PATH"; + const MODE: &str = "RUSTACK_DIRECTORY_LEASE_CHILD_MODE"; + if let Some(path) = std::env::var_os(PATH) { + let result = DirectoryLease::acquire(path.into(), deadline()).await; + if std::env::var(MODE).unwrap() == "busy" { + assert!(matches!(result, Err(DirectoryLeaseError::InUse))); + } else { + let lease = result.unwrap(); + assert_eq!( + lease.owner_id().to_string(), + std::env::var("RUSTACK_DIRECTORY_LEASE_CHILD_OWNER").unwrap() + ); + assert_ne!( + lease.run_id().to_string(), + std::env::var("RUSTACK_DIRECTORY_LEASE_CHILD_RUN").unwrap() + ); + } + return; + } + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let owner = first.owner_id(); + let run = first.run_id(); + assert_child(&root, "busy", owner, run).await; + drop(first); + assert_child(&root, "available", owner, run).await; +} + +async fn assert_child(root: &TempDir, mode: &str, owner: Uuid, run: Uuid) { + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "docker::ownership::tests::test_should_enforce_exclusion_and_stable_identity_across_processes", "--test-threads=1"]) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_PATH", root.path()) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_MODE", mode) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_OWNER", owner.to_string()) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_RUN", run.to_string()) + .stdout(Stdio::null()).kill_on_drop(true).spawn().unwrap(); + let status = if let Ok(status) = timeout(Duration::from_secs(10), child.wait()).await { + status.unwrap() + } else { + child.start_kill().unwrap(); + timeout(Duration::from_secs(5), child.wait()) + .await + .unwrap() + .unwrap(); + panic!("ownership child timed out"); + }; + assert!(status.success()); +} + +#[tokio::test] +async fn test_should_retain_legacy_provenance_without_rewriting_or_production_admission() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let owner = first.owner_id(); + first.require_journaled().unwrap(); + drop(first); + let mut bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + bytes.get_mut(..8).unwrap().copy_from_slice(LEGACY_MAGIC); + let checksum = Sha256::digest(bytes.get(..24).unwrap()); + bytes.get_mut(24..).unwrap().copy_from_slice(&checksum); + tokio::fs::write(managed(&root, OWNER), &bytes) + .await + .unwrap(); + let legacy = lease(&root).await; + assert_eq!(legacy.owner_id(), owner); + assert!(legacy.require_journaled().is_err()); + assert_eq!(tokio::fs::read(managed(&root, OWNER)).await.unwrap(), bytes); + drop(legacy); + let again = lease(&root).await; + assert_eq!(again.owner_id(), owner); + assert!(again.require_journaled().is_err()); +} + +#[rstest::rstest] +#[case("daemon.v1")] +#[case("daemon.pending")] +#[case("containers.v1")] +#[case("containers.pending")] +#[case("pins.v1")] +#[case("pins.pending")] +#[case("unknown-state")] +#[tokio::test] +async fn test_should_refuse_new_identity_over_unexplained_owner_missing_state(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + drop(first); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + tokio::fs::write(managed(&root, name), b"retained-evidence") + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!(!tokio::fs::try_exists(managed(&root, OWNER)).await.unwrap()); + assert_eq!( + tokio::fs::read(managed(&root, name)).await.unwrap(), + b"retained-evidence" + ); +} + +#[tokio::test] +async fn test_should_publish_journal_generation_before_effect_and_keep_downgrade_barrier() { + let root = tempfile::tempdir().unwrap(); + let owner = lease(&root).await; + owner.require_journaled().unwrap(); + let bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + assert_eq!(bytes.get(..8), Some(MAGIC.as_slice())); + assert_ne!(bytes.get(..8), Some(LEGACY_MAGIC.as_slice())); + let id = owner.owner_id(); + drop(owner); + let again = lease(&root).await; + again.require_journaled().unwrap(); + assert_eq!(again.owner_id(), id); +} + +#[tokio::test] +async fn test_should_retain_exact_blocking_acquisition_through_deadline_and_abandoned_observers() { + static POOL: Semaphore = Semaphore::const_new(1); + let root = tempfile::tempdir().unwrap(); + let data = DataDirectory::try_from(root.path().join("unpublished")).unwrap(); + let at = deadline(); + let cancelled = Arc::new(AtomicBool::new(false)); + let (release, held) = mpsc::sync_channel(1); + let (entered, saw) = oneshot::channel(); + let task = spawn_work( + POOL.acquire().await.unwrap(), + Control { + deadline: at, + cancelled: cancelled.clone(), + }, + move |control| { + entered.send(()).unwrap(); + held.recv().unwrap(); + acquire(&data, control) + }, + ); + let mut pending = DirectoryAcquisition { + code: false, + cancelled, + deadline: at, + state: AcquisitionState::Running(task), + }; + saw.await.unwrap(); + assert!(matches!( + pending + .wait(Instant::now() + Duration::from_millis(10)) + .await, + Err(DirectoryLeaseError::Deadline) + )); + pending.cancel(); + assert_eq!(POOL.available_permits(), 0); + assert!(matches!( + pending + .wait(Instant::now() + Duration::from_millis(10)) + .await, + Err(DirectoryLeaseError::Deadline) + )); + release.send(()).unwrap(); + assert!(matches!( + pending.wait(deadline()).await, + Ok(Err(DirectoryLeaseError::Cancelled)) + )); + assert!(matches!( + pending.wait(deadline()).await, + Ok(Err(DirectoryLeaseError::Worker)) + )); + assert_eq!(POOL.available_permits(), 1); + assert!(!root.path().join("unpublished").exists()); +} +#[tokio::test] +async fn test_should_cache_panicked_acquisition_completion_without_repolling_its_handle() { + static POOL: Semaphore = Semaphore::const_new(1); + let cancelled = Arc::new(AtomicBool::new(false)); + let task = spawn_work( + POOL.acquire().await.unwrap(), + Control { + deadline: deadline(), + cancelled: cancelled.clone(), + }, + |_| -> Result { + panic!("injected filesystem worker panic") + }, + ); + let mut pending = DirectoryAcquisition { + code: false, + cancelled, + deadline: deadline(), + state: AcquisitionState::Running(task), + }; + for _ in 0..2 { + assert!(matches!( + pending.wait(deadline()).await, + Ok(Err(DirectoryLeaseError::Worker)) + )); + } + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_keep_failed_ready_filesystem_result_charged_until_disposal() { + static POOL: Semaphore = Semaphore::const_new(1); + let task = spawn_work( + POOL.acquire().await.unwrap(), + Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + }, + |_| -> Result<(), DirectoryLeaseError> { Err(DirectoryLeaseError::InvalidState) }, + ); + timeout(Duration::from_secs(2), async { + while !task.is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(POOL.available_permits(), 0); + let (result, permit) = task.await.unwrap(); + assert!(matches!(result, Err(DirectoryLeaseError::InvalidState))); + assert_eq!(POOL.available_permits(), 0); + drop(permit); + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_acquire_code_profile_with_retained_job_and_refuse_older_profile_without_rewrite() + { + let root = tempfile::tempdir().unwrap(); + let at = Instant::now() + Duration::from_secs(5); + let mut job = DirectoryAcquisition::begin_code(root.path().into(), at).unwrap(); + let lease = job.wait(at).await.unwrap().unwrap(); + assert!(lease.code_journaled()); + drop(lease); + let old = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire(old.path().into(), at) + .await + .unwrap(); + drop(lease); + let owner = old.path().join(".rustack-lambda-docker/owner.v1"); + let before = fs::read(&owner).await.unwrap(); + let mut job = DirectoryAcquisition::begin_code(old.path().into(), at).unwrap(); + assert!(matches!( + job.wait(at).await.unwrap(), + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(fs::read(owner).await.unwrap(), before); +} diff --git a/crates/rustack-lambda-core/src/docker/pin.rs b/crates/rustack-lambda-core/src/docker/pin.rs new file mode 100644 index 00000000..5fa9a112 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pin.rs @@ -0,0 +1,264 @@ +//! Only generated private alias identities can reach image mutation endpoints. + +use std::fmt; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + DirectoryLease, DockerError, DockerPort, Operation, RawRequest, decode, escaped, + models::{ImageDeleteResponseItem, ImageInspect}, + mutation::MutationGuard, + unit, +}; +use crate::artifact::{ImageReference, Sha256Digest}; + +/// Trusted runtime owner/run identity; never populated from a Lambda request. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ImagePinScope { + lease: DirectoryLease, +} +impl ImagePinScope { + pub(super) fn directory(&self) -> &DirectoryLease { + &self.lease + } + + /// Retain a real directory lease for this namespace and its allocated pins. + /// + /// ```no_run + /// # fn scope(lease: rustack_lambda_core::docker::DirectoryLease) { + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(lease); + /// # } + /// ``` + #[must_use] + pub fn new(lease: DirectoryLease) -> Self { + Self { lease } + } + + #[cfg(test)] + pub(super) fn testing(owner: Uuid, run: Uuid) -> Result { + if owner.is_nil() || run.is_nil() { + return Err(DockerError::Configuration); + } + Ok(Self::new(DirectoryLease::testing(owner, run))) + } + /// Allocate a fresh private alias before issuing any Docker mutation. + /// + /// # Errors + /// Returns validation failure if the generated reference cannot validate. + pub(super) fn allocate(&self) -> Result { + ImagePin::historical(self.lease.clone(), self.lease.run_id(), Uuid::new_v4()) + } +} + +/// An allocated alias; arbitrary source references cannot become cleanup keys. +#[derive(Clone)] +#[non_exhaustive] +pub struct ImagePin { + pub(super) id: Uuid, + pub(super) run: Uuid, + reference: ImageReference, + scope: ImagePinScope, +} +impl ImagePin { + pub(super) fn historical( + lease: DirectoryLease, + run: Uuid, + id: Uuid, + ) -> Result { + if run.is_nil() || id.is_nil() { + return Err(DockerError::Configuration); + } + let reference = format!( + "rustack-internal/lambda-{}/{}:{}", + lease.owner_id().simple(), + run.simple(), + id.simple() + ) + .parse()?; + Ok(Self { + id, + run, + reference, + scope: ImagePinScope::new(lease), + }) + } + pub(super) fn directory(&self) -> &DirectoryLease { + &self.scope.lease + } +} +impl fmt::Debug for ImagePin { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ImagePin") + .field("id", &self.id) + .finish_non_exhaustive() + } +} +impl ImagePin { + /// Exact private alias; its namespace is reserved for the owning runtime. + /// + /// ```no_run + /// # fn reference(pin:&rustack_lambda_core::docker::ImagePin) { + /// let reference = pin.reference(); + /// # } + /// ``` + #[must_use] + pub fn reference(&self) -> &ImageReference { + &self.reference + } +} + +impl DockerPort { + /// Create only the allocated private alias, targeting a captured image ID. + /// + /// # Errors + /// Returns bounded wire failure. Any uncertain outcome retains pin ownership. + /// + /// ```no_run + /// # async fn tag(port:&rustack_lambda_core::docker::DockerPort,id:&rustack_lambda_core::artifact::Sha256Digest,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.tag_image_pin(id,pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn tag_image_pin( + &self, + id: &Sha256Digest, + pin: &ImagePin, + deadline: Instant, + ) -> Result<(), DockerError> { + self.tag_image_pin_inner(id, pin, deadline, None).await + } + pub(super) async fn tag_image_pin_tracked( + &self, + id: &Sha256Digest, + pin: &ImagePin, + deadline: Instant, + mutation: MutationGuard, + ) -> Result<(), DockerError> { + self.tag_image_pin_inner(id, pin, deadline, Some(mutation)) + .await + } + async fn tag_image_pin_inner( + &self, + id: &Sha256Digest, + pin: &ImagePin, + deadline: Instant, + mutation: Option, + ) -> Result<(), DockerError> { + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("repo", pin.reference.repository()) + .append_pair("tag", &pin.id.simple().to_string()) + .finish(); + let reply = self + .raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation, + operation: Operation::TagPin, + method: Method::POST, + target: format!( + "/v{}/images/{}/tag?{query}", + self.version, + escaped(id.as_str()) + ), + body: Bytes::new(), + expected: StatusCode::CREATED, + }, + deadline, + ) + .await?; + unit(reply, deadline) + } + + /// Inspect a known private alias using reserved cleanup capacity. + /// + /// # Errors + /// Returns missing alias status, bounded metadata or transport failure. + /// + /// ```no_run + /// # async fn inspect(port:&rustack_lambda_core::docker::DockerPort,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.inspect_image_pin(pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_image_pin( + &self, + pin: &ImagePin, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::InspectPin, + Method::GET, + format!("/images/{}/json", escaped(pin.reference.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Remove only an allocated private alias, without force or ancestor pruning. + /// + /// # Errors + /// Returns status, bounded response or transport failure. The owner verifies + /// identity under its exclusive namespace before removal and confirms absence. + /// + /// ```no_run + /// # async fn remove(port:&rustack_lambda_core::docker::DockerPort,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.remove_image_pin(pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn remove_image_pin( + &self, + pin: &ImagePin, + deadline: Instant, + ) -> Result, DockerError> { + decode( + self.call( + Operation::RemovePin, + Method::DELETE, + format!( + "/images/{}?force=false&noprune=true", + escaped(pin.reference.as_str()) + ), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + pub(super) async fn image_manifests( + &self, + id: &Sha256Digest, + deadline: Instant, + ) -> Result { + let suffix = if self.api_version().minor() >= 48 { + "?manifests=true" + } else { + "" + }; + decode( + self.call( + Operation::Image, + Method::GET, + format!("/images/{}/json{suffix}", escaped(id.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } +} diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs new file mode 100644 index 00000000..21c77981 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -0,0 +1,1007 @@ +//! Immutable, bounded container creation policy; lifetime ownership remains above IO. + +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + str::FromStr, + sync::Arc, + time::Duration, +}; + +use rustack_lambda_model::types::ImageConfig; +use rustack_lambda_runtime::protocol::EnvironmentId; +use serde::Serialize; +use tokio::time::Instant; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + CodeVolumeIdentity, DirectoryLease, DockerError, DockerKey, DockerPort, DockerSdkEnvironment, + DockerZipRuntime, + models::{ + ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, HealthConfig, + HostConfig, HostConfigCgroupnsModeEnum, HostConfigLogConfig, Mount, MountType, + MountVolumeOptions, RestartPolicy, RestartPolicyNameEnum, + }, + mutation::MutationGuard, + sdk::SdkAffiliation, + zip::{AllocationClaim, ZipPlan}, +}; +use crate::{ + artifact::{DeploymentArtifact, PinnedImage}, + executor::{ExecutionRevision, ExecutorBackend}, + function_config::{validate_environment_entries, validate_provided_settings}, + storage::compute_sha256, +}; + +const MIB: u64 = 1024 * 1024; +const RELAY_MEMORY: u64 = 64 * MIB; +const CPU_QUANTUM: u64 = 10_000_000; + +#[derive(Debug, Clone, Copy)] +pub(super) enum ContainerRole { + Relay, + Function, +} + +/// Operator network selection; a bridge still requires daemon startup validation. +#[derive(Clone, PartialEq, Eq, Serialize)] +#[non_exhaustive] +pub struct NetworkSelection(String); +impl NetworkSelection { + pub(super) fn value(&self) -> &str { + &self.0 + } +} +impl fmt::Debug for NetworkSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("NetworkSelection()") + } +} +impl FromStr for NetworkSelection { + type Err = DockerError; + fn from_str(value: &str) -> Result { + if value == "none" { + return Ok(Self(value.into())); + } + if matches!(value, "host" | "bridge" | "default") { + return Err(DockerError::Configuration); + } + value.parse::()?; + Ok(Self(value.into())) + } +} + +/// Integral relay session budgets independent of handler and deployment budgets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[non_exhaustive] +pub struct RelayBudgets { + io: Duration, + idle: Duration, + retirement: Duration, +} +impl Default for RelayBudgets { + fn default() -> Self { + Self { + io: Duration::from_secs(5), + idle: Duration::from_secs(600), + retirement: Duration::from_secs(10), + } + } +} +impl RelayBudgets { + /// Validate whole-second IO/retirement1..60s and idle1..86400s budgets. + /// + /// # Errors + /// Rejects zero, fractional or excessive durations. + /// + /// ``` + /// let budgets = rustack_lambda_core::docker::RelayBudgets::new( + /// std::time::Duration::from_secs(5), std::time::Duration::from_secs(600), + /// std::time::Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(io: Duration, idle: Duration, retirement: Duration) -> Result { + for (duration, maximum) in [(io, 60), (idle, 86400), (retirement, 60)] { + if duration.subsec_nanos() != 0 || !(1..=maximum).contains(&duration.as_secs()) { + return Err(DockerError::Configuration); + } + } + Ok(Self { + io, + idle, + retirement, + }) + } +} + +/// Validated operator limits, independent of user function configuration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[non_exhaustive] +pub struct ContainerPolicy { + pub(super) network: NetworkSelection, + uid: u32, + gid: u32, + requested_nano_cpus: u64, + pub(super) nano_cpus: i64, + relay: RelayBudgets, +} +impl ContainerPolicy { + pub(super) fn idle(&self) -> Duration { + self.relay.idle + } + /// Return the requested maximum and its downward-normalized CPU allocation. + /// + /// ``` + /// use rustack_lambda_core::docker::{ContainerPolicy, RelayBudgets}; + /// let policy = ContainerPolicy::new("none".parse()?, 1000, 1000, + /// 333_333_333, RelayBudgets::default())?; + /// assert_eq!(policy.cpu_allocation().effective_nano_cpus, 330_000_000); + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn cpu_allocation(&self) -> DockerCpuAllocation { + DockerCpuAllocation { + requested_nano_cpus: self.requested_nano_cpus, + effective_nano_cpus: self.nano_cpus.unsigned_abs(), + } + } + /// Validate UID/GID and0.1..16 CPUs, rounding CPU down to0.01 increments. + /// + /// # Errors + /// Rejects invalid IDs or CPU values, preserving bounded operator policy. + /// + /// ``` + /// let policy = rustack_lambda_core::docker::ContainerPolicy::new( + /// "none".parse()?, 1000, 1000, 1_000_000_000, + /// rustack_lambda_core::docker::RelayBudgets::default())?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new( + network: NetworkSelection, + uid: u32, + gid: u32, + nano_cpus: u64, + relay: RelayBudgets, + ) -> Result { + if !(1..=2_147_483_647).contains(&uid) + || !(1..=2_147_483_647).contains(&gid) + || !(100_000_000..=16_000_000_000).contains(&nano_cpus) + { + return Err(DockerError::Configuration); + } + Ok(Self { + network, + uid, + gid, + requested_nano_cpus: nano_cpus, + nano_cpus: i64::try_from(nano_cpus - nano_cpus % CPU_QUANTUM) + .map_err(|_| DockerError::Configuration)?, + relay, + }) + } +} + +/// Captured CPU maximum and the portable allocation actually sent to Docker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct DockerCpuAllocation { + /// Original validated operator maximum, in billionths of one CPU. + pub requested_nano_cpus: u64, + /// Maximum rounded down to one-percent CPU increments, never upward. + pub effective_nano_cpus: u64, +} + +struct Identity { + environment: EnvironmentId, + relay: DockerKey, + function: DockerKey, +} +struct Bodies { + relay: ContainerCreateBody, + function: ContainerCreateBody, + memory: u64, +} + +/// Fixed create requests retaining revision, image availability and namespace leases. +#[derive(TypedBuilder)] +#[builder(builder_method(vis = ""))] +#[non_exhaustive] +pub struct EnvironmentPlan { + pub(super) directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + identity: Identity, + bodies: Bodies, + warm_key: String, + #[builder(default)] + pub(super) zip: Option>, + #[builder(default)] + sdk: Option, +} +impl fmt::Debug for EnvironmentPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPlan") + .field("environment", &self.identity.environment) + .field("resource", &self.revision.resource_id) + .field("memory_bytes", &self.bodies.memory) + .finish_non_exhaustive() + } +} +impl EnvironmentPlan { + pub(super) fn with_sdk_environment( + mut self, + sdk: &DockerSdkEnvironment, + ) -> Result { + let guard = sdk.liveness(); + guard.live()?; + let entries = self + .bodies + .function + .env + .take() + .ok_or(DockerError::Configuration)?; + self.bodies.function.env = Some(sdk.environment(entries)?); + self.warm_key = + compute_sha256(format!("sdk-v1:{}:{}", self.warm_key, sdk.identity()).as_bytes()); + self.sdk = Some(guard); + Ok(self) + } + pub(super) fn image_id(&self, role: ContainerRole) -> &str { + match role { + ContainerRole::Relay => self.relay.resolved().metadata().image_id.as_str(), + ContainerRole::Function => self.bodies.function.image.as_deref().unwrap_or_default(), + } + } + pub(super) fn warm_key(&self) -> &str { + &self.warm_key + } + pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { + if self.zip.is_some() { + self.directory.same_instance(directory) + } else { + self.directory.owner_id() == directory.owner_id() + && self.directory.run_id() == directory.run_id() + } + } + pub(super) fn revision(&self) -> &ExecutionRevision { + &self.revision + } + pub(super) fn key(&self, role: ContainerRole) -> &DockerKey { + match role { + ContainerRole::Relay => self.relay_key(), + ContainerRole::Function => self.function_key(), + } + } + pub(super) fn owned( + &self, + role: ContainerRole, + inspected: &ContainerInspectResponse, + ) -> Result { + let body = match role { + ContainerRole::Relay => &self.bodies.relay, + ContainerRole::Function => &self.bodies.function, + }; + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let expected = body.labels.as_ref().ok_or(DockerError::Configuration)?; + let actual = config.labels.as_ref().ok_or(DockerError::Quarantined)?; + let id = inspected.id.as_deref().ok_or(DockerError::Protocol)?; + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || inspected.name.as_deref() != Some(format!("/{}", self.key(role).as_str()).as_str()) + || config.image != body.image + || !expected + .iter() + .all(|(key, value)| actual.get(key) == Some(value)) + { + return Err(DockerError::Quarantined); + } + if matches!(role, ContainerRole::Function) { + if let Some(zip) = &self.zip { + let mounts = inspected.mounts.as_ref().ok_or(DockerError::Quarantined)?; + if mounts.len() != 1 + || mounts.first().is_none_or(|mount| { + mount.typ.as_deref() != Some("volume") + || mount.name.as_deref() != Some(zip.volume.as_str()) + || mount.destination.as_deref() != Some("/var/task") + || mount.rw != Some(false) + || mount.driver.as_deref() != Some("local") + }) + { + return Err(DockerError::Quarantined); + } + } + } + id.parse() + } + pub(super) fn claim_code(&self) -> Result, DockerError> { + self.zip.as_ref().map(ZipPlan::claim).transpose() + } + pub(super) fn commit_code_claim( + &self, + claim: &mut Option, + ) -> Result<(), DockerError> { + match (&self.zip, claim) { + (Some(plan), Some(claim)) if claim.belongs_to(plan) => { + claim.commit(); + Ok(()) + } + (None, None) => Ok(()), + _ => Err(DockerError::Configuration), + } + } + pub(super) fn code_clean(&self) -> bool { + self.zip.as_ref().is_none_or(|zip| zip.clean()) + } + fn require_code(&self, role: ContainerRole) -> Result<(), DockerError> { + if matches!(role, ContainerRole::Function) { + if let Some(zip) = &self.zip { + zip.ready_mount()?; + } + } + Ok(()) + } + /// Allocate identities and validate both immutable create bodies before IO. + /// + /// # Errors + /// Rejects unavailable/mismatched artifacts, invalid captured settings or + /// effective image configuration. This does not reserve environment capacity. + /// + /// ```no_run + /// # fn plan(directory: rustack_lambda_core::docker::DirectoryLease, relay: rustack_lambda_core::artifact::PinnedImage, revision: std::sync::Arc, policy: rustack_lambda_core::docker::ContainerPolicy) { + /// let plan = rustack_lambda_core::docker::EnvironmentPlan::new(directory, relay, revision, policy); + /// # } + /// ``` + pub fn new( + directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + policy: ContainerPolicy, + ) -> Result { + Self::construct(directory, relay, revision, policy, None) + } + + /// Plan provided.al2023 execution over an affiliated immutable base and producer. + /// + /// Conversion and allocation begin only after environment admission. + /// + /// # Errors + /// Rejects incompatible runtime/revision, source scope, platform or policy. + /// + /// ```no_run + /// # fn plan(directory:rustack_lambda_core::docker::DirectoryLease,relay:rustack_lambda_core::artifact::PinnedImage,revision:std::sync::Arc,policy:rustack_lambda_core::docker::ContainerPolicy,source:rustack_lambda_core::docker::DockerZipRuntime) { + /// let plan=rustack_lambda_core::docker::EnvironmentPlan::new_zip(directory,relay,revision,policy,source); + /// # } + /// ``` + pub fn new_zip( + directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + policy: ContainerPolicy, + source: DockerZipRuntime, + ) -> Result { + Self::construct(directory, relay, revision, policy, Some(source)) + } + + fn construct( + directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + policy: ContainerPolicy, + zip: Option, + ) -> Result { + validate_revision(&revision)?; + let function = match (&revision.artifact, &zip) { + (DeploymentArtifact::Image(_), None) => image(&revision)?, + (DeploymentArtifact::Zip(_), Some(source)) => { + if !source.same_scope(&directory) + || source.base.resolved().metadata().architecture != revision.architecture + { + return Err(DockerError::Configuration); + } + source.available()?; + &source.base + } + _ => return Err(DockerError::Configuration), + }; + if !relay.is_live() || relay.resolved().metadata().architecture != revision.architecture { + return Err(DockerError::Configuration); + } + let code = if zip.is_some() { + Some(CodeVolumeIdentity::new( + revision + .resource_id + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + revision + .revision_id + .parse() + .map_err(|_| DockerError::Configuration)?, + )?) + } else { + None + }; + let environment = match &code { + Some(code) => code + .environment_id() + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + None => EnvironmentId::default(), + }; + let identity = Identity { + environment, + relay: format!("rustack-relay-{environment}").parse()?, + function: format!("rustack-function-{environment}").parse()?, + }; + let labels = plan_labels(&directory, environment, &revision); + let (mut function_body, memory) = + function_create_body(function, &revision, &identity, &policy, &labels)?; + let warm_key = warm_identity(&directory, &relay, &revision, &policy, function)?; + let relay_body = relay_create_body(&relay, policy, environment, &labels)?; + let zip = match (zip, code) { + (Some(source), Some(code)) => Some(ZipPlan::new(source, code)?), + (None, None) => None, + _ => return Err(DockerError::Configuration), + }; + if let Some(zip) = &zip { + function_body + .host_config + .as_mut() + .ok_or(DockerError::Configuration)? + .mounts = Some(vec![Mount { + typ: Some(MountType::VOLUME), + source: Some(zip.volume.as_str().into()), + target: Some("/var/task".into()), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]); + } + let memory = memory + .checked_add(RELAY_MEMORY) + .ok_or(DockerError::Configuration)?; + Ok(Self::builder() + .directory(directory) + .relay(relay) + .revision(revision) + .identity(identity) + .bodies(Bodies { + relay: relay_body, + function: function_body, + memory, + }) + .warm_key(warm_key) + .zip(zip) + .build()) + } + + /// Fresh environment UUID used by both labels and control protocol. + /// + /// ```no_run + /// # fn id(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let id = plan.environment_id(); + /// # } + /// ``` + #[must_use] + pub fn environment_id(&self) -> EnvironmentId { + self.identity.environment + } + + /// Preallocated relay identity retained through uncertain allocation cleanup. + /// + /// ```no_run + /// # fn key(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let name = plan.relay_key(); + /// # } + /// ``` + #[must_use] + pub fn relay_key(&self) -> &DockerKey { + &self.identity.relay + } + + /// Preallocated function identity; never populated from invocation input. + /// + /// ```no_run + /// # fn key(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let name = plan.function_key(); + /// # } + /// ``` + #[must_use] + pub fn function_key(&self) -> &DockerKey { + &self.identity.function + } + + /// Checked function plus relay memory required before allocation admission. + /// + /// ```no_run + /// # fn memory(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let bytes = plan.memory_bytes(); + /// # } + /// ``` + #[must_use] + pub fn memory_bytes(&self) -> u64 { + self.bodies.memory + } + + /// Create the relay after its owner has recorded uncertain allocation. + /// + /// # Errors + /// Returns revoked availability, deadline, wire/status or schema failure. + /// Every uncertain outcome still requires owner reconciliation. + /// + /// ```no_run + /// # async fn create(plan: &rustack_lambda_core::docker::EnvironmentPlan, port: &rustack_lambda_core::docker::DockerPort) { + /// let created = plan.create_relay(port, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_relay( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + self.live()?; + port.create_container_on_platform( + &self.identity.relay, + &self.bodies.relay, + self.revision.architecture, + deadline, + ) + .await + } + + /// Create the function only after its own relay is verified ready. + /// + /// # Errors + /// Returns revoked availability, deadline, wire/status or schema failure. + /// Caller owns startup ordering and every uncertain allocation outcome. + /// + /// ```no_run + /// # async fn create(plan: &rustack_lambda_core::docker::EnvironmentPlan, port: &rustack_lambda_core::docker::DockerPort) { + /// let created = plan.create_function(port, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_function( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + self.live()?; + self.require_code(ContainerRole::Function)?; + port.create_container_on_platform( + &self.identity.function, + &self.bodies.function, + self.revision.architecture, + deadline, + ) + .await + } + pub(super) async fn create_tracked( + &self, + role: ContainerRole, + port: &DockerPort, + deadline: Instant, + mutation: MutationGuard, + ) -> Result { + self.live()?; + self.require_code(role)?; + let body = match role { + ContainerRole::Relay => &self.bodies.relay, + ContainerRole::Function => &self.bodies.function, + }; + port.create_container_inner( + self.key(role), + body, + Some(self.revision.architecture), + deadline, + Some(mutation), + ) + .await + } + pub(super) fn live(&self) -> Result<(), DockerError> { + if let Some(sdk) = &self.sdk { + sdk.live()?; + } + match &self.revision.artifact { + DeploymentArtifact::Image(function) if self.relay.is_live() && function.is_live() => { + Ok(()) + } + DeploymentArtifact::Zip(_) if self.relay.is_live() => { + self.zip.as_ref().ok_or(DockerError::Configuration)?.live() + } + DeploymentArtifact::Image(_) | DeploymentArtifact::Zip(_) => Err(DockerError::Stopped), + } + } +} + +fn plan_labels( + directory: &DirectoryLease, + environment: EnvironmentId, + revision: &ExecutionRevision, +) -> HashMap { + HashMap::from([ + ( + "rustack.lambda.ownerId".into(), + directory.owner_id().to_string(), + ), + ( + "rustack.lambda.runId".into(), + directory.run_id().to_string(), + ), + ( + "rustack.lambda.environmentId".into(), + environment.to_string(), + ), + ( + "rustack.lambda.functionResourceId".into(), + revision.resource_id.to_string(), + ), + ]) +} +fn relay_create_body( + relay: &PinnedImage, + policy: ContainerPolicy, + environment: EnvironmentId, + labels: &HashMap, +) -> Result { + let user = format!("{}:{}", policy.uid, policy.gid); + let mut relay = body(relay, None, &user, labels, "relay")?; + relay.env = Some(vec![ + format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={environment}"), + format!( + "RUSTACK_LAMBDA_RELAY_IO_SECONDS={}", + policy.relay.io.as_secs() + ), + format!( + "RUSTACK_LAMBDA_RELAY_IDLE_SECONDS={}", + policy.relay.idle.as_secs() + ), + format!( + "RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS={}", + policy.relay.retirement.as_secs() + ), + ]); + relay.attach_stdin = Some(true); + relay.open_stdin = Some(true); + relay.stdin_once = Some(true); + relay.host_config = Some(limits( + policy.network.0, + RELAY_MEMORY, + 100_000_000, + 64, + MIB, + )?); + Ok(relay) +} +fn function_create_body( + function: &PinnedImage, + revision: &ExecutionRevision, + identity: &Identity, + policy: &ContainerPolicy, + labels: &HashMap, +) -> Result<(ContainerCreateBody, u64), DockerError> { + let user = format!("{}:{}", policy.uid, policy.gid); + let zip = matches!(revision.artifact, DeploymentArtifact::Zip(_)); + let selected_user = if zip { + user.clone() + } else { + image_user( + &function.resolved().metadata().execution.metadata().user, + &user, + )? + }; + let provided = ImageConfig { + entry_point: Some(vec!["/var/task/bootstrap".into()]), + command: Some(Vec::new()), + working_directory: Some("/var/task".into()), + }; + let overrides = if zip { + Some(&provided) + } else { + revision.image_config.as_ref() + }; + let mut body = body(function, overrides, &selected_user, labels, "function")?; + body.env = Some(function_environment( + revision, + identity.environment, + function, + )?); + let memory = u64::from(revision.memory_mb) + .checked_mul(MIB) + .ok_or(DockerError::Configuration)?; + body.host_config = Some(limits( + format!("container:{}", identity.relay.as_str()), + memory, + policy.nano_cpus, + 256, + u64::from(revision.ephemeral_storage_mb) + .checked_mul(MIB) + .ok_or(DockerError::Configuration)?, + )?); + Ok((body, memory)) +} + +fn warm_identity( + directory: &DirectoryLease, + relay: &PinnedImage, + revision: &ExecutionRevision, + policy: &ContainerPolicy, + function: &PinnedImage, +) -> Result { + let environment: BTreeMap<_, _> = revision.environment.iter().collect(); + let encoded = serde_json::to_vec(&( + ( + directory.owner_id().to_string(), + directory.run_id().to_string(), + ), + ( + revision.resource_id.to_string(), + &revision.revision_id, + &revision.function_name, + &revision.qualifier, + ), + ( + revision.architecture, + function.resolved().metadata(), + relay.resolved().metadata(), + ), + ( + &revision.image_config, + environment, + revision.timeout, + revision.memory_mb, + revision.ephemeral_storage_mb, + ), + ( + &policy, + &revision.runtime, + &revision.handler, + match &revision.artifact { + DeploymentArtifact::Zip(code) => Some(code.sha256()), + DeploymentArtifact::Image(_) => None, + }, + ), + )) + .map_err(|_| DockerError::Configuration)?; + Ok(compute_sha256(&encoded)) +} + +fn image(revision: &ExecutionRevision) -> Result<&PinnedImage, DockerError> { + match &revision.artifact { + DeploymentArtifact::Image(image) + if image.is_live() + && image.resolved().metadata().architecture == revision.architecture => + { + Ok(image) + } + _ => Err(DockerError::Configuration), + } +} +fn validate_revision(revision: &ExecutionRevision) -> Result<(), DockerError> { + if revision.revision_id.len() != 36 { + return Err(DockerError::Configuration); + } + let id = Uuid::parse_str(&revision.revision_id).map_err(|_| DockerError::Configuration)?; + let version = &revision.qualifier; + let numeric_version = version.len() <= 20 + && version + .parse::() + .is_ok_and(|number| number > 0 && number.to_string() == *version); + if revision.backend != ExecutorBackend::Docker + || id.is_nil() + || id.to_string() != revision.revision_id + || !(1..=64).contains(&revision.function_name.len()) + || !revision + .function_name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte)) + || (version != "$LATEST" && !numeric_version) + || !(1..=900).contains(&revision.timeout.as_secs()) + || revision.timeout.subsec_nanos() != 0 + || !(128..=10240).contains(&revision.memory_mb) + || !(512..=10240).contains(&revision.ephemeral_storage_mb) + { + return Err(DockerError::Configuration); + } + match &revision.artifact { + DeploymentArtifact::Image(_) + if revision.runtime.is_none() && revision.handler.is_none() => + { + image(revision)?; + } + DeploymentArtifact::Zip(_) if revision.image_config.is_none() => { + validate_provided_settings(revision.runtime.as_deref(), revision.handler.as_deref()) + .map_err(|_| DockerError::Configuration)?; + } + _ => return Err(DockerError::Configuration), + } + Ok(()) +} + +fn body( + image: &PinnedImage, + overrides: Option<&ImageConfig>, + user: &str, + labels: &HashMap, + role: &str, +) -> Result { + let plan = image.resolved().metadata().execution.plan(overrides)?; + let mut labels = labels.clone(); + labels.insert("rustack.lambda.role".into(), role.into()); + Ok(ContainerCreateBody { + image: Some(image.resolved().metadata().image_id.as_str().into()), + entrypoint: Some(plan.entrypoint().to_vec()), + cmd: Some(plan.command().to_vec()), + working_dir: Some(plan.working_directory().into()), + user: Some(user.into()), + labels: Some(labels), + tty: Some(false), + attach_stdin: Some(false), + attach_stdout: Some(true), + attach_stderr: Some(true), + open_stdin: Some(false), + stdin_once: Some(false), + healthcheck: Some(HealthConfig { + test: Some(vec!["NONE".into()]), + ..Default::default() + }), + ..Default::default() + }) +} + +fn limits( + network: String, + memory: u64, + cpu: i64, + pids: i64, + temporary: u64, +) -> Result { + let memory = i64::try_from(memory).map_err(|_| DockerError::Configuration)?; + Ok(HostConfig { + cgroupns_mode: Some(HostConfigCgroupnsModeEnum::PRIVATE), + network_mode: Some(network), + memory: Some(memory), + memory_swap: Some(memory), + oom_kill_disable: Some(false), + nano_cpus: Some(cpu), + pids_limit: Some(pids), + readonly_rootfs: Some(true), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + privileged: Some(false), + publish_all_ports: Some(false), + auto_remove: Some(false), + restart_policy: Some(RestartPolicy { + name: Some(RestartPolicyNameEnum::NO), + maximum_retry_count: Some(0), + }), + log_config: Some(HostConfigLogConfig { + typ: Some("none".into()), + config: None, + }), + tmpfs: Some(HashMap::from([( + "/tmp".into(), + format!("rw,nosuid,nodev,size={temporary},mode=1777"), + )])), + ..Default::default() + }) +} + +fn image_user(value: &str, fallback: &str) -> Result { + if value.is_empty() { + return Ok(fallback.into()); + } + let mut parts = value.split(':'); + let user = parts.next().ok_or(DockerError::Configuration)?; + let group = parts.next(); + if parts.next().is_some() || !account(user) || group.is_some_and(|group| !account(group)) { + return Err(DockerError::Configuration); + } + if user == "root" || user.parse::().is_ok_and(|uid| uid == 0) { + Ok(fallback.into()) + } else { + Ok(value.into()) + } +} +fn account(value: &str) -> bool { + if value.is_empty() || value.len() > 64 { + return false; + } + if value.bytes().all(|byte| byte.is_ascii_digit()) { + return value.parse::().is_ok_and(|id| id <= 2_147_483_647); + } + value + .as_bytes() + .first() + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) +} + +fn function_environment( + revision: &ExecutionRevision, + environment: EnvironmentId, + image: &PinnedImage, +) -> Result, DockerError> { + if revision.environment.len() > 130 { + return Err(DockerError::Configuration); + } + validate_environment_entries( + revision + .environment + .iter() + .filter(|(key, _)| !matches!(key.as_str(), "AWS_REGION" | "AWS_DEFAULT_REGION")), + ) + .map_err(|_| DockerError::Configuration)?; + let region = revision + .environment + .get("AWS_REGION") + .ok_or(DockerError::Configuration)?; + if !(1..=64).contains(®ion.len()) + || !region + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + || revision.environment.get("AWS_DEFAULT_REGION") != Some(region) + { + return Err(DockerError::Configuration); + } + let mut output: BTreeMap> = BTreeMap::new(); + for entry in &image.resolved().metadata().execution.metadata().environment { + let (key, value) = entry + .split_once('=') + .map_or((entry.as_str(), None), |(key, value)| { + (key, Some(value.to_owned())) + }); + output.insert(key.to_owned(), value); + } + for (key, value) in &revision.environment { + output.insert(key.clone(), Some(value.clone())); + } + let authoritative = [ + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001".into(), + format!("AWS_LAMBDA_FUNCTION_NAME={}", revision.function_name), + format!("AWS_LAMBDA_FUNCTION_VERSION={}", revision.qualifier), + format!("AWS_LAMBDA_FUNCTION_MEMORY_SIZE={}", revision.memory_mb), + format!("AWS_LAMBDA_FUNCTION_TIMEOUT={}", revision.timeout.as_secs()), + "AWS_LAMBDA_INITIALIZATION_TYPE=on-demand".into(), + format!( + "AWS_LAMBDA_LOG_GROUP_NAME=/aws/lambda/{}", + revision.function_name + ), + format!("AWS_LAMBDA_LOG_STREAM_NAME=rustack/{environment}"), + "LAMBDA_TASK_ROOT=/var/task".into(), + ]; + if let DeploymentArtifact::Zip(_) = &revision.artifact { + output.insert( + "_HANDLER".into(), + Some(revision.handler.clone().ok_or(DockerError::Configuration)?), + ); + output.insert( + "AWS_EXECUTION_ENV".into(), + Some("AWS_Lambda_provided.al2023".into()), + ); + } + for entry in authoritative { + let (key, value) = entry.split_once('=').ok_or(DockerError::Configuration)?; + output.insert(key.to_owned(), Some(value.to_owned())); + } + Ok(output + .into_iter() + .map(|(key, value)| match value { + Some(value) => format!("{key}={value}"), + None => key, + }) + .collect()) +} + +#[cfg(test)] +#[path = "plan_tests.rs"] +pub(super) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/plan_tests.rs b/crates/rustack-lambda-core/src/docker/plan_tests.rs new file mode 100644 index 00000000..06d1e7aa --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/plan_tests.rs @@ -0,0 +1,643 @@ +use std::{collections::HashSet, sync::atomic::AtomicBool}; + +use tokio::{ + sync::watch, + time::{Instant, timeout}, +}; + +use super::*; +use crate::{ + artifact::{ + Architecture, ImageExecutionMetadata, ImageMetadata, ResolvedImage, ValidatedArtifact, + }, + identity::FunctionResourceId, +}; + +fn directory() -> DirectoryLease { + DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()) +} +pub(in crate::docker) fn policy() -> ContainerPolicy { + ContainerPolicy::new( + "none".parse().unwrap(), + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap() +} + +#[rstest::rstest] +#[case(100_000_000, 100_000_000)] +#[case(109_999_999, 100_000_000)] +#[case(333_333_333, 330_000_000)] +#[case(1_000_000_000, 1_000_000_000)] +#[case(15_999_999_999, 15_990_000_000)] +#[case(16_000_000_000, 16_000_000_000)] +fn test_should_capture_requested_cpu_and_normalize_without_exceeding_it( + #[case] requested: u64, + #[case] effective: u64, +) -> anyhow::Result<()> { + let policy = ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + requested, + RelayBudgets::default(), + )?; + let allocation = policy.cpu_allocation(); + assert_eq!(allocation.requested_nano_cpus, requested); + assert_eq!(allocation.effective_nano_cpus, effective); + assert_eq!(u64::try_from(policy.nano_cpus)?, effective); + assert_eq!( + serde_json::to_value(allocation)?, + serde_json::json!({ + "requestedNanoCpus": requested, "effectiveNanoCpus": effective, + }) + ); + Ok(()) +} + +proptest::proptest! { + #[test] + fn test_should_keep_normalization_bounded_and_idempotent(requested in 100_000_000_u64..=16_000_000_000_u64) { + let first = ContainerPolicy::new("none".parse()?, 1000, 1000, requested, RelayBudgets::default())?; + let allocation = first.cpu_allocation(); + proptest::prop_assert!(allocation.effective_nano_cpus <= requested); + proptest::prop_assert!(requested - allocation.effective_nano_cpus < CPU_QUANTUM); + proptest::prop_assert_eq!(allocation.effective_nano_cpus % CPU_QUANTUM, 0); + let second = ContainerPolicy::new(first.network.clone(), 1000, 1000, + allocation.effective_nano_cpus, RelayBudgets::default())?; + proptest::prop_assert_eq!(first.nano_cpus, second.nano_cpus); + } +} + +#[rstest::rstest] +#[case(0)] +#[case(99_999_999)] +#[case(16_000_000_001)] +fn test_should_reject_requested_cpu_outside_the_original_range(#[case] requested: u64) { + assert!( + ContainerPolicy::new( + "none".parse().unwrap(), + 1000, + 1000, + requested, + RelayBudgets::default() + ) + .is_err() + ); +} +pub(in crate::docker) fn image_with(user: &str, environment: Vec) -> PinnedImage { + let mut metadata = ResolvedImage::fixture(Architecture::Arm64) + .metadata() + .clone(); + metadata.execution = ImageExecutionMetadata::builder() + .entrypoint(vec!["/bootstrap".into()]) + .command(vec!["handler".into()]) + .working_directory("/var/task".into()) + .environment(environment) + .user(user.into()) + .build() + .try_into() + .unwrap(); + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + metadata.selected_manifest.as_ref(), + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + ) + .unwrap(); + PinnedImage::fixture(metadata.try_into().unwrap()) +} +pub(in crate::docker) fn revision() -> ExecutionRevision { + ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("policy-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(Architecture::Arm64) + .artifact(DeploymentArtifact::Image(image_with( + "root", + vec!["BASE=value".into()], + ))) + .image_config(None) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ("TEST_VALUE".into(), "secret-value".into()), + ])) + .timeout(Duration::from_secs(3)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build() +} +fn plan(revision: ExecutionRevision) -> Result { + EnvironmentPlan::new( + directory(), + image_with("0", vec![]), + Arc::new(revision), + policy(), + ) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} + +#[test] +fn test_should_capture_fixed_images_identities_labels_and_complete_resource_policy() { + let directory = directory(); + let owner = directory.owner_id(); + let run = directory.run_id(); + let revision = revision(); + let resource = revision.resource_id; + let image = image(&revision) + .unwrap() + .resolved() + .metadata() + .image_id + .clone(); + let plan = EnvironmentPlan::new( + directory, + image_with("0", vec![]), + Arc::new(revision), + policy(), + ) + .unwrap(); + assert_ne!(plan.relay_key().as_str(), plan.function_key().as_str()); + assert_eq!(plan.memory_bytes(), 256 * MIB); + for (body, role, memory, cpu, pids, tmp) in [ + (&plan.bodies.relay, "relay", 64 * MIB, 100_000_000, 64, MIB), + ( + &plan.bodies.function, + "function", + 192 * MIB, + 1_000_000_000, + 256, + 512 * MIB, + ), + ] { + assert_eq!(body.image.as_deref(), Some(image.as_str())); + assert_eq!(body.tty, Some(false)); + assert_eq!(body.user.as_deref(), Some("1000:1000")); + assert_eq!( + body.healthcheck.as_ref().unwrap().test.as_ref().unwrap(), + &["NONE"] + ); + let labels = body.labels.as_ref().unwrap(); + assert_eq!(labels["rustack.lambda.ownerId"], owner.to_string()); + assert_eq!(labels["rustack.lambda.runId"], run.to_string()); + assert_eq!( + labels["rustack.lambda.environmentId"], + plan.environment_id().to_string() + ); + assert_eq!( + labels["rustack.lambda.functionResourceId"], + resource.to_string() + ); + assert_eq!(labels["rustack.lambda.role"], role); + let limits = body.host_config.as_ref().unwrap(); + assert_eq!( + limits.cgroupns_mode, + Some(HostConfigCgroupnsModeEnum::PRIVATE) + ); + assert_eq!(limits.memory, Some(i64::try_from(memory).unwrap())); + assert_eq!(limits.memory_swap, limits.memory); + assert_eq!(limits.oom_kill_disable, Some(false)); + assert_eq!(limits.nano_cpus, Some(cpu)); + assert_eq!(limits.pids_limit, Some(pids)); + assert_eq!(limits.readonly_rootfs, Some(true)); + assert_eq!(limits.cap_drop.as_ref().unwrap(), &["ALL"]); + assert_eq!( + limits.security_opt.as_ref().unwrap(), + &["no-new-privileges:true"] + ); + assert_eq!(limits.privileged, Some(false)); + assert_eq!(limits.auto_remove, Some(false)); + assert_eq!(limits.publish_all_ports, Some(false)); + assert_eq!( + limits.restart_policy.as_ref().unwrap().name, + Some(RestartPolicyNameEnum::NO) + ); + assert_eq!( + limits.log_config.as_ref().unwrap().typ.as_deref(), + Some("none") + ); + assert_eq!( + limits.tmpfs.as_ref().unwrap()["/tmp"], + format!("rw,nosuid,nodev,size={tmp},mode=1777") + ); + assert!( + limits.binds.is_none() && limits.devices.is_none() && limits.port_bindings.is_none() + ); + } + assert_eq!(plan.bodies.relay.attach_stdin, Some(true)); + assert_eq!(plan.bodies.relay.open_stdin, Some(true)); + assert_eq!(plan.bodies.relay.stdin_once, Some(true)); + assert_eq!(plan.bodies.function.attach_stdin, Some(false)); + assert_eq!( + plan.bodies + .function + .host_config + .as_ref() + .unwrap() + .network_mode + .as_deref(), + Some(format!("container:{}", plan.relay_key().as_str()).as_str()) + ); + assert!(!format!("{plan:?}").contains("secret-value")); +} + +#[test] +fn test_should_preserve_explicit_image_overrides_and_authoritative_environment_order() { + let mut captured = revision(); + captured.qualifier = "27".into(); + captured.artifact = DeploymentArtifact::Image(image_with( + "1234:4321", + vec![ + "TEST_VALUE=old".into(), + "AWS_LAMBDA_RUNTIME_API=outside:9001".into(), + "DELETED".into(), + ], + )); + captured.image_config = Some(ImageConfig { + entry_point: Some(vec![]), + command: Some(vec!["/actual".into(), "$(not-a-shell)".into()]), + working_directory: Some("/app".into()), + }); + let plan = plan(captured).unwrap(); + let body = &plan.bodies.function; + assert_eq!(body.user.as_deref(), Some("1234:4321")); + assert_eq!(body.entrypoint.as_ref().unwrap(), &[""]); + assert_eq!(body.cmd.as_ref().unwrap(), &["/actual", "$(not-a-shell)"]); + assert_eq!(body.working_dir.as_deref(), Some("/app")); + let env = body.env.as_ref().unwrap(); + assert!(env.iter().any(|entry| entry == "DELETED")); + let keys: HashSet<_> = env + .iter() + .map(|entry| entry.split('=').next().unwrap()) + .collect(); + assert_eq!(keys.len(), env.len()); + assert_eq!( + env.iter() + .find(|entry| entry.starts_with("AWS_LAMBDA_RUNTIME_API=")) + .unwrap(), + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001" + ); + assert!( + !env.iter() + .any(|entry| entry.contains("outside:9001") || entry == "TEST_VALUE=old") + ); + assert!(env.contains(&"TEST_VALUE=secret-value".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_VERSION=27".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_TIMEOUT=3".into())); + assert!(env.contains(&"LAMBDA_TASK_ROOT=/var/task".into())); + let relay = plan.bodies.relay.env.as_ref().unwrap(); + assert_eq!(relay.len(), 4); + assert!(!relay.iter().any(|entry| entry.contains("secret-value"))); +} + +#[rstest::rstest] +#[case::absent("", "1000:1000")] +#[case::root("root", "1000:1000")] +#[case::numeric_root("0:4321", "1000:1000")] +#[case::padded_root("0000", "1000:1000")] +#[case::numeric("1234", "1234")] +#[case::named("nobody:nogroup", "nobody:nogroup")] +#[case::named_numeric_group("app_user:0", "app_user:0")] +fn test_should_select_nonroot_container_users(#[case] user: &str, #[case] expected: &str) { + assert_eq!(image_user(user, "1000:1000").unwrap(), expected); +} +#[rstest::rstest] +#[case(":1000")] +#[case("1000:")] +#[case("root:group:extra")] +#[case("+0")] +#[case("4294967295")] +#[case("account name")] +#[case("/root")] +fn test_should_reject_malformed_image_users(#[case] user: &str) { + assert!(image_user(user, "1000:1000").is_err()); +} + +#[rstest::rstest] +#[case("host")] +#[case("bridge")] +#[case("default")] +#[case("container:other")] +#[case("")] +#[case("bad/network")] +fn test_should_reject_unsafe_network_selections(#[case] value: &str) { + assert!(value.parse::().is_err()); +} +#[test] +fn test_should_accept_disconnected_and_named_operator_networks_without_claiming_probe_success() { + for name in ["none", "rustack-test", "ab0123"] { + assert!(name.parse::().is_ok()); + } + for (uid, gid, cpu) in [ + (0, 1000, 1_000_000_000), + (1000, 0, 1_000_000_000), + (u32::MAX, 1000, 1_000_000_000), + (1000, 1000, 0), + (1000, 1000, 99_999_999), + (1000, 1000, 16_000_000_001), + ] { + assert!( + ContainerPolicy::new( + "none".parse().unwrap(), + uid, + gid, + cpu, + RelayBudgets::default() + ) + .is_err() + ); + } + for cpu in [100_000_000, 16_000_000_000] { + assert!( + ContainerPolicy::new( + "none".parse().unwrap(), + 1, + 2_147_483_647, + cpu, + RelayBudgets::default() + ) + .is_ok() + ); + } + for durations in [ + ( + Duration::ZERO, + Duration::from_secs(600), + Duration::from_secs(10), + ), + ( + Duration::from_secs(61), + Duration::from_secs(600), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_secs(86401), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_millis(1500), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_secs(600), + Duration::from_secs(61), + ), + ] { + assert!(RelayBudgets::new(durations.0, durations.1, durations.2).is_err()); + } +} + +#[rstest::rstest] +#[case("backend")] +#[case("zip")] +#[case("runtime")] +#[case("handler")] +#[case("nil_revision")] +#[case("revision_case")] +#[case("name")] +#[case("name_length")] +#[case("alias")] +#[case("zero_version")] +#[case("padded_version")] +#[case("overflow_version")] +#[case("timeout_zero")] +#[case("timeout_excess")] +#[case("timeout_fraction")] +#[case("memory_low")] +#[case("memory_high")] +#[case("tmp_low")] +#[case("tmp_high")] +#[case("architecture")] +#[case("region_missing")] +#[case("region_mismatch")] +#[case("region_invalid")] +#[case("reserved_env")] +#[case("environment_bytes")] +#[case("environment_nul")] +#[case("empty_argv")] +fn test_should_reject_invalid_captured_revisions_before_any_daemon_operation( + #[case] scenario: &str, +) { + let mut revision = revision(); + match scenario { + "backend" => revision.backend = ExecutorBackend::Native, + "zip" => revision.artifact = DeploymentArtifact::Zip(ValidatedArtifact::fixture()), + "runtime" => revision.runtime = Some("provided.al2023".into()), + "handler" => revision.handler = Some("bootstrap".into()), + "nil_revision" => revision.revision_id = Uuid::nil().to_string(), + "revision_case" => revision.revision_id = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA".into(), + "name" => revision.function_name = "bad/name".into(), + "name_length" => revision.function_name = "x".repeat(65), + "alias" => revision.qualifier = "prod".into(), + "zero_version" => revision.qualifier = "0".into(), + "padded_version" => revision.qualifier = "01".into(), + "overflow_version" => revision.qualifier = "18446744073709551616".into(), + "timeout_zero" => revision.timeout = Duration::ZERO, + "timeout_excess" => revision.timeout = Duration::from_secs(901), + "timeout_fraction" => revision.timeout = Duration::from_millis(1500), + "memory_low" => revision.memory_mb = 127, + "memory_high" => revision.memory_mb = 10241, + "tmp_low" => revision.ephemeral_storage_mb = 511, + "tmp_high" => revision.ephemeral_storage_mb = 10241, + "architecture" => revision.architecture = Architecture::X86_64, + "region_missing" => { + revision.environment.remove("AWS_REGION"); + } + "region_mismatch" => { + revision + .environment + .insert("AWS_DEFAULT_REGION".into(), "different".into()); + } + "region_invalid" => { + revision + .environment + .insert("AWS_REGION".into(), "outside\nregion".into()); + } + "reserved_env" => { + revision + .environment + .insert("AWS_LAMBDA_RUNTIME_API".into(), "outside:9001".into()); + } + "environment_bytes" => { + revision + .environment + .insert("TOO_LARGE".into(), "x".repeat(4096)); + } + "environment_nul" => { + revision.environment.insert("BAD_VALUE".into(), "\0".into()); + } + "empty_argv" => { + revision.image_config = Some(ImageConfig { + entry_point: Some(vec![]), + command: Some(vec![]), + working_directory: None, + }); + } + _ => panic!("unknown fixture scenario"), + } + assert!(plan(revision).is_err(), "{scenario}"); +} + +#[tokio::test] +async fn test_should_retain_the_real_directory_until_allocation_plan_is_released() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let plan = EnvironmentPlan::new( + directory, + image_with("", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(super::super::DirectoryLeaseError::InUse) + )); + drop(plan); + assert!( + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .is_ok() + ); +} + +#[tokio::test] +async fn test_should_submit_exact_captured_platform_and_refuse_revoked_availability_before_io() { + let mut daemon = super::super::tests::Daemon::new(); + let mut wire = daemon.wire().await; + let plan = Arc::new(plan(revision()).unwrap()); + let worker = { + let plan = plan.clone(); + let port = wire.port(); + tokio::spawn(async move { plan.create_relay(&port, deadline()).await }) + }; + let request = daemon.next().await; + assert!(request.head.contains("platform=linux%2Farm64")); + request + .reply(201, b"{\"Id\":\"created\",\"Warnings\":[]}") + .await; + assert_eq!(worker.await.unwrap().unwrap().id, "created"); + let (release, _) = watch::channel(()); + let (stop, stopping) = watch::channel(false); + let revoked = PinnedImage::new( + image_with("", vec![]).resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopping, + ); + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(revoked); + let plan = EnvironmentPlan::new( + directory(), + image_with("", vec![]), + Arc::new(revision), + policy(), + ) + .unwrap(); + stop.send_replace(true); + assert!( + plan.create_function(&wire.port(), deadline()) + .await + .is_err() + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_normalize_inherited_duplicates_empty_values_and_deletion_markers_once() { + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(image_with( + "1000", + vec![ + "INHERITED=first".into(), + "INHERITED=last".into(), + "REMOVED=old".into(), + "REMOVED".into(), + "EMPTY=".into(), + "TEST_VALUE=old".into(), + "TEST_VALUE".into(), + ], + )); + let plan = plan(revision).unwrap(); + let env = plan.bodies.function.env.as_ref().unwrap(); + for (key, expected) in [ + ("INHERITED", "INHERITED=last"), + ("REMOVED", "REMOVED"), + ("EMPTY", "EMPTY="), + ("TEST_VALUE", "TEST_VALUE=secret-value"), + ] { + let matching: Vec<_> = env + .iter() + .filter(|entry| entry.split('=').next() == Some(key)) + .collect(); + assert_eq!(matching, vec![expected]); + } +} + +#[tokio::test] +async fn test_should_retain_both_live_image_leases_through_allocation_plan_lifetime() { + let image = image_with("", vec![]); + let (relay_release, mut relay_released) = watch::channel(()); + let (function_release, mut function_released) = watch::channel(()); + let (_, stopped) = watch::channel(false); + let make = |release| { + PinnedImage::new( + image.resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped.clone(), + ) + }; + let relay = make(relay_release); + let function = make(function_release); + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(function.clone()); + let plan = + EnvironmentPlan::new(directory(), relay.clone(), Arc::new(revision), policy()).unwrap(); + drop(relay); + drop(function); + assert!(!relay_released.has_changed().unwrap()); + assert!(!function_released.has_changed().unwrap()); + drop(plan); + let _ = timeout(Duration::from_secs(1), relay_released.changed()) + .await + .unwrap(); + let _ = timeout(Duration::from_secs(1), function_released.changed()) + .await + .unwrap(); + assert!(relay_released.has_changed().is_err()); + assert!(function_released.has_changed().is_err()); +} + +pub(in crate::docker) fn function_body(plan: &EnvironmentPlan) -> &ContainerCreateBody { + &plan.bodies.function +} + +pub(in crate::docker) fn relay_body(plan: &EnvironmentPlan) -> &ContainerCreateBody { + &plan.bodies.relay +} diff --git a/crates/rustack-lambda-core/src/docker/pool/actor.rs b/crates/rustack-lambda-core/src/docker/pool/actor.rs new file mode 100644 index 00000000..048c6428 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/actor.rs @@ -0,0 +1,539 @@ +//! Pool registry, FIFO dispatch and retained emergency retirement. + +use std::{ + collections::{HashMap, VecDeque}, + mem, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use rustack_lambda_runtime::session::RuntimeResult; +use tokio::{ + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{Id, JoinSet}, + time::{Instant, interval, sleep, sleep_until}, +}; +use typed_builder::TypedBuilder; + +use super::{Call, Offer, PoolConfig, PoolStatus}; +use crate::{ + docker::{ + EnvironmentError, EnvironmentHandle, EnvironmentPhase, EnvironmentPort, + connection::stopped, environment::Retirement, + }, + executor::ExecutorError, + invocation::CleanupTask, +}; + +pub(super) struct Warm { + key: String, + environment: EnvironmentHandle, +} +pub(super) struct Completion { + warm: Option, + failed: bool, +} +pub(super) struct Record { + retirement: Retirement, + settlement: Option>>>, + failed: bool, +} +impl Record { + fn maintenance(retirement: Retirement) -> Self { + Self { + retirement, + settlement: None, + failed: false, + } + } + fn publish(self, failed: bool) { + if let Some(settlement) = self.settlement { + settlement.send_replace(Some(if self.failed || failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + } +} +#[derive(TypedBuilder)] +pub(super) struct Actor { + environments: EnvironmentPort, + config: PoolConfig, + inbox: mpsc::Receiver, + stop: watch::Sender, + failed: Arc, + status: watch::Sender, + emergency: Arc, + runtime: Handle, + #[builder(default)] + pending: VecDeque, + #[builder(default)] + warm: VecDeque, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + receipts: HashMap, + #[builder(default)] + settled: bool, + #[builder(default)] + eviction: Option, +} +impl Actor { + pub(super) async fn run(mut self) -> Result<(), EnvironmentError> { + let mut stop = self.stop.subscribe(); + let mut tick = interval(Duration::from_millis(100)); + let mut stopping = false; + loop { + if *stop.borrow() || self.inbox.is_closed() { + stopping = true; + } + if stopping { + self.inbox.close(); + while let Ok(call) = self.inbox.try_recv() { + call.reject(self.closed_error()); + } + while let Some(call) = self.pending.pop_front() { + call.reject(self.closed_error()); + } + while self.jobs.len() < 32 { + let Some(warm) = self.warm.pop_front() else { + break; + }; + self.retire(warm.environment); + } + } else { + self.expire(); + self.dispatch(); + } + self.status.send_replace(PoolStatus { + active: self.jobs.len(), + warm: self.warm.len(), + ..PoolStatus::default() + }); + if stopping && self.jobs.is_empty() && self.warm.is_empty() { + self.settled = true; + return if self.failed.load(Ordering::Acquire) { + Err(EnvironmentError::Worker) + } else { + Ok(()) + }; + } + tokio::select! { biased; + () = stopped(&mut stop), if !stopping => { stopping = true; }, + completion = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => { if let Some(completion) = completion { self.completed(completion); } }, + call = self.inbox.recv(), if !stopping => { match call { Some(call) => self.pending.push_back(call), None => stopping = true } }, + _ = tick.tick() => {}, + } + } + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + self.stop.send_replace(true); + } + fn closed_error(&self) -> EnvironmentError { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + } + fn expire(&mut self) { + let mut index = 0; + while index < self.pending.len() { + let error = self.pending.get(index).and_then(Call::error); + if let Some(error) = error { + if let Some(call) = self.pending.remove(index) { + call.reject(error); + } + } else { + index += 1; + } + } + let mut index = 0; + while index < self.warm.len() && self.jobs.len() < 32 { + if self + .warm + .get(index) + .is_some_and(|warm| warm.environment.status().phase != EnvironmentPhase::Idle) + { + if let Some(warm) = self.warm.remove(index) { + self.retire(warm.environment); + } + } else { + index += 1; + } + } + } + fn dispatch(&mut self) { + if self.eviction.is_some() { + return; + } + while self.jobs.len() < 32 { + let Some(mut call) = self.pending.pop_front() else { + return; + }; + if let Some(error) = call.error() { + call.reject(error); + continue; + } + if let Err(error) = call.plan.live() { + call.reject(error.into()); + continue; + } + let key = call.plan.warm_key().to_owned(); + let warm = self + .warm + .iter() + .position(|warm| warm.key == key) + .and_then(|index| self.warm.remove(index)); + let environment = match warm { + Some(warm) => Ok(warm.environment), + None => self + .environments + .spawn_shared(call.plan.clone(), call.deadline()), + }; + match environment { + Ok(environment) => { + call.queue.take(); + let record = Record { + retirement: environment.retirement(), + settlement: Some(call.settlement.clone()), + failed: false, + }; + let stop = self.stop.subscribe(); + let job = self.jobs.spawn(invoke(environment, key, call, stop)); + self.receipts.insert(job.id(), record); + } + Err(EnvironmentError::Busy) => { + self.pending.push_front(call); + if let Some(warm) = self.warm.pop_front() { + self.eviction = Some(self.retire(warm.environment)); + } + return; + } + Err(error) => { + call.reject(error); + } + } + } + } + fn retire(&mut self, environment: EnvironmentHandle) -> Id { + let receipt = environment.retirement(); + receipt.cancel(); + drop(environment); + self.retire_receipt(receipt) + } + fn retire_receipt(&mut self, receipt: Retirement) -> Id { + self.retire_record(Record::maintenance(receipt)) + } + fn retire_record(&mut self, record: Record) -> Id { + record.retirement.cancel(); + let job = self.jobs.spawn(retire(record.retirement.clone())); + self.receipts.insert(job.id(), record); + job.id() + } + fn completed(&mut self, result: Result<(Id, Completion), tokio::task::JoinError>) { + let id = match &result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let receipt = self.receipts.remove(&id); + if self.eviction == Some(id) { + self.eviction = None; + } + if let Ok((_, completion)) = result { + if completion.failed { + self.fail(); + } + if let Some(warm) = completion.warm { + let count = self.warm.iter().filter(|idle| idle.key == warm.key).count(); + if !*self.stop.borrow() + && warm.environment.status().phase == EnvironmentPhase::Idle + && self.warm.len() < usize::from(self.config.0.warm) + && count < usize::from(self.config.0.warm_per_revision) + { + self.warm.push_back(warm); + } else { + self.retire(warm.environment); + } + } + if let Some(record) = receipt { + record.publish(completion.failed); + } + } else { + self.fail(); + if let Some(mut record) = receipt { + record.failed = true; + self.retire_record(record); + } + } + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + while let Ok(call) = self.inbox.try_recv() { + call.reject(self.closed_error()); + } + for call in self.pending.drain(..) { + call.reject(EnvironmentError::Worker); + } + if self.receipts.is_empty() && self.warm.is_empty() { + return; + } + let mut jobs = mem::take(&mut self.jobs); + let mut receipts: Vec<_> = mem::take(&mut self.receipts).into_values().collect(); + for warm in self.warm.drain(..) { + receipts.push(Record::maintenance(warm.environment.retirement())); + drop(warm); + } + for record in &mut receipts { + record.failed = true; + record.retirement.cancel(); + } + jobs.abort_all(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover(receipts).await + }); + } +} +async fn recover(receipts: Vec) -> Result<(), ExecutorError> { + let mut cleanup = JoinSet::new(); + let mut index = HashMap::new(); + let mut failed = false; + for record in receipts { + let job = cleanup.spawn(retire(record.retirement.clone())); + index.insert(job.id(), record); + } + while let Some(result) = cleanup.join_next_with_id().await { + let id = match &result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(mut record) = index.remove(&id) else { + continue; + }; + if let Ok((_, completion)) = result { + failed |= completion.failed; + if record.retirement.removed() { + record.publish(completion.failed); + continue; + } + } else { + failed = true; + record.failed = true; + } + let job = cleanup.spawn(retire(record.retirement.clone())); + index.insert(job.id(), record); + } + if failed { + Err(ExecutorError::CleanupFailed( + "Pool recovery supervision failed".into(), + )) + } else { + Ok(()) + } +} +async fn retire(receipt: Retirement) -> Completion { + let mut failed = false; + loop { + let result = receipt.clone().settle().await; + failed |= result.is_err(); + if receipt.removed() { + return Completion { warm: None, failed }; + } + sleep(Duration::from_secs(1)).await; + } +} +async fn invoke( + mut environment: EnvironmentHandle, + key: String, + call: Call, + mut stop: watch::Receiver, +) -> Completion { + let deadline = call.deadline(); + let mut cancellation = call.cancellation.clone(); + let operation = async { + if let Some(error) = call.error() { + return Err(error); + } + environment + .wait_ready(deadline.min(Instant::now() + Duration::from_secs(60))) + .await?; + environment + .invoke(call.invocation.clone(), call.capture_logs) + .await + }; + let result = tokio::select! { biased; + () = stopped(&mut stop) => Err(EnvironmentError::Stopped), + () = stopped(&mut cancellation) => Err(EnvironmentError::Cancelled), + () = sleep_until(deadline) => Err(EnvironmentError::Deadline), + result = operation => result, + }; + if matches!(&result, Ok(response) if matches!(response.result, RuntimeResult::Success(_))) + && call.error().is_none() + && !*stop.borrow() + { + let (receipt, consumed) = oneshot::channel(); + let sent = call + .reply + .send(Offer { + result, + receipt: Some(receipt), + }) + .is_ok(); + if sent { + let accepted = tokio::select! { biased; + () = stopped(&mut stop) => false, + () = sleep_until(deadline) => false, + result = consumed => result.is_ok() && Instant::now() < deadline, + () = stopped(&mut cancellation) => false, + }; + if accepted { + return Completion { + warm: Some(Warm { key, environment }), + failed: false, + }; + } + } + let receipt = environment.retirement(); + drop(environment); + retire(receipt).await + } else { + let receipt = environment.retirement(); + drop(environment); + let completion = retire(receipt).await; + let _ = call.reply.send(Offer { + result, + receipt: None, + }); + completion + } +} + +#[cfg(test)] +pub(in crate::docker) fn testing_start( + environments: EnvironmentPort, + config: PoolConfig, + warm: Vec<(String, EnvironmentHandle)>, +) -> super::EnvironmentPool { + testing_start_inner(environments, config, warm, false, None) +} +#[cfg(test)] +pub(super) fn testing_failed_worker_start( + environments: EnvironmentPort, + warm: Vec<(String, EnvironmentHandle)>, +) -> super::EnvironmentPool { + testing_start_inner(environments, PoolConfig::default(), warm, true, None) +} +#[cfg(test)] +pub(super) fn testing_failed_call_start( + environments: EnvironmentPort, + plan: crate::docker::EnvironmentPlan, + invocation: rustack_lambda_runtime::session::Invocation, + environment: EnvironmentHandle, +) -> (super::EnvironmentPool, super::PoolInvocation) { + let deadline = invocation.parts().1; + let (stop, cancellation) = watch::channel(false); + let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); + let key = plan.warm_key().to_owned(); + let call = Call { + plan: Arc::new(plan), + invocation, + capture_logs: false, + cancellation, + reply, + queue: None, + settlement, + }; + let pool = testing_start_inner( + environments, + PoolConfig::default(), + vec![(key, environment)], + true, + Some(call), + ); + let invocation = super::PoolInvocation { + deadline, + stop, + response: Some(response), + settlement: observed, + failed: pool.port.failed.clone(), + consumed: false, + }; + (pool, invocation) +} +#[cfg(test)] +fn testing_start_inner( + environments: EnvironmentPort, + config: PoolConfig, + warm: Vec<(String, EnvironmentHandle)>, + failed_worker: bool, + failed_call: Option, +) -> super::EnvironmentPool { + let runtime = Handle::current(); + let maximum = usize::from(config.0.queue); + let (sender, inbox) = mpsc::channel(maximum); + let (stop, _) = watch::channel(false); + let failed = Arc::new(AtomicBool::new(false)); + let (status, observed) = watch::channel(PoolStatus::default()); + let emergency = Arc::new(CleanupTask::default()); + let mut actor = Actor::builder() + .environments(environments) + .config(config) + .inbox(inbox) + .stop(stop.clone()) + .failed(failed.clone()) + .status(status) + .emergency(emergency.clone()) + .runtime(runtime.clone()) + .warm( + warm.into_iter() + .map(|(key, environment)| Warm { key, environment }) + .collect(), + ) + .build(); + if failed_worker { + let warm = actor.warm.pop_front().unwrap(); + let receipt = warm.environment.retirement(); + let settlement = failed_call.as_ref().map(|call| call.settlement.clone()); + let job = actor.jobs.spawn(async move { + let _owned = warm; + let _call = failed_call; + std::future::pending::().await + }); + actor.receipts.insert( + job.id(), + Record { + retirement: receipt, + settlement, + failed: false, + }, + ); + job.abort(); + } + let port = super::PoolPort { + sender, + queue: Arc::new(tokio::sync::Semaphore::new(maximum)), + maximum, + stop, + failed, + status: observed, + }; + super::EnvironmentPool { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + } +} diff --git a/crates/rustack-lambda-core/src/docker/pool/mod.rs b/crates/rustack-lambda-core/src/docker/pool/mod.rs new file mode 100644 index 00000000..9b9adc42 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/mod.rs @@ -0,0 +1,469 @@ +//! Bounded warm dispatch; external resource ownership stays with `EnvironmentManager`. + +mod actor; +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +#[cfg(test)] +pub(super) use actor::testing_start; +use rustack_lambda_runtime::session::Invocation; +use tokio::{ + runtime::Handle, + sync::{Semaphore, mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + DockerError, EnvironmentError, EnvironmentPlan, EnvironmentPort, EnvironmentResponse, + check_deadline, +}; +use crate::invocation::CleanupTask; + +/// Operator limits for waiting calls and retained idle environments. +#[derive(Debug, Clone, Copy, TypedBuilder)] +#[non_exhaustive] +pub struct PoolSettings { + /// Globally retained idle environments,0..32. + #[builder(default = 32)] + pub warm: u8, + /// Idle environments per immutable identity,0..32. + #[builder(default = 1)] + pub warm_per_revision: u8, + /// Accepted waiting calls across mailbox and actor queue,1..256. + #[builder(default = 256)] + pub queue: u16, +} +impl Default for PoolSettings { + fn default() -> Self { + Self::builder().build() + } +} +/// Validated warm retention and admission limits. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct PoolConfig(PoolSettings); +impl TryFrom for PoolConfig { + type Error = EnvironmentError; + fn try_from(settings: PoolSettings) -> Result { + if settings.warm > 32 + || settings.warm_per_revision > 32 + || !(1..=256).contains(&settings.queue) + { + return Err(EnvironmentError::Configuration); + } + Ok(Self(settings)) + } +} +/// Read-only pool projection; environment accounting includes quarantine separately. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct PoolStatus { + /// Accepted calls still waiting for dispatch. + pub queued: usize, + /// Invocation/retirement jobs; at most32. + pub active: usize, + /// Idle handles retained for reuse. + pub warm: usize, + /// Whether admission remains open and healthy. + pub accepting: bool, + /// A worker/root supervision failure closed admission. + pub worker_failed: bool, +} +struct Offer { + result: Result, + receipt: Option>, +} +struct Call { + plan: Arc, + invocation: Invocation, + capture_logs: bool, + cancellation: watch::Receiver, + reply: oneshot::Sender, + queue: Option, + settlement: watch::Sender>>, +} +impl Call { + fn deadline(&self) -> Instant { + self.invocation.parts().1 + } + fn error(&self) -> Option { + if Instant::now() >= self.deadline() { + Some(EnvironmentError::Deadline) + } else if self.reply.is_closed() + || *self.cancellation.borrow() + || self.cancellation.has_changed().is_err() + { + Some(EnvironmentError::Cancelled) + } else { + None + } + } + fn reject(self, error: EnvironmentError) { + // No environment has been allocated for a rejected queued call. + self.settlement.send_replace(Some(Ok(()))); + let _ = self.reply.send(Offer { + result: Err(error), + receipt: None, + }); + } +} +/// Exclusive response and retained cleanup receipt for one accepted pool call. +/// +/// Drop requests cancellation; only `cancel_and_wait` proves settlement. +#[non_exhaustive] +pub struct PoolInvocation { + deadline: Instant, + stop: watch::Sender, + response: Option>, + settlement: watch::Receiver>>, + failed: Arc, + consumed: bool, +} +impl fmt::Debug for PoolInvocation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PoolInvocation") + .field("deadline", &self.deadline) + .field("consumed", &self.consumed) + .field("settled", &self.settlement.borrow().is_some()) + .finish_non_exhaustive() + } +} +impl PoolInvocation { + /// Consume one result while retaining the receiver across cancelled waiters. + /// + /// # Errors + /// Returns invocation failure, deadline, or unavailable supervision. A closed + /// response lane is not cleanup proof; use `cancel_and_wait` before releasing + /// higher-level work permits after cancellation or uncertain failure. + /// + /// ```no_run + /// # async fn result(call: &mut rustack_lambda_core::docker::PoolInvocation) { + /// let result = call.wait().await; + /// # } + /// ``` + pub async fn wait(&mut self) -> Result { + if self.consumed || self.response.is_none() { + return Err(EnvironmentError::Configuration); + } + let response = self + .response + .as_mut() + .ok_or(EnvironmentError::Configuration)?; + let received = timeout_at(self.deadline, response) + .await + .map_err(|_| EnvironmentError::Deadline)?; + self.response.take(); + let offer = received.map_err(|_| { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + })?; + // Remove a completed receiver even when the deadline check rejects its + // offer. Dropping that offer revokes any unconsumed warm-return receipt. + if Instant::now() >= self.deadline { + return Err(EnvironmentError::Deadline); + } + if let Some(receipt) = offer.receipt { + receipt.send(()).map_err(|()| EnvironmentError::Worker)?; + } + self.consumed = true; + offer.result + } + + /// Request cancellation and await independently owned physical settlement. + /// + /// A cancelled/expired waiter can resume with a fresh cleanup deadline. + /// Consumed success belongs to the pool and cannot be cancelled by this call. + /// + /// # Errors + /// Returns Deadline while cleanup remains retained, or Worker when recovery + /// was uncertain or the settlement producer disappeared without proof. + /// Cleanup deadlines over ten minutes return Configuration after signalling. + /// + /// ```no_run + /// # async fn stop(call: &mut rustack_lambda_core::docker::PoolInvocation) { + /// let settled = call.cancel_and_wait(tokio::time::Instant::now() + /// + std::time::Duration::from_secs(60)).await; + /// # } + /// ``` + pub async fn cancel_and_wait(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + if !self.consumed { + self.stop.send_replace(true); + self.response.take(); + } + Self::check_cleanup_deadline(deadline)?; + loop { + let result = self.settlement.borrow().clone(); + if let Some(result) = result { + return result; + } + timeout_at(deadline, self.settlement.changed()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + Self::check_cleanup_deadline(deadline)?; + } + } + fn check_cleanup_deadline(deadline: Instant) -> Result<(), EnvironmentError> { + check_deadline(deadline).map_err(|error| match error { + DockerError::Deadline => EnvironmentError::Deadline, + DockerError::Configuration => EnvironmentError::Configuration, + error => error.into(), + }) + } +} +impl Drop for PoolInvocation { + fn drop(&mut self) { + if !self.consumed { + self.stop.send_replace(true); + } + } +} +/// Cloneable call port; the retained pool owner must outlive its shutdown. +#[derive(Clone)] +#[non_exhaustive] +pub struct PoolPort { + sender: mpsc::Sender, + queue: Arc, + maximum: usize, + stop: watch::Sender, + failed: Arc, + status: watch::Receiver, +} +impl fmt::Debug for PoolPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PoolPort") + .field("status", &self.status()) + .finish() + } +} +impl PoolPort { + /// Execute once through a matching warm environment or bounded cold allocation. + /// + /// # Errors + /// Returns queue overload, deadline, cancellation, initialization or runtime + /// failure. Dropping the future signals retained cleanup without replay. + /// + /// ```no_run + /// # async fn invoke(port: rustack_lambda_core::docker::PoolPort, plan: rustack_lambda_core::docker::EnvironmentPlan, call: rustack_lambda_runtime::session::Invocation) { + /// let result = port.invoke(plan, call, false).await; + /// # } + /// ``` + pub async fn invoke( + &self, + plan: EnvironmentPlan, + invocation: Invocation, + capture_logs: bool, + ) -> Result { + self.start(plan, invocation, capture_logs)?.wait().await + } + /// Admit once and retain separate response/cancellation-settlement ownership. + /// + /// # Errors + /// Rejects expired calls, unavailable image pins, overload or closed admission + /// before enqueueing. Scope validation also occurs before physical allocation. + /// + /// ```no_run + /// # fn start(port: &rustack_lambda_core::docker::PoolPort, plan: rustack_lambda_core::docker::EnvironmentPlan, event: rustack_lambda_runtime::session::Invocation) { + /// let call = port.start(plan, event, false); + /// # } + /// ``` + pub fn start( + &self, + plan: EnvironmentPlan, + invocation: Invocation, + capture_logs: bool, + ) -> Result { + let deadline = invocation.parts().1; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if !self.status().accepting { + return Err(self.closed_error()); + } + plan.live()?; + let queue = self + .queue + .clone() + .try_acquire_owned() + .map_err(|_| EnvironmentError::Busy)?; + let (stop, cancellation) = watch::channel(false); + let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); + self.sender + .try_send(Call { + plan: Arc::new(plan), + invocation, + capture_logs, + cancellation, + reply, + queue: Some(queue), + settlement, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => self.closed_error(), + })?; + Ok(PoolInvocation { + deadline, + stop, + response: Some(response), + settlement: observed, + failed: self.failed.clone(), + consumed: false, + }) + } + fn closed_error(&self) -> EnvironmentError { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + } + /// Read pool counts without waiting for the business mailbox. + /// + /// ```no_run + /// # fn status(port: &rustack_lambda_core::docker::PoolPort) { let counts = port.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> PoolStatus { + let mut status = *self.status.borrow(); + status.queued = self.maximum.saturating_sub(self.queue.available_permits()); + status.worker_failed = self.failed.load(Ordering::Acquire); + status.accepting = !*self.stop.borrow() && !status.worker_failed; + status + } +} +/// Retained pool root, shut down before the environment and image owners. +#[non_exhaustive] +pub struct EnvironmentPool { + port: PoolPort, + task: Option>>, + primary: Option>, + settled: Option>, + emergency: Arc, +} +impl fmt::Debug for EnvironmentPool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPool") + .field("port", &self.port) + .field("settled", &self.settled) + .finish_non_exhaustive() + } +} +impl EnvironmentPool { + /// Start bounded checkout over an already-owned environment service. + /// + /// # Errors + /// Returns Worker when no Tokio runtime is available. This does not establish + /// Docker startup readiness or enable the production executor. + /// + /// ```no_run + /// # fn start(environments: rustack_lambda_core::docker::EnvironmentPort) { + /// let pool = rustack_lambda_core::docker::EnvironmentPool::start(environments, Default::default()); + /// # } + /// ``` + pub fn start( + environments: EnvironmentPort, + config: PoolConfig, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| EnvironmentError::Worker)?; + let maximum = usize::from(config.0.queue); + let (sender, inbox) = mpsc::channel(maximum); + let (stop, _) = watch::channel(false); + let failed = Arc::new(AtomicBool::new(false)); + let (status, observed) = watch::channel(PoolStatus::default()); + let emergency = Arc::new(CleanupTask::default()); + let actor = actor::Actor::builder() + .environments(environments) + .config(config) + .inbox(inbox) + .stop(stop.clone()) + .failed(failed.clone()) + .status(status) + .emergency(emergency.clone()) + .runtime(runtime.clone()) + .build(); + let port = PoolPort { + sender, + queue: Arc::new(Semaphore::new(maximum)), + maximum, + stop, + failed, + status: observed, + }; + Ok(Self { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + }) + } + /// Clone the call port without transferring root shutdown ownership. + /// + /// ```no_run + /// # fn port(pool: &rustack_lambda_core::docker::EnvironmentPool) { let port = pool.port(); } + /// ``` + #[must_use] + pub fn port(&self) -> PoolPort { + self.port.clone() + } + /// Close admission, retire active/idle environments and join retained cleanup. + /// + /// # Errors + /// Returns timeout while cleanup stays owned, or a cached supervision failure. + /// A cancelled waiter does not lose the join or emergency cleanup receipt. + /// + /// ```no_run + /// # async fn stop(pool: &mut rustack_lambda_core::docker::EnvironmentPool) { + /// let clean = pool.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(60)).await; + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.port.stop.send_replace(true); + check_deadline(deadline)?; + if let Some(settled) = &self.settled { + return settled.clone(); + } + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker) + .and_then(|result| result), + None => Err(EnvironmentError::Worker), + }; + self.task.take(); + self.primary = Some(result); + } + timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + let result = self.primary.clone().ok_or(EnvironmentError::Worker)?; + self.settled = Some(result.clone()); + result + } +} +impl Drop for EnvironmentPool { + fn drop(&mut self) { + self.port.stop.send_replace(true); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/pool/tests.rs b/crates/rustack-lambda-core/src/docker/pool/tests.rs new file mode 100644 index 00000000..63ee0c00 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/tests.rs @@ -0,0 +1,731 @@ +use std::{sync::atomic::AtomicUsize, time::Duration}; + +use super::{actor::testing_start, *}; +use crate::docker::{ + DirectoryLease, EnvironmentConfig, EnvironmentManager, EnvironmentSettings, RelayBudgets, + environment::test_peer::pair, + plan::tests::{image_with, policy, revision}, + tests::Daemon, +}; + +mod settlement; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn directory() -> DirectoryLease { + DirectoryLease::testing(uuid::Uuid::new_v4(), uuid::Uuid::new_v4()) +} +fn plan(directory: &DirectoryLease) -> EnvironmentPlan { + EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap() +} +fn invocation(until: Instant) -> Invocation { + Invocation::new( + rustack_lambda_runtime::protocol::AttemptId::default(), + until, + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture" + .parse() + .unwrap(), + rustack_lambda_runtime::metadata::local_trace_id().unwrap(), + rustack_lambda_runtime::protocol::Payload::try_from(bytes::Bytes::from_static(b"{}")) + .unwrap(), + ) + .unwrap() +} +#[test] +fn test_should_reject_invalid_pool_limits_and_allow_disabled_warm_retention() { + for settings in [ + PoolSettings::builder().warm(33).build(), + PoolSettings::builder().warm_per_revision(33).build(), + PoolSettings::builder().queue(0).build(), + PoolSettings::builder().queue(257).build(), + ] { + assert!(PoolConfig::try_from(settings).is_err()); + } + assert!( + PoolConfig::try_from(PoolSettings::builder().warm(0).warm_per_revision(0).build()).is_ok() + ); +} +#[tokio::test] +async fn test_should_reject_queued_call_after_pool_stop_without_allocation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let port = pool.port(); + pool.shutdown(deadline()).await.unwrap(); + assert!(matches!( + port.invoke(plan(&lease), invocation(deadline()), false) + .await, + Err(EnvironmentError::Stopped) + )); + assert_eq!(environments.port().status().allocated, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_cache_root_failure_and_release_queued_permits_without_mutation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + pool.task.as_ref().unwrap().abort(); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(pool.port().status().worker_failed); + assert_eq!(environments.port().status().allocated, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_key_allocation_values_without_fresh_identities_or_hashmap_order() { + let lease = directory(); + let original = revision(); + let make = |revision, policy| { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + Arc::new(revision), + policy, + ) + .unwrap() + }; + let first = make(original.clone(), policy()); + let mut reordered = original.clone(); + let mut entries: Vec<_> = original.environment.iter().collect(); + entries.sort(); + reordered.environment = entries + .into_iter() + .rev() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let second = make(reordered, policy()); + assert_ne!(first.environment_id(), second.environment_id()); + assert_eq!(first.warm_key(), second.warm_key()); + let mut changed = original.clone(); + changed + .environment + .insert("TEST_VALUE".into(), "another-secret".into()); + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let mut changed = original.clone(); + changed.memory_mb = 256; + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let mut changed = original.clone(); + changed.revision_id = uuid::Uuid::new_v4().to_string(); + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let another_policy = crate::docker::ContainerPolicy::new( + "none".parse().unwrap(), + 1001, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap(); + assert_ne!( + first.warm_key(), + make(original.clone(), another_policy).warm_key() + ); + assert_ne!( + first.warm_key(), + EnvironmentPlan::new( + directory(), + image_with("1000", vec![]), + Arc::new(original), + policy() + ) + .unwrap() + .warm_key() + ); + assert!(!format!("{first:?}").contains("secret-value")); +} +async fn idle(port: &PoolPort) { + tokio::time::timeout(Duration::from_secs(3), async { + while port.status().active != 0 || port.status().warm != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_reuse_only_consumed_warm_results_and_join_idle_shutdown() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let count = Arc::new(AtomicUsize::new(0)); + let served = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(make().warm_key().into(), handle)], + ); + let port = pool.port(); + for expected in 1..=3 { + port.invoke(make(), invocation(deadline()), false) + .await + .unwrap(); + idle(&port).await; + assert_eq!(count.load(Ordering::Acquire), expected); + } + pool.shutdown(deadline()).await.unwrap(); + served.await.unwrap(); + assert_eq!(port.status().warm, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case::abandoned(true)] +#[case::live_but_unpolled(false)] +#[tokio::test] +async fn test_should_retire_unconsumed_ready_reply_even_without_public_future_polling( + #[case] abandoned: bool, +) { + use std::{future::Future, sync::atomic::AtomicUsize, task::Poll}; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let key = plan.warm_key().to_owned(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let served = tokio::spawn(peer.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(key, handle)], + ); + let port = pool.port(); + let mut call = Box::pin(port.invoke( + plan, + invocation(Instant::now() + Duration::from_millis(250)), + false, + )); + assert!(matches!( + call.as_mut() + .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())), + Poll::Pending + )); + tokio::time::sleep(Duration::from_millis(20)).await; + if abandoned { + drop(call); + } else { + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + receipt.removed(), + "unpolled waiter retained its environment past deadline" + ); + assert!(matches!( + call.as_mut() + .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())), + Poll::Ready(Err(EnvironmentError::Deadline)) + )); + } + tokio::time::timeout(Duration::from_secs(2), served) + .await + .unwrap() + .unwrap(); + assert!(receipt.removed()); + pool.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().warm, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_one_queue_across_mailbox_and_waiting_dispatch_then_cancel_without_replay() + { + use std::{ + future::Future, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start( + environments.port(), + PoolSettings::builder().queue(1).build().try_into().unwrap(), + ) + .unwrap(); + let port = pool.port(); + let mut waiting = Box::pin(port.invoke(plan(&lease), invocation(deadline()), false)); + assert!(matches!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::sleep(Duration::from_millis(120)).await; + assert_eq!(port.status().queued, 1); + assert_eq!(port.status().active, 0); + assert!(matches!( + port.invoke(plan(&lease), invocation(deadline()), false) + .await, + Err(EnvironmentError::Busy) + )); + drop(waiting); + tokio::time::timeout(Duration::from_secs(2), async { + while port.status().queued != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(daemon.requests.try_recv().is_err()); + pool.shutdown(deadline()).await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::idle(false)] +#[case::offered_result(true)] +#[tokio::test] +async fn test_should_recover_idle_or_dispatched_environment_after_root_abort( + #[case] offered: bool, +) { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let key = plan.warm_key().into(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let served = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(key, handle)], + ); + let port = pool.port(); + let mut call = Box::pin(port.invoke(plan, invocation(deadline()), false)); + if offered { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::timeout(Duration::from_secs(2), async { + while count.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + pool.task.as_ref().unwrap().abort(); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(receipt.removed()); + served.await.unwrap(); + drop(call); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_one_lru_eviction_barrier_while_cleanup_is_delayed() { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let (oldest, first) = crate::docker::environment::test_peer::pair(); + let first_receipt = oldest.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let first_task = tokio::spawn(first.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let (next, second) = crate::docker::environment::test_peer::pair(); + let second_receipt = next.retirement(); + let second_task = tokio::spawn(second.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![("oldest".into(), oldest), ("next".into(), next)], + ); + let port = pool.port(); + let mut waiting = Box::pin(port.invoke(plan(&lease), invocation(deadline()), false)); + assert!(matches!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::timeout(Duration::from_secs(2), stopped) + .await + .unwrap() + .unwrap(); + tokio::time::sleep(Duration::from_millis(350)).await; + assert!(!first_receipt.removed()); + assert!(!second_receipt.removed()); + assert_eq!(port.status().warm, 1); + assert_eq!(port.status().active, 1); + drop(waiting); + release.send(()).unwrap(); + pool.shutdown(deadline()).await.unwrap(); + first_task.await.unwrap(); + second_task.await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_retirement_owned_when_shutdown_waiter_is_cancelled() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let served = tokio::spawn(peer.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![("idle".into(), handle)], + ); + assert!( + tokio::time::timeout(Duration::from_millis(30), pool.shutdown(deadline())) + .await + .is_err() + ); + stopped.await.unwrap(); + assert!(!receipt.removed()); + release.send(()).unwrap(); + pool.shutdown(deadline()).await.unwrap(); + pool.shutdown(deadline()).await.unwrap(); + served.await.unwrap(); + assert!(receipt.removed()); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_reject_oversized_captured_inputs_before_warm_fingerprinting() { + let lease = directory(); + let mut invalid = revision(); + invalid + .environment + .insert("OVERSIZED".into(), "s".repeat(8192)); + assert!( + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + Arc::new(invalid), + policy() + ) + .is_err() + ); + let mut invalid = revision(); + invalid.image_config = Some(rustack_lambda_model::types::ImageConfig { + command: Some(vec!["x".repeat(32768)]), + ..Default::default() + }); + assert!( + EnvironmentPlan::new( + lease, + image_with("1000", vec![]), + Arc::new(invalid), + policy() + ) + .is_err() + ); +} + +#[tokio::test] +async fn test_should_preserve_retirement_above_an_aborted_dispatch_worker() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let served = tokio::spawn(peer.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = + actor::testing_failed_worker_start(environments.port(), vec![("owned".into(), handle)]); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(pool.port().status().worker_failed); + assert!(receipt.removed()); + served.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_wait_for_capacity_in_fifo_order_and_never_replay_rejected_startup() { + use std::{ + future::Future, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let port = pool.port(); + let first = plan(&lease); + let first_key = first.relay_key().as_str().to_owned(); + let second = plan(&lease); + let second_key = second.relay_key().as_str().to_owned(); + let mut first = Box::pin(port.invoke(first, invocation(deadline()), false)); + let mut second = Box::pin(port.invoke(second, invocation(deadline()), false)); + for call in [&mut first, &mut second] { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + } + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + for key in [first_key, second_key] { + let request = daemon.next().await; + assert!( + percent_encoding::percent_decode_str(&request.head) + .decode_utf8() + .unwrap() + .contains(&key) + ); + request.reply(400, b"{}").await; + } + assert!(first.await.is_err()); + assert!(second.await.is_err()); + pool.shutdown(deadline()).await.unwrap(); + assert!(daemon.requests.try_recv().is_err()); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::no_global_retention(0, 1, 0)] +#[case::no_revision_retention(32, 0, 0)] +#[case::one_per_revision(32, 1, 1)] +#[tokio::test] +async fn test_should_retire_successful_excess_concurrent_returns( + #[case] warm: u8, + #[case] per_revision: u8, + #[case] retained: usize, +) { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let key = make().warm_key().to_owned(); + let count = Arc::new(AtomicUsize::new(0)); + let mut seeded = Vec::new(); + let mut peers = Vec::new(); + for _ in 0..2 { + let (handle, peer) = crate::docker::environment::test_peer::pair(); + seeded.push((key.clone(), handle)); + peers.push(tokio::spawn(peer.serve(count.clone()))); + } + let config = PoolSettings::builder() + .warm(warm) + .warm_per_revision(per_revision) + .build() + .try_into() + .unwrap(); + let mut pool = actor::testing_start(environments.port(), config, seeded); + let port = pool.port(); + let mut first = Box::pin(port.invoke(make(), invocation(deadline()), false)); + let mut second = Box::pin(port.invoke(make(), invocation(deadline()), false)); + for call in [&mut first, &mut second] { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + } + tokio::time::timeout(Duration::from_secs(2), async { + while count.load(Ordering::Acquire) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + first.await.unwrap(); + second.await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), async { + while port.status().active != 0 || port.status().warm != retained { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + pool.shutdown(deadline()).await.unwrap(); + for peer in peers { + peer.await.unwrap(); + } + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_impossible_memory_without_evicting_warm_work_or_blocking_fitting_calls() +{ + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = EnvironmentSettings::builder() + .memory_bytes(256 * 1024 * 1024) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let fitting = plan(&lease); + let (handle, peer) = pair(); + let count = Arc::new(AtomicUsize::new(0)); + let served = tokio::spawn(peer.serve(count.clone())); + let mut pool = testing_start( + environments.port(), + PoolConfig::default(), + vec![(fitting.warm_key().into(), handle)], + ); + let mut oversized = revision(); + oversized.memory_mb = 256; + let oversized = EnvironmentPlan::new( + lease, + image_with("1000", vec![]), + Arc::new(oversized), + policy(), + ) + .unwrap(); + assert!(matches!( + pool.port() + .invoke(oversized, invocation(deadline()), false) + .await, + Err(EnvironmentError::MemoryLimit) + )); + assert_eq!(pool.port().status().warm, 1); + pool.port() + .invoke(fitting, invocation(deadline()), false) + .await + .unwrap(); + idle(&pool.port()).await; + assert_eq!(count.load(Ordering::Acquire), 1); + assert_eq!(environments.port().status().allocated, 0); + pool.shutdown(deadline()).await.unwrap(); + served.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs b/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs new file mode 100644 index 00000000..e10c0c1d --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs @@ -0,0 +1,298 @@ +use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Waker}, +}; + +use super::*; +use crate::docker::environment::test_peer; + +#[rstest::rstest] +#[case::cancel(false, false)] +#[case::expired(true, false)] +#[case::root_abort(false, true)] +#[tokio::test] +async fn test_should_retain_per_call_settlement_across_expiry_waiter_drop_and_root_recovery( + #[case] expired: bool, + #[case] abort: bool, +) { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let (handle, peer) = test_peer::pair(); + let physical = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let serving = tokio::spawn(peer.serve_gated(count.clone(), gate, stopping)); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(plan.warm_key().into(), handle)], + ); + let until = if expired { + Instant::now() + Duration::from_millis(250) + } else { + deadline() + }; + let mut call = pool.port().start(plan, invocation(until), false).unwrap(); + timeout_at(deadline(), async { + while count.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + if abort { + pool.task.as_ref().unwrap().abort(); + } else if expired { + tokio::time::sleep_until(until).await; + assert!(matches!(call.wait().await, Err(EnvironmentError::Deadline))); + } + let mut waiting = Box::pin(call.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + timeout_at(deadline(), stopped).await.unwrap().unwrap(); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(!physical.removed()); + assert!(matches!( + call.cancel_and_wait(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + assert!(!physical.removed()); + release.send(()).unwrap(); + let result = call.cancel_and_wait(deadline()).await; + assert!(physical.removed()); + if abort { + assert!(matches!(result, Err(EnvironmentError::Worker))); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + } else { + result.unwrap(); + call.cancel_and_wait(deadline()).await.unwrap(); + pool.shutdown(deadline()).await.unwrap(); + } + serving.await.unwrap(); + assert_eq!(count.load(Ordering::Acquire), 1); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_resume_pending_response_wait_and_leave_consumed_success_warm() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let (handle, peer) = test_peer::pair(); + let physical = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let serving = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(make().warm_key().into(), handle)], + ); + let port = pool.port(); + for expected in 1..=2 { + let mut call = port.start(make(), invocation(deadline()), false).unwrap(); + let mut waiting = Box::pin(call.wait()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + call.wait().await.unwrap(); + assert!(matches!( + call.wait().await, + Err(EnvironmentError::Configuration) + )); + call.cancel_and_wait(deadline()).await.unwrap(); + drop(call); + idle(&port).await; + assert!(!physical.removed()); + assert_eq!(count.load(Ordering::Acquire), expected); + } + pool.shutdown(deadline()).await.unwrap(); + serving.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_settle_cancelled_queued_call_without_waiting_for_an_unrelated_allocation() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = EnvironmentManager::start( + wire.port(), + lease.clone(), + crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(), + ) + .unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let mut call = pool + .port() + .start(plan(&lease), invocation(deadline()), false) + .unwrap(); + call.cancel_and_wait(deadline()).await.unwrap(); + assert_eq!(environments.port().status().allocated, 1); + assert!(daemon.requests.try_recv().is_err()); + pool.shutdown(deadline()).await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_call_receipt_above_a_failed_dispatch_worker() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let (environment, peer) = test_peer::pair(); + let physical = environment.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let serving = tokio::spawn(peer.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let (mut pool, mut call) = actor::testing_failed_call_start( + environments.port(), + plan(&lease), + invocation(deadline()), + environment, + ); + timeout_at(deadline(), stopped).await.unwrap().unwrap(); + assert!(matches!(call.wait().await, Err(EnvironmentError::Worker))); + let mut waiting = Box::pin(call.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(!physical.removed()); + release.send(()).unwrap(); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(physical.removed()); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + serving.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_treat_a_closed_response_and_missing_settlement_as_uncertain() { + let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); + drop(reply); + drop(settlement); + let mut call = PoolInvocation { + deadline: deadline(), + stop: watch::channel(false).0, + response: Some(response), + settlement: observed, + failed: Arc::new(AtomicBool::new(false)), + consumed: false, + }; + assert!(matches!(call.wait().await, Err(EnvironmentError::Stopped))); + assert!(matches!( + call.wait().await, + Err(EnvironmentError::Configuration) + )); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); +} + +#[rstest::rstest] +#[case::expired(true)] +#[case::excessive(false)] +#[tokio::test] +async fn test_should_signal_cancellation_before_rejecting_invalid_cleanup_deadline( + #[case] expired: bool, +) { + let (reply, response) = oneshot::channel(); + let (_settled, observed) = watch::channel(Some(Ok(()))); + let mut call = PoolInvocation { + deadline: deadline(), + stop: watch::channel(false).0, + response: Some(response), + settlement: observed, + failed: Arc::new(AtomicBool::new(false)), + consumed: false, + }; + let until = if expired { + Instant::now() + } else { + Instant::now() + Duration::from_secs(601) + }; + let error = call.cancel_and_wait(until).await.unwrap_err(); + if expired { + assert!(matches!(error, EnvironmentError::Deadline)); + } else { + assert!(matches!(error, EnvironmentError::Configuration)); + } + assert!(*call.stop.borrow()); + assert!(reply.is_closed()); + call.cancel_and_wait(deadline()).await.unwrap(); +} diff --git a/crates/rustack-lambda-core/src/docker/probe.rs b/crates/rustack-lambda-core/src/docker/probe.rs new file mode 100644 index 00000000..088957f9 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe.rs @@ -0,0 +1,441 @@ +//! Actual resource qualification through the retained environment owner. + +mod endpoint; +use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; + +use bytes::Bytes; +pub use endpoint::{EndpointProbe, EndpointProbeConfig, probe_endpoint}; +use rustack_core::json::{JsonError, JsonLimits, from_slice}; +use rustack_lambda_model::types::ImageConfig; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + probe::{REPORT_BYTES, ResourceReport}, + protocol::{Architecture as RuntimeArchitecture, AttemptId, Payload}, + session::{Invocation, RuntimeResult}, +}; +use thiserror::Error; +use tokio::time::Instant; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + ContainerPolicy, DirectoryLease, EnvironmentError, EnvironmentPlan, EnvironmentPort, + QualifiedNetwork, check_deadline, +}; +use crate::{ + artifact::{Architecture, DeploymentArtifact, PinnedImage}, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, +}; + +/// Specific bounded resource qualification mismatch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum ProbeFailure { + /// Wrong report schema, attempt or compiled architecture. + #[error("resource probe identity mismatch")] + Identity, + /// Controller files are not rooted at the current process membership. + #[error("resource probe cgroup namespace mismatch")] + Namespace, + /// Memory hard cap was not enforced. + #[error("resource probe memory mismatch")] + Memory, + /// Extra swap was permitted or unavailable. + #[error("resource probe swap mismatch")] + Swap, + /// PID cap was not enforced. + #[error("resource probe PID mismatch")] + Pids, + /// CPU quota/period was not enforced. + #[error("resource probe CPU mismatch")] + Cpu, + /// The actual process has a root UID/GID. + #[error("resource probe user mismatch")] + User, + /// Effective capabilities are not empty. + #[error("resource probe capability mismatch")] + Capabilities, + /// `no_new_privs` was not enforced. + #[error("resource probe privilege mismatch")] + Privilege, + /// No active seccomp filter. + #[error("resource probe seccomp mismatch")] + Seccomp, + /// The effective root mount is writable. + #[error("resource probe root filesystem mismatch")] + Root, + /// The effective tmp mount differs from the captured policy. + #[error("resource probe temporary filesystem mismatch")] + Temporary, + /// The observed gateway was not the exact expected instance/attempt/platform. + #[error("endpoint probe identity mismatch")] + Endpoint, + /// The trusted probe did not return a report. + #[error("resource probe execution failed")] + Outcome, +} +/// Probe failures preserve bounded classes and redacted typed sources. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum EnvironmentProbeError { + /// Invalid settings or mismatched retained runtime scope. + #[error("invalid environment probe configuration")] + Configuration, + /// Bounded strict report decoding failed. + #[error("invalid environment probe report")] + Report(#[from] JsonError), + /// Actual observations did not satisfy policy. + #[error("{0}")] + Rejected(#[from] ProbeFailure), + /// Retained allocation/invocation failed. + #[error("environment probe failed: {0}")] + Environment(#[from] EnvironmentError), + /// Cleanup did not confirm physical/IO settlement; lower owner retains it. + #[error("environment probe cleanup not confirmed")] + Cleanup(#[source] EnvironmentError), +} +impl fmt::Debug for EnvironmentProbeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// Validated positive whole-second startup and independent cleanup budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ProbeSettings { + start: Duration, + cleanup: Duration, +} +impl Default for ProbeSettings { + fn default() -> Self { + Self { + start: Duration::from_secs(60), + cleanup: Duration::from_secs(10), + } + } +} +impl ProbeSettings { + /// Validate both budgets in1..60 whole seconds. + /// + /// # Errors + /// Rejects zero, fractional or excessive durations. + /// + /// ``` + /// let settings=rustack_lambda_core::docker::ProbeSettings::new( + /// std::time::Duration::from_secs(30),std::time::Duration::from_secs(10))?; + /// # Ok::<(),rustack_lambda_core::docker::EnvironmentProbeError>(()) + /// ``` + pub fn new(start: Duration, cleanup: Duration) -> Result { + for duration in [start, cleanup] { + if duration.subsec_nanos() != 0 || !(1..=60).contains(&duration.as_secs()) { + return Err(EnvironmentProbeError::Configuration); + } + } + Ok(Self { start, cleanup }) + } +} +/// Operator-owned private probe inputs; no user image or endpoint is accepted. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct EnvironmentProbeConfig { + /// Persistent namespace capability. + pub directory: DirectoryLease, + /// Pinned runtime image used for both relay and trusted probe function. + pub relay: PinnedImage, + /// Matching retained environment service. + pub environments: EnvironmentPort, + /// Operator policy, bound to the qualified network before allocation. + pub policy: ContainerPolicy, + /// Independent read-only network qualification. + pub network: QualifiedNetwork, + /// Validated startup/cleanup budgets. + #[builder(default)] + pub settings: ProbeSettings, +} +/// Private proof of actual matching kernel observations after joined retirement. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EnvironmentProbe { + report: ResourceReport, + directory: DirectoryLease, + relay: PinnedImage, + policy: ContainerPolicy, + environments: EnvironmentPort, +} +impl EnvironmentProbe { + /// Inspect bounded observation data; it cannot reconstruct this proof. + /// + /// ```no_run + /// # fn inspect(probe:&rustack_lambda_core::docker::EnvironmentProbe){ + /// let memory=probe.report().memory_max; + /// # } + /// ``` + #[must_use] + pub fn report(&self) -> &ResourceReport { + &self.report + } + /// Check the exact retained inputs and accepting environment service. + /// + /// ```no_run + /// # fn matches(probe:&rustack_lambda_core::docker::EnvironmentProbe, inputs:&rustack_lambda_core::docker::EnvironmentProbeConfig){ + /// let current=probe.matches(inputs); + /// # } + /// ``` + #[must_use] + pub fn matches(&self, inputs: &EnvironmentProbeConfig) -> bool { + let metadata = self.relay.resolved().metadata(); + let other = inputs.relay.resolved().metadata(); + self.directory.owner_id() == inputs.directory.owner_id() + && self.directory.run_id() == inputs.directory.run_id() + && self.relay.is_live() + && inputs.relay.is_live() + && metadata.image_id == other.image_id + && metadata.inspection_hash == other.inspection_hash + && inputs + .network + .apply(&inputs.policy) + .is_ok_and(|policy| policy == self.policy) + && self.environments.same_service(&inputs.environments) + && self.environments.status().accepting + } +} + +/// Execute the pinned Rust probe once and issue proof only after actual retirement. +/// +/// This does not grant production readiness. The environment manager must outlive +/// a dropped/expired waiter, retaining uncertain container/IO cleanup. +/// +/// # Errors +/// Returns invalid inputs, runtime/wire failure, strict report mismatch or cleanup +/// uncertainty. A failed accepted probe is never replayed. +/// +/// ```no_run +/// # async fn probe(config:rustack_lambda_core::docker::EnvironmentProbeConfig){ +/// let evidence=rustack_lambda_core::docker::probe_environment(config, +/// tokio::time::Instant::now()+std::time::Duration::from_secs(60)).await; +/// # } +/// ``` +pub async fn probe_environment( + config: EnvironmentProbeConfig, + deadline: Instant, +) -> Result { + let deadline = deadline.min(Instant::now() + config.settings.start); + let payload = Payload::try_from(Bytes::from_static(b"{}")) + .map_err(|_| EnvironmentProbeError::Configuration)?; + let (report, policy) = invoke_probe( + &config, + ProbeCommand::Resource, + payload, + deadline, + |bytes, attempt, architecture, policy| { + let report = decode_report(bytes)?; + verify(&report, attempt, architecture, policy)?; + Ok(report) + }, + ) + .await?; + check_live(&config, deadline)?; + Ok(EnvironmentProbe { + report, + directory: config.directory, + relay: config.relay, + policy, + environments: config.environments, + }) +} +#[derive(Debug, Clone, Copy)] +enum ProbeCommand { + Resource, + Endpoint, +} +impl ProbeCommand { + fn entry(self) -> &'static str { + match self { + Self::Resource => "probe", + Self::Endpoint => "endpoint-probe", + } + } + fn name(self) -> &'static str { + match self { + Self::Resource => "rustack-runtime-probe", + Self::Endpoint => "rustack-endpoint-probe", + } + } +} +fn check_live( + config: &EnvironmentProbeConfig, + deadline: Instant, +) -> Result<(), EnvironmentProbeError> { + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline.into()); + } + if !config.environments.same_scope(&config.directory) + || !config.relay.is_live() + || !config.environments.status().accepting + { + return Err(EnvironmentProbeError::Configuration); + } + Ok(()) +} +async fn invoke_probe( + config: &EnvironmentProbeConfig, + command: ProbeCommand, + payload: Payload, + deadline: Instant, + validate: impl FnOnce( + &[u8], + AttemptId, + Architecture, + &ContainerPolicy, + ) -> Result, +) -> Result<(T, ContainerPolicy), EnvironmentProbeError> { + check_deadline(deadline).map_err(EnvironmentError::from)?; + check_live(config, deadline)?; + let policy = config + .network + .apply(&config.policy) + .map_err(EnvironmentError::from)?; + let architecture = config.relay.resolved().metadata().architecture; + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name(command.name().into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(architecture) + .artifact(DeploymentArtifact::Image(config.relay.clone())) + .image_config(Some(ImageConfig { + entry_point: Some(vec![ + "/rustack-lambda-runtime".into(), + command.entry().into(), + ]), + command: Some(vec![]), + working_directory: None, + })) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ])) + .timeout(Duration::from_secs(10)) + .memory_mb(128) + .ephemeral_storage_mb(512) + .build(); + let plan = EnvironmentPlan::new( + config.directory.clone(), + config.relay.clone(), + Arc::new(revision), + policy.clone(), + ) + .map_err(EnvironmentError::from)?; + let attempt = AttemptId::default(); + let call = Invocation::new( + attempt, + deadline, + format!( + "arn:aws:lambda:us-east-1:000000000000:function:{}", + command.name() + ) + .parse() + .map_err(|_| EnvironmentProbeError::Configuration)?, + local_trace_id().map_err(|_| EnvironmentProbeError::Configuration)?, + payload, + ) + .map_err(|_| EnvironmentProbeError::Configuration)?; + let mut environment = config.environments.spawn(plan, deadline)?; + let result = async { + environment.wait_ready(deadline).await?; + let response = environment.invoke(call, false).await?; + let RuntimeResult::Success(payload) = response.result else { + return Err(ProbeFailure::Outcome.into()); + }; + validate(payload.as_ref(), attempt, architecture, &policy) + } + .await; + environment + .shutdown(Instant::now() + config.settings.cleanup) + .await + .map_err(EnvironmentProbeError::Cleanup)?; + Ok((result?, policy)) +} + +fn decode_report(bytes: &[u8]) -> Result { + Ok(from_slice( + bytes, + JsonLimits::new(REPORT_BYTES, 8, 128, 256, 32)?, + )?) +} +fn verify( + report: &ResourceReport, + attempt: AttemptId, + architecture: Architecture, + policy: &ContainerPolicy, +) -> Result<(), ProbeFailure> { + if report.version != 1 + || report.attempt != attempt + || report.architecture + != match architecture { + Architecture::Arm64 => RuntimeArchitecture::Arm64, + Architecture::X86_64 => RuntimeArchitecture::X86_64, + } + { + return Err(ProbeFailure::Identity); + } + if report.memory_max != Some(128 * 1024 * 1024) { + return Err(ProbeFailure::Memory); + } + if !report.cgroup_root { + return Err(ProbeFailure::Namespace); + } + if report.swap_max != Some(0) { + return Err(ProbeFailure::Swap); + } + if report.pids_max != Some(256) { + return Err(ProbeFailure::Pids); + } + let expected = u64::try_from(policy.nano_cpus) + .ok() + .and_then(|cpu| cpu.checked_mul(report.cpu_period)) + .map(|total| total / 1_000_000_000); + if !(1000..=1_000_000).contains(&report.cpu_period) + || !report + .cpu_quota + .is_some_and(|quota| (1000..=16_000_000).contains("a)) + || expected == Some(0) + || report.cpu_quota != expected + || expected.is_none() + { + return Err(ProbeFailure::Cpu); + } + if report.euid == 0 || report.egid == 0 { + return Err(ProbeFailure::User); + } + if report.cap_effective != 0 { + return Err(ProbeFailure::Capabilities); + } + if !report.no_new_privileges { + return Err(ProbeFailure::Privilege); + } + if report.seccomp_mode != 2 { + return Err(ProbeFailure::Seccomp); + } + if !report.root_readonly { + return Err(ProbeFailure::Root); + } + let temporary = &report.temporary; + if temporary.bytes != 512 * 1024 * 1024 + || temporary.mode != 0o1777 + || !temporary.tmpfs + || !temporary.nosuid + || !temporary.nodev + { + return Err(ProbeFailure::Temporary); + } + Ok(()) +} + +#[cfg(test)] +pub(super) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/probe/endpoint.rs b/crates/rustack-lambda-core/src/docker/probe/endpoint.rs new file mode 100644 index 00000000..a053f9ac --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe/endpoint.rs @@ -0,0 +1,160 @@ +//! Matching real resource evidence and retained container-origin endpoint proof. + +use bytes::Bytes; +use rustack_core::json::{JsonLimits, from_slice}; +use rustack_lambda_runtime::{ + probe::{EndpointChallenge, EndpointReport, GatewayEndpoint, GatewayInstanceId, REPORT_BYTES}, + protocol::{Architecture as RuntimeArchitecture, AttemptId, Payload}, +}; +use serde_json::to_vec; +use tokio::time::Instant; + +use super::{ + EnvironmentProbe, EnvironmentProbeConfig, EnvironmentProbeError, ProbeCommand, ProbeFailure, + check_live, invoke_probe, +}; +use crate::{artifact::Architecture, docker::DockerGatewayEndpoint}; + +/// Immutable endpoint qualification inputs, requiring actual matching kernel proof. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EndpointProbeConfig { + base: EnvironmentProbeConfig, + resource: EnvironmentProbe, + endpoint: DockerGatewayEndpoint, + instance: GatewayInstanceId, +} +impl EndpointProbeConfig { + pub(in crate::docker) fn gateway_endpoint(&self) -> &DockerGatewayEndpoint { + &self.endpoint + } + /// Capture the exact qualified resource inputs and configured endpoint instance. + /// # Errors + /// Rejects disconnected networks and revoked/mismatching retained resource evidence. + /// ```no_run + /// # fn config(base:rustack_lambda_core::docker::EnvironmentProbeConfig,resource:rustack_lambda_core::docker::EnvironmentProbe,endpoint:rustack_lambda_core::docker::DockerGatewayEndpoint,instance:rustack_lambda_core::docker::GatewayInstanceId)->Result<(),rustack_lambda_core::docker::EnvironmentProbeError>{ + /// let config=rustack_lambda_core::docker::EndpointProbeConfig::new(base,resource,endpoint,instance)?; + /// # Ok(()) } + /// ``` + pub fn new( + base: EnvironmentProbeConfig, + resource: EnvironmentProbe, + endpoint: DockerGatewayEndpoint, + instance: GatewayInstanceId, + ) -> Result { + if base.network.disconnected() || !resource.matches(&base) { + return Err(EnvironmentProbeError::Configuration); + } + Ok(Self { + base, + resource, + endpoint, + instance, + }) + } + fn check(&self, deadline: Instant) -> Result<(), EnvironmentProbeError> { + check_live(&self.base, deadline)?; + if self.base.network.disconnected() || !self.resource.matches(&self.base) { + return Err(EnvironmentProbeError::Configuration); + } + Ok(()) + } +} +/// Private receipt issued only after actual validated IO and physical retirement. +/// ```compile_fail +/// let forged=serde_json::from_str::("{}"); +/// ``` +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EndpointProbe { + report: EndpointReport, + inputs: EndpointProbeConfig, +} +impl EndpointProbe { + /// Recheck exact endpoint/identity and all retained live resource inputs. + /// ```no_run + /// # fn matches(proof:&rustack_lambda_core::docker::EndpointProbe,inputs:&rustack_lambda_core::docker::EndpointProbeConfig){ + /// let current=proof.matches(inputs); + /// # } + /// ``` + #[must_use] + pub fn matches(&self, inputs: &EndpointProbeConfig) -> bool { + self.inputs.endpoint == inputs.endpoint + && self.inputs.instance == inputs.instance + && self.inputs.resource.matches(&inputs.base) + && inputs.resource.matches(&inputs.base) + && self.inputs.resource.matches(&self.inputs.base) + && !inputs.base.network.disconnected() + } + /// Bounded observed report data; cloning/serializing data cannot issue proof. + /// ```no_run + /// # fn report(proof:&rustack_lambda_core::docker::EndpointProbe){let data=proof.report();} + /// ``` + #[must_use] + pub fn report(&self) -> &EndpointReport { + &self.report + } +} +/// Execute one endpoint probe; a failed accepted invocation is never replayed. +/// # Errors +/// Rejects stale/invalid inputs, network/protocol/report mismatch, timeout or +/// uncertain cleanup. Dropped callers leave retirement with the environment owner. +/// ```no_run +/// # async fn probe(config:rustack_lambda_core::docker::EndpointProbeConfig){ +/// let proof=rustack_lambda_core::docker::probe_endpoint(config,tokio::time::Instant::now()+std::time::Duration::from_secs(60)).await; +/// # } +/// ``` +pub async fn probe_endpoint( + config: EndpointProbeConfig, + deadline: Instant, +) -> Result { + let deadline = deadline.min(Instant::now() + config.base.settings.start); + config.check(deadline)?; + let challenge = EndpointChallenge::new( + GatewayEndpoint::from(config.endpoint.clone()), + config.instance, + ); + let bytes = to_vec(&challenge).map_err(|_| EnvironmentProbeError::Configuration)?; + if bytes.len() > REPORT_BYTES { + return Err(EnvironmentProbeError::Configuration); + } + let payload = + Payload::try_from(Bytes::from(bytes)).map_err(|_| EnvironmentProbeError::Configuration)?; + let (report, _policy) = invoke_probe( + &config.base, + ProbeCommand::Endpoint, + payload, + deadline, + |bytes, attempt, architecture, _policy| { + verify_report(bytes, attempt, architecture, config.instance) + }, + ) + .await?; + config.check(deadline)?; + Ok(EndpointProbe { + report, + inputs: config, + }) +} + +fn verify_report( + bytes: &[u8], + attempt: AttemptId, + architecture: Architecture, + instance: GatewayInstanceId, +) -> Result { + let report: EndpointReport = from_slice(bytes, JsonLimits::new(REPORT_BYTES, 8, 16, 256, 32)?)?; + let expected = match architecture { + Architecture::Arm64 => RuntimeArchitecture::Arm64, + Architecture::X86_64 => RuntimeArchitecture::X86_64, + }; + if report.attempt() != attempt + || report.architecture() != expected + || report.instance_id() != instance + { + return Err(ProbeFailure::Endpoint.into()); + } + Ok(report) +} +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/probe/endpoint/tests.rs b/crates/rustack-lambda-core/src/docker/probe/endpoint/tests.rs new file mode 100644 index 00000000..d7792eb5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe/endpoint/tests.rs @@ -0,0 +1,198 @@ +use std::time::Duration; + +use rstest::rstest; +use rustack_lambda_runtime::protocol::AttemptId; +use serde_json::{json, to_vec}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{EndpointProbe, EndpointProbeConfig, verify_report}; +use crate::{ + artifact::Architecture, + docker::{ + ContainerPolicy, DirectoryLease, DockerWire, EnvironmentConfig, EnvironmentManager, + EnvironmentProbe, EnvironmentProbeConfig, EnvironmentProbeError, GatewayInstanceId, + NetworkSelection, RelayBudgets, plan::tests::image_with, probe::tests::report, + probe_endpoint, tests::Daemon, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn report_bytes(attempt: AttemptId, id: GatewayInstanceId) -> Vec { + to_vec(&json!({"version":1,"attempt":attempt,"architecture":"arm64","instanceId":id})).unwrap() +} +#[rstest] +#[case::attempt(0)] +#[case::architecture(1)] +#[case::instance(2)] +#[case::version(3)] +#[case::unknown(4)] +#[test] +fn test_should_require_exact_endpoint_report_schema_attempt_architecture_and_instance( + #[case] case: u8, +) { + let attempt = AttemptId::default(); + let id = GatewayInstanceId::generate().unwrap(); + let mut value = json!({"version":1,"attempt":attempt,"architecture":"arm64","instanceId":id}); + match case { + 0 => value["attempt"] = json!(AttemptId::default()), + 1 => value["architecture"] = json!("x86_64"), + 2 => value["instanceId"] = json!(GatewayInstanceId::generate().unwrap()), + 3 => value["version"] = json!(2), + _ => value["extra"] = json!("PRIVATE-PEER"), + } + let error = + verify_report(&to_vec(&value).unwrap(), attempt, Architecture::Arm64, id).unwrap_err(); + assert!(!format!("{error:?}:{error}").contains("PRIVATE-PEER")); +} +#[test] +fn test_should_decode_bounded_endpoint_report_without_turning_data_into_a_receipt() { + let attempt = AttemptId::default(); + let id = GatewayInstanceId::generate().unwrap(); + let bytes = report_bytes(attempt, id); + let report = verify_report(&bytes, attempt, Architecture::Arm64, id).unwrap(); + assert_eq!(report.instance_id(), id); + assert!(verify_report(&vec![b' '; 4097], attempt, Architecture::Arm64, id).is_err()); + let duplicate = format!( + "{{\"version\":1,\"version\":1,\"attempt\":\"{attempt}\",\"architecture\":\"arm64\",\"\ + instanceId\":{}}}", + serde_json::to_string(&id).unwrap() + ); + assert!(verify_report(duplicate.as_bytes(), attempt, Architecture::Arm64, id).is_err()); +} +async fn fixture() -> (Daemon, DockerWire, EnvironmentManager, EndpointProbeConfig) { + let mut daemon = Daemon::new(); + let wire = daemon.wire().await; + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let owner = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + let selection: NetworkSelection = "endpoint-fixture".parse().unwrap(); + let port = wire.port(); + let (network, ()) = tokio::join!(selection.qualify(&port, deadline()), async { + daemon.next().await.reply(200,&to_vec(&json!({"Name":"endpoint-fixture","Id":"a".repeat(64),"Driver":"bridge","Scope":"local","Ingress":false,"ConfigOnly":false})).unwrap()).await; + }); + let network = network.unwrap(); + let policy = ContainerPolicy::new( + selection, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap(); + let relay = image_with("1000", vec![]); + let base = EnvironmentProbeConfig::builder() + .directory(directory.clone()) + .relay(relay.clone()) + .environments(owner.port()) + .network(network.clone()) + .policy(policy.clone()) + .build(); + // Synthetic private receipt only tests scope/cleanup logic; real proof is separately gated. + let resource = EnvironmentProbe { + report: report(AttemptId::default()), + directory, + relay, + policy: network.apply(&policy).unwrap(), + environments: owner.port(), + }; + let config = EndpointProbeConfig::new( + base, + resource, + "http://private-gateway.example:4566".parse().unwrap(), + GatewayInstanceId::generate().unwrap(), + ) + .unwrap(); + (daemon, wire, owner, config) +} +#[tokio::test] +async fn test_should_bind_endpoint_receipt_to_exact_origin_instance_scope_policy_and_live_owner() { + let (daemon, mut wire, mut owner, config) = fixture().await; + let attempt = AttemptId::default(); + let receipt = EndpointProbe { + report: verify_report( + &report_bytes(attempt, config.instance), + attempt, + Architecture::Arm64, + config.instance, + ) + .unwrap(), + inputs: config.clone(), + }; + assert!(receipt.matches(&config)); + let mut changed = config.clone(); + changed.endpoint = "http://other-gateway.example".parse().unwrap(); + assert!(!receipt.matches(&changed)); + let mut changed = config.clone(); + changed.instance = GatewayInstanceId::generate().unwrap(); + assert!(!receipt.matches(&changed)); + let mut changed = config.clone(); + changed.base.directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + assert!(!receipt.matches(&changed)); + let mut changed = config.clone(); + changed.base.policy = ContainerPolicy::new( + "endpoint-fixture".parse().unwrap(), + 1000, + 1000, + 200_000_000, + RelayBudgets::default(), + ) + .unwrap(); + assert!(!receipt.matches(&changed)); + assert!(matches!( + probe_endpoint(config.clone(), Instant::now()).await, + Err(EnvironmentProbeError::Environment(_)) + )); + let mut base = config.base.clone(); + base.network = "none" + .parse::() + .unwrap() + .qualify(&wire.port(), deadline()) + .await + .unwrap(); + assert!( + EndpointProbeConfig::new( + base, + config.resource.clone(), + config.endpoint.clone(), + config.instance + ) + .is_err() + ); + owner.shutdown(deadline()).await.unwrap(); + assert!(!receipt.matches(&config)); + assert!(probe_endpoint(config, deadline()).await.is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest] +#[case::rejected(false)] +#[case::abandoned(true)] +#[tokio::test] +async fn test_should_retain_endpoint_environment_cleanup_without_replaying_admitted_probe( + #[case] abandoned: bool, +) { + let (mut daemon, mut wire, mut owner, config) = fixture().await; + let task = tokio::spawn(probe_endpoint(config, deadline())); + let held = daemon.next().await; + if abandoned { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(owner.port().status().allocated, 1); + held.reply(400, b"{}").await; + } else { + held.reply(400, b"{}").await; + assert!(matches!( + task.await.unwrap(), + Err(EnvironmentProbeError::Environment(_)) + )); + } + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().allocated, 0); + assert!(daemon.requests.try_recv().is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/probe/tests.rs b/crates/rustack-lambda-core/src/docker/probe/tests.rs new file mode 100644 index 00000000..5e212399 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe/tests.rs @@ -0,0 +1,182 @@ +use rustack_lambda_runtime::{ + probe::TemporaryReport, protocol::Architecture as RuntimeArchitecture, +}; +use serde_json::{Value, json}; + +use super::*; +use crate::docker::{ + DockerWire, EnvironmentConfig, EnvironmentManager, RelayBudgets, + plan::tests::{image_with, policy}, + tests::Daemon, +}; + +pub(super) fn report(attempt: AttemptId) -> ResourceReport { + ResourceReport::builder() + .version(1) + .attempt(attempt) + .architecture(RuntimeArchitecture::Arm64) + .cgroup_root(true) + .memory_max(Some(128 * 1024 * 1024)) + .swap_max(Some(0)) + .pids_max(Some(256)) + .cpu_quota(Some(100_000)) + .cpu_period(100_000) + .euid(1000) + .egid(1000) + .cap_effective(0) + .no_new_privileges(true) + .seccomp_mode(2) + .root_readonly(true) + .temporary( + TemporaryReport::builder() + .bytes(512 * 1024 * 1024) + .mode(0o1777) + .tmpfs(true) + .nosuid(true) + .nodev(true) + .build(), + ) + .build() +} +#[rstest::rstest] +#[case::version("version",json!(2),ProbeFailure::Identity)] +#[case::namespace("cgroupRoot",json!(false),ProbeFailure::Namespace)] +#[case::architecture("architecture",json!("x86_64"),ProbeFailure::Identity)] +#[case::memory("memoryMax",json!(134_217_729),ProbeFailure::Memory)] +#[case::unlimited_memory("memoryMax", Value::Null, ProbeFailure::Memory)] +#[case::swap("swapMax",json!(1),ProbeFailure::Swap)] +#[case::pids("pidsMax",json!(257),ProbeFailure::Pids)] +#[case::cpu("cpuQuota", Value::Null, ProbeFailure::Cpu)] +#[case::period("cpuPeriod",json!(0),ProbeFailure::Cpu)] +#[case::overflow("cpuPeriod",json!(u64::MAX),ProbeFailure::Cpu)] +#[case::root_uid("euid",json!(0),ProbeFailure::User)] +#[case::root_gid("egid",json!(0),ProbeFailure::User)] +#[case::capabilities("capEffective",json!(1),ProbeFailure::Capabilities)] +#[case::privileges("noNewPrivileges",json!(false),ProbeFailure::Privilege)] +#[case::seccomp("seccompMode",json!(0),ProbeFailure::Seccomp)] +#[case::rootfs("rootReadonly",json!(false),ProbeFailure::Root)] +#[case::tmp_size("temporary",json!({"bytes":1,"mode":1023,"tmpfs":true,"nosuid":true,"nodev":true}),ProbeFailure::Temporary)] +#[case::tmp_flags("temporary",json!({"bytes":536_870_912,"mode":1023,"tmpfs":true,"nosuid":false,"nodev":true}),ProbeFailure::Temporary)] +#[test] +fn test_should_reject_each_actual_policy_mismatch( + #[case] field: &str, + #[case] value: Value, + #[case] failure: ProbeFailure, +) { + let attempt = AttemptId::default(); + let mut report = serde_json::to_value(report(attempt)).unwrap(); + report[field] = value; + let decoded = decode_report(&serde_json::to_vec(&report).unwrap()).unwrap(); + assert_eq!( + verify(&decoded, attempt, Architecture::Arm64, &policy()), + Err(failure) + ); +} +#[test] +fn test_should_verify_exact_attempt_and_checked_cpu_rounding_and_bound_reports() { + let attempt = AttemptId::default(); + let mut observed = report(attempt); + verify(&observed, attempt, Architecture::Arm64, &policy()).unwrap(); + assert_eq!( + verify( + &observed, + AttemptId::default(), + Architecture::Arm64, + &policy() + ), + Err(ProbeFailure::Identity) + ); + let policy = ContainerPolicy::new( + "none".parse().unwrap(), + 1000, + 1000, + 333_333_333, + RelayBudgets::default(), + ) + .unwrap(); + observed.cpu_quota = Some(33_000); + verify(&observed, attempt, Architecture::Arm64, &policy).unwrap(); + for quota in [33_333, 34_000] { + observed.cpu_quota = Some(quota); + assert_eq!( + verify(&observed, attempt, Architecture::Arm64, &policy), + Err(ProbeFailure::Cpu) + ); + } + observed.cpu_quota = Some(33_000); + assert!(decode_report(&vec![b' '; REPORT_BYTES + 1]).is_err()); + let mut raw = serde_json::to_value(&observed).unwrap(); + raw["unknown"] = json!(true); + assert!(decode_report(&serde_json::to_vec(&raw).unwrap()).is_err()); + assert!(decode_report(br#"{"version":1,"version":1}"#).is_err()); + assert!(ProbeSettings::new(Duration::ZERO, Duration::from_secs(1)).is_err()); + assert!(ProbeSettings::new(Duration::from_secs(1), Duration::from_millis(1500)).is_err()); +} +#[rstest::rstest] +#[case::rejected(false)] +#[case::abandoned(true)] +#[tokio::test] +async fn test_should_settle_rejected_probe_creation_and_retain_abandoned_waiter_cleanup( + #[case] abandoned: bool, +) { + let mut daemon = Daemon::new(); + let mut wire: DockerWire = daemon.wire().await; + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let mut environments = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let network = "none" + .parse::() + .unwrap() + .qualify(&wire.port(), deadline) + .await + .unwrap(); + let config = EnvironmentProbeConfig::builder() + .directory(directory) + .relay(image_with("1000", vec![])) + .environments(environments.port()) + .policy(policy()) + .network(network) + .build(); + let mut foreign = config.clone(); + foreign.directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + assert!(matches!( + probe_environment(foreign, deadline).await, + Err(EnvironmentProbeError::Configuration) + )); + let task = tokio::spawn(probe_environment(config, deadline)); + let creation = daemon.next().await; + if abandoned { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(environments.port().status().allocated, 1); + creation.reply(400, b"{}").await; + } else { + creation.reply(400, b"{}").await; + assert!(matches!( + task.await.unwrap(), + Err(EnvironmentProbeError::Environment(_)) + )); + assert_eq!(environments.port().status().allocated, 0); + } + assert!(daemon.requests.try_recv().is_err()); + environments.shutdown(deadline).await.unwrap(); + assert_eq!(environments.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +// Synthetic receipt for private binding/revocation unit regressions only. Actual +// runtime qualification remains covered by the explicit real-daemon suites. +pub(in crate::docker) fn synthetic_resource( + config: &EnvironmentProbeConfig, +) -> super::EnvironmentProbe { + super::EnvironmentProbe { + report: report(AttemptId::default()), + directory: config.directory.clone(), + relay: config.relay.clone(), + policy: config.network.apply(&config.policy).unwrap(), + environments: config.environments.clone(), + } +} diff --git a/crates/rustack-lambda-core/src/docker/pull.rs b/crates/rustack-lambda-core/src/docker/pull.rs new file mode 100644 index 00000000..1fa6807c --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull.rs @@ -0,0 +1,368 @@ +//! Explicit registry policy, secret credentials and retained progress requests. + +pub(super) mod progress; + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + num::NonZeroU16, + str::FromStr, + sync::Arc, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use bytes::Bytes; +use http::{Method, StatusCode}; +use secrecy::{ExposeSecret, SecretString, zeroize::Zeroize}; +use serde_json::to_vec; +use tokio::{sync::watch, time::Instant}; + +use super::{DockerError, DockerPort, Operation, RawReply, RawRequest}; +use crate::artifact::{Architecture, ImageReference}; + +/// Canonical exact registry host/port; debug omits operator infrastructure. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub struct RegistryHost(String); +impl fmt::Debug for RegistryHost { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryHost()") + } +} +impl FromStr for RegistryHost { + type Err = DockerError; + fn from_str(value: &str) -> Result { + if value.is_empty() + || value.len() > 253 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-:[]".contains(&b)) + { + return Err(DockerError::Configuration); + } + let canonical = value.to_ascii_lowercase(); + let canonical = if canonical == "index.docker.io" { + "docker.io".to_owned() + } else { + canonical + }; + let port = if canonical.starts_with('[') { + canonical + .split_once(']') + .and_then(|(_, tail)| tail.strip_prefix(':')) + } else { + canonical.split_once(':').map(|(_, port)| port) + }; + if port.is_some_and(|port| port.parse::().is_err()) { + return Err(DockerError::Configuration); + } + if !canonical.starts_with('[') { + let name = canonical + .split_once(':') + .map_or(canonical.as_str(), |(host, _)| host); + if name.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || !label + .bytes() + .next() + .is_some_and(|b| b.is_ascii_alphanumeric()) + || !label + .bytes() + .last() + .is_some_and(|b| b.is_ascii_alphanumeric()) + }) { + return Err(DockerError::Configuration); + } + } + let reference: ImageReference = format!("{canonical}/a") + .parse() + .map_err(|_| DockerError::Configuration)?; + if reference.registry() != Some(canonical.as_str()) { + return Err(DockerError::Configuration); + } + Ok(Self(canonical)) + } +} +impl TryFrom<&str> for RegistryHost { + type Error = DockerError; + fn try_from(value: &str) -> Result { + value.parse() + } +} +impl RegistryHost { + /// Canonical host, excluding any scheme/path. + /// ``` + /// let host: rustack_lambda_core::docker::RegistryHost="registry.example:5000".parse()?; + /// assert_eq!(host.as_str(),"registry.example:5000"); + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +enum Credential { + Basic(SecretString, SecretString), + Token(SecretString), +} +/// Validated mutually exclusive registry credentials, never serializable publicly. +#[non_exhaustive] +pub struct RegistryCredential(Credential); +impl fmt::Debug for Credential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Credential()") + } +} +impl fmt::Debug for RegistryCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryCredential()") + } +} +fn secret(value: &SecretString, max: usize) -> Result<(), DockerError> { + let value = value.expose_secret(); + if value.is_empty() || value.len() > max || value.chars().any(char::is_control) { + return Err(DockerError::Configuration); + } + Ok(()) +} +impl RegistryCredential { + /// Validate explicit basic credentials without trimming or discovery. + /// # Errors + /// Rejects empty, overlong or control-bearing secrets. + /// ``` + /// let credentials=rustack_lambda_core::docker::RegistryCredential::basic("user".into(),"password".into())?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn basic(username: SecretString, password: SecretString) -> Result { + secret(&username, 256)?; + secret(&password, 4096)?; + Ok(Self(Credential::Basic(username, password))) + } + /// Validate an explicit identity token. + /// # Errors + /// Rejects empty, overlong or control-bearing tokens. + /// ``` + /// let credentials=rustack_lambda_core::docker::RegistryCredential::identity_token("token".into())?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn identity_token(token: SecretString) -> Result { + secret(&token, 8192)?; + Ok(Self(Credential::Token(token))) + } + fn bytes(&self) -> usize { + match &self.0 { + Credential::Basic(user, password) => user + .expose_secret() + .len() + .saturating_add(password.expose_secret().len()), + Credential::Token(token) => token.expose_secret().len(), + } + } + fn header(&self, host: &RegistryHost) -> Result { + let mut fields = BTreeMap::from([("serveraddress", host.as_str())]); + match &self.0 { + Credential::Basic(user, password) => { + fields.insert("username", user.expose_secret()); + fields.insert("password", password.expose_secret()); + } + Credential::Token(token) => { + fields.insert("identitytoken", token.expose_secret()); + } + } + let mut bytes = to_vec(&fields).map_err(|_| DockerError::Schema)?; + let encoded: SecretString = URL_SAFE.encode(&bytes).into(); + bytes.zeroize(); + if encoded.expose_secret().len() > 32 * 1024 { + return Err(DockerError::TooLarge); + } + Ok(encoded) + } +} + +/// Bounded unique registry credential records; YAML/file loading is caller-owned. +#[derive(Default)] +#[non_exhaustive] +pub struct RegistryCredentials(BTreeMap); +impl fmt::Debug for RegistryCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RegistryCredentials()") + } +} +impl TryFrom> for RegistryCredentials { + type Error = DockerError; + fn try_from(records: Vec<(RegistryHost, RegistryCredential)>) -> Result { + if records.len() > 32 { + return Err(DockerError::Configuration); + } + let mut total = 0_usize; + let mut map = BTreeMap::new(); + for (host, credential) in records { + total = total + .checked_add(host.0.len()) + .and_then(|n| n.checked_add(credential.bytes())) + .ok_or(DockerError::TooLarge)?; + if total > 64 * 1024 || map.insert(host, credential).is_some() { + return Err(DockerError::Configuration); + } + } + Ok(Self(map)) + } +} + +#[derive(Debug)] +struct Approved { + hosts: BTreeSet, + credentials: RegistryCredentials, + timeout: Duration, +} +/// Default-never policy for explicitly configured deployment preparation. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct ImagePullPolicy(Option>); +impl ImagePullPolicy { + /// Validate exact approved registries, matching credentials and total budget. + /// # Errors + /// Rejects empty/duplicate/over32 hosts, unapproved credentials and budgets + /// outside1..600 whole seconds. + /// ``` + /// let policy=rustack_lambda_core::docker::ImagePullPolicy::if_missing( + /// vec!["registry.example".parse()?],Default::default(),std::time::Duration::from_secs(300))?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn if_missing( + hosts: Vec, + credentials: RegistryCredentials, + timeout: Duration, + ) -> Result { + if hosts.is_empty() + || hosts.len() > 32 + || timeout.subsec_nanos() != 0 + || !(1..=600).contains(&timeout.as_secs()) + { + return Err(DockerError::Configuration); + } + let len = hosts.len(); + let hosts: BTreeSet<_> = hosts.into_iter().collect(); + if hosts.len() != len || credentials.0.keys().any(|host| !hosts.contains(host)) { + return Err(DockerError::Configuration); + } + Ok(Self(Some(Arc::new(Approved { + hosts, + credentials, + timeout, + })))) + } + fn authorize( + &self, + reference: &ImageReference, + ) -> Result<(&Approved, RegistryHost), DockerError> { + let policy = self + .0 + .as_deref() + .ok_or(DockerError::Status(StatusCode::NOT_FOUND))?; + let host: RegistryHost = reference.registry().unwrap_or("docker.io").parse()?; + if !policy.hosts.contains(&host) { + return Err(DockerError::PullDenied); + } + let repository = reference.repository(); + let local = reference + .registry() + .and_then(|registry| repository.strip_prefix(registry)) + .and_then(|path| path.strip_prefix('/')) + .unwrap_or(repository); + if host.as_str() == "docker.io" && local.starts_with("rustack-internal/") { + return Err(DockerError::PullDenied); + } + Ok((policy, host)) + } + pub(super) async fn pull( + &self, + docker: &DockerPort, + reference: &ImageReference, + architecture: Architecture, + cancel: watch::Receiver<()>, + stop: watch::Receiver, + ) -> Result<(), DockerError> { + let (policy, host) = self.authorize(reference)?; + let auth = policy + .credentials + .0 + .get(&host) + .map(|credential| credential.header(&host)) + .transpose()?; + docker + .pull_image( + reference, + architecture, + Transfer { auth, cancel, stop }, + Instant::now() + policy.timeout, + ) + .await + } +} + +pub(super) struct Transfer { + pub(super) auth: Option, + pub(super) cancel: watch::Receiver<()>, + pub(super) stop: watch::Receiver, +} +impl fmt::Debug for Transfer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Transfer()") + } +} + +impl DockerPort { + async fn pull_image( + &self, + reference: &ImageReference, + architecture: Architecture, + transfer: Transfer, + deadline: Instant, + ) -> Result<(), DockerError> { + if self.raw.binding.is_none() { + return Err(DockerError::DaemonBinding); + } + let tag = reference + .digest() + .or_else(|| { + reference + .as_str() + .strip_prefix(reference.repository()) + .and_then(|suffix| suffix.strip_prefix(':')) + }) + .unwrap_or("latest"); + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("fromImage", reference.repository()) + .append_pair("tag", tag) + .append_pair("platform", architecture.docker_platform()) + .finish(); + let result = self + .raw + .request_owned( + RawRequest { + archive: None, + pull: Some(transfer), + binding: None, + mutation: None, + operation: Operation::Pull, + method: Method::POST, + target: format!("/v{}/images/create?{query}", self.version), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline, + ) + .await?; + match result { + RawReply::Pulled => Ok(()), + _ => Err(DockerError::Protocol), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/pull/progress.rs b/crates/rustack-lambda-core/src/docker/pull/progress.rs new file mode 100644 index 00000000..4b184005 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/progress.rs @@ -0,0 +1,98 @@ +//! Incremental pull metadata; peer messages never escape as diagnostics. + +use std::fmt; + +use rustack_core::json::{JsonLimits, from_slice}; +use serde::{Deserialize, de::IgnoredAny}; + +use crate::docker::DockerError; + +pub(in crate::docker) const MAX_TOTAL: usize = 16 * 1024 * 1024; +const MAX_LINE: usize = 64 * 1024; +const MAX_RECORDS: usize = 65_536; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Record { + error: Option, + error_detail: Option, +} +impl fmt::Debug for Record { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Record()") + } +} +#[derive(Default)] +pub(in crate::docker) struct Progress { + line: Vec, + bytes: usize, + records: usize, +} +impl fmt::Debug for Progress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Progress()") + } +} +impl Progress { + pub(in crate::docker) fn push(&mut self, bytes: &[u8]) -> Result<(), DockerError> { + self.bytes = self + .bytes + .checked_add(bytes.len()) + .ok_or(DockerError::TooLarge)?; + if self.bytes > MAX_TOTAL { + return Err(DockerError::TooLarge); + } + for byte in bytes { + if *byte == b'\n' { + if self.line.last() == Some(&b'\r') { + self.line.pop(); + } + if self.line.is_empty() || self.line.len() > MAX_LINE { + return Err(DockerError::Protocol); + } + if self + .line + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + != Some(b'{') + { + return Err(DockerError::Protocol); + } + let record: Record = + from_slice(&self.line, JsonLimits::new(MAX_LINE, 8, 128, 8192, 256)?)?; + if record.error.is_some() || record.error_detail.is_some() { + return Err(DockerError::PullFailed); + } + self.records = self.records.checked_add(1).ok_or(DockerError::TooLarge)?; + if self.records > MAX_RECORDS { + return Err(DockerError::TooLarge); + } + self.line.clear(); + } else { + if self.line.len() > MAX_LINE || (self.line.len() == MAX_LINE && *byte != b'\r') { + return Err(DockerError::TooLarge); + } + if self.line.len() == self.line.capacity() { + let extra = (MAX_LINE + 1) + .saturating_sub(self.line.len()) + .min(16 * 1024); + self.line + .try_reserve_exact(extra) + .map_err(|_| DockerError::Allocation)?; + } + self.line.push(*byte); + } + } + Ok(()) + } + pub(in crate::docker) fn finish(self) -> Result<(), DockerError> { + if self.records == 0 || !self.line.is_empty() { + return Err(DockerError::Protocol); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/pull/progress/tests.rs b/crates/rustack-lambda-core/src/docker/pull/progress/tests.rs new file mode 100644 index 00000000..7967f984 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/progress/tests.rs @@ -0,0 +1,73 @@ +use rstest::rstest; + +use super::{MAX_LINE, MAX_RECORDS, MAX_TOTAL, Progress}; +use crate::docker::DockerError; + +#[rstest] +#[case(b"{}\n".as_slice())] +#[case(b"{\"status\":\"Pull complete\"}\r\n".as_slice())] +#[case(b"{\"error\":null,\"errorDetail\":null,\"aux\":{\"Digest\":\"data\"}}\n".as_slice())] +fn test_should_consume_every_fragment_boundary(#[case] source: &[u8]) { + for split in 0..=source.len() { + let mut progress = Progress::default(); + progress.push(&source[..split]).unwrap(); + progress.push(&source[split..]).unwrap(); + progress.finish().unwrap(); + } +} +#[rstest] +#[case(b"\n".as_slice())] +#[case(b"[]\n".as_slice())] +#[case(b"[null,null]\n".as_slice())] +#[case(b"null\n".as_slice())] +#[case(b"\"text\"\n".as_slice())] +#[case(b"{bad}\n".as_slice())] +#[case(b"{\"x\":1,\"x\":2}\n".as_slice())] +fn test_should_reject_nonobject_or_malformed_records(#[case] source: &[u8]) { + assert!(Progress::default().push(source).is_err()); +} +#[rstest] +#[case(b"{\"error\":\"SECRET-PEER\"}\n".as_slice())] +#[case(b"{\"errorDetail\":{\"message\":\"SECRET-PEER\"}}\n".as_slice())] +#[case(b"{\"errorDetail\":{}}\n".as_slice())] +fn test_should_reject_in_band_errors_without_peer_text(#[case] source: &[u8]) { + let error = Progress::default().push(source).unwrap_err(); + assert!(matches!(error, DockerError::PullFailed)); + assert!(!format!("{error:?}: {error}").contains("SECRET")); +} +#[test] +fn test_should_require_nonempty_complete_lines_at_eof() { + assert!(Progress::default().finish().is_err()); + let mut progress = Progress::default(); + progress.push(b"{}").unwrap(); + assert!(progress.finish().is_err()); +} +#[test] +fn test_should_bound_line_bytes_records_and_total_progress() { + let mut line = b"{}".to_vec(); + line.resize(MAX_LINE, b' '); + let mut progress = Progress::default(); + progress.push(&line).unwrap(); + progress.push(b"\r\n").unwrap(); + progress.finish().unwrap(); + line.push(b' '); + assert!(matches!( + Progress::default().push(&line), + Err(DockerError::TooLarge) + )); + let source = b"{}\n".repeat(MAX_RECORDS); + let mut progress = Progress::default(); + progress.push(&source).unwrap(); + assert!(matches!(progress.push(b"{}\n"), Err(DockerError::TooLarge))); + assert!(matches!( + Progress::default().push(&vec![b' '; MAX_TOTAL + 1]), + Err(DockerError::TooLarge) + )); +} +#[test] +fn test_should_reject_deep_and_oversized_progress_scalars() { + let long = format!("{{\"status\":\"{}\"}}\n", "x".repeat(8193)); + assert!(Progress::default().push(long.as_bytes()).is_err()); + let deep = format!("{{\"aux\":{}0{}}}\n", "[".repeat(9), "]".repeat(9)); + assert!(Progress::default().push(deep.as_bytes()).is_err()); +} diff --git a/crates/rustack-lambda-core/src/docker/pull/tests.rs b/crates/rustack-lambda-core/src/docker/pull/tests.rs new file mode 100644 index 00000000..4c4793de --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/tests.rs @@ -0,0 +1,183 @@ +use std::time::Duration; + +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use http::StatusCode; +use rstest::rstest; +use secrecy::ExposeSecret; +use serde_json::{Value, from_slice}; + +use super::{ImagePullPolicy, RegistryCredential, RegistryCredentials, RegistryHost}; +use crate::docker::DockerError; + +mod preparation; +mod real; +mod wire; + +#[rstest] +#[case("")] +#[case("https://registry.example")] +#[case("registry.example/path")] +#[case("user@registry.example")] +#[case("*.example")] +#[case("registry")] +#[case("registry.example:0")] +#[case("registry.example:65536")] +#[case("registry.example\n")] +#[case("-registry.example")] +#[case("registry-.example")] +#[case("registry.-example")] +fn test_should_reject_unsafe_ambiguous_registry_hosts(#[case] host: &str) { + assert!(host.parse::().is_err()); +} +#[rstest] +#[case("registry.example")] +#[case("localhost:5000")] +#[case("[::1]:5000")] +fn test_should_validate_and_redact_exact_registry_hosts(#[case] host: &str) { + let parsed: RegistryHost = host.parse().unwrap(); + assert_eq!(parsed.as_str(), host); + assert!(!format!("{parsed:?}").contains(host)); +} +#[test] +fn test_should_canonicalize_dns_case_and_only_documented_default_hub_alias() { + assert_eq!( + "INDEX.DOCKER.IO".parse::().unwrap(), + "docker.io".parse().unwrap() + ); + assert_eq!( + "REGISTRY.example:5000" + .parse::() + .unwrap() + .as_str(), + "registry.example:5000" + ); + assert!("a".repeat(254).parse::().is_err()); +} +#[test] +fn test_should_bound_and_redact_credentials_with_padded_docker_auth_encoding() { + let host = "registry.example".parse().unwrap(); + let credential = + RegistryCredential::basic("PRIVATE-USER".into(), "PRIVATE-PASSWORD".into()).unwrap(); + let header = credential.header(&host).unwrap(); + let json: Value = from_slice(&URL_SAFE.decode(header.expose_secret()).unwrap()).unwrap(); + assert_eq!(json["username"], "PRIVATE-USER"); + assert_eq!(json["password"], "PRIVATE-PASSWORD"); + assert_eq!(json["serveraddress"], "registry.example"); + assert!(!format!("{credential:?} {header:?}").contains("PRIVATE")); + assert!(RegistryCredential::basic("u".repeat(257).into(), "p".into()).is_err()); + assert!(RegistryCredential::basic("u".into(), "p".repeat(4097).into()).is_err()); + assert!(RegistryCredential::identity_token("t".repeat(8193).into()).is_err()); + assert!(RegistryCredential::identity_token("t\n".into()).is_err()); + assert!(RegistryCredential::basic("".into(), "p".into()).is_err()); + let token = RegistryCredential::identity_token("t".repeat(8192).into()).unwrap(); + assert!(token.header(&host).unwrap().expose_secret().len() <= 32 * 1024); +} +fn policy(host: &str) -> ImagePullPolicy { + ImagePullPolicy::if_missing( + vec![host.parse().unwrap()], + RegistryCredentials::default(), + Duration::from_secs(10), + ) + .unwrap() +} +#[test] +fn test_should_require_exact_host_port_allowlist_and_never_pull_internal_aliases() { + let configured = policy("registry.example:5000"); + assert!( + configured + .authorize(&"registry.example:5000/team/fn:v1".parse().unwrap()) + .is_ok() + ); + for source in [ + "registry.example:5001/team/fn:v1", + "registry.example.evil/team/fn:v1", + "evilregistry.example/team/fn:v1", + "worker:v1", + ] { + assert!(matches!( + configured.authorize(&source.parse().unwrap()), + Err(DockerError::PullDenied) + )); + } + let hub = policy("docker.io"); + assert!(hub.authorize(&"worker:v1".parse().unwrap()).is_ok()); + for source in [ + "rustack-internal/lambda-owner/run:pin", + "docker.io/rustack-internal/run:pin", + "index.docker.io/rustack-internal/run:pin", + ] { + assert!(matches!( + hub.authorize(&source.parse().unwrap()), + Err(DockerError::PullDenied) + )); + } + assert!(matches!( + ImagePullPolicy::default().authorize(&"worker:v1".parse().unwrap()), + Err(DockerError::Status(StatusCode::NOT_FOUND)) + )); +} +#[test] +fn test_should_reject_duplicate_unapproved_or_excessive_policy_records_and_budgets() { + let host: RegistryHost = "registry.example".parse().unwrap(); + assert!( + ImagePullPolicy::if_missing( + vec![], + RegistryCredentials::default(), + Duration::from_secs(1) + ) + .is_err() + ); + assert!( + ImagePullPolicy::if_missing( + vec![host.clone(), host.clone()], + RegistryCredentials::default(), + Duration::from_secs(1) + ) + .is_err() + ); + assert!( + ImagePullPolicy::if_missing( + vec![host.clone(); 33], + RegistryCredentials::default(), + Duration::from_secs(1) + ) + .is_err() + ); + for timeout in [ + Duration::ZERO, + Duration::from_millis(1500), + Duration::from_secs(601), + ] { + assert!( + ImagePullPolicy::if_missing( + vec![host.clone()], + RegistryCredentials::default(), + timeout + ) + .is_err() + ); + } + let make = || RegistryCredential::identity_token("secret".into()).unwrap(); + assert!( + RegistryCredentials::try_from(vec![(host.clone(), make()), (host.clone(), make())]) + .is_err() + ); + let map = RegistryCredentials::try_from(vec![(host, make())]).unwrap(); + assert!( + ImagePullPolicy::if_missing( + vec!["other.example".parse().unwrap()], + map, + Duration::from_secs(1) + ) + .is_err() + ); + let records: Vec<(RegistryHost, RegistryCredential)> = (0..9) + .map(|i| { + ( + format!("registry{i}.example").parse().unwrap(), + RegistryCredential::identity_token("x".repeat(8192).into()).unwrap(), + ) + }) + .collect(); + assert!(RegistryCredentials::try_from(records).is_err()); +} diff --git a/crates/rustack-lambda-core/src/docker/pull/tests/preparation.rs b/crates/rustack-lambda-core/src/docker/pull/tests/preparation.rs new file mode 100644 index 00000000..42bafb3b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/tests/preparation.rs @@ -0,0 +1,280 @@ +use std::time::Duration; + +use http::StatusCode; +use tempfile::tempdir; +use tokio::{ + spawn, + time::{Instant, timeout}, +}; + +use super::policy; +use crate::{ + artifact::{Architecture, ArtifactError}, + docker::{ + DirectoryLease, DockerError, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, + lease::tests::{FakePins, Mode}, + tests::Daemon, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} + +#[tokio::test] +async fn test_should_never_pull_when_a_local_image_is_already_valid() { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let fake = FakePins::new(Mode::Normal).await; + let bound = fake + .wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory), + ImageLeaseConfig::default(), + policy("registry.example"), + ) + .unwrap(); + // FakePins only serves inspection/tag/retirement, so an unexpected pull fails. + let image = owner + .port() + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + assert!(image.is_live()); + drop(image); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_pull_only_after_initial_missing_root_and_never_publish_invalid_post_pull_image() + { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + port, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy("registry.example"), + ) + .unwrap(); + let port = owner.port(); + let reference = "registry.example/fn:v1".parse().unwrap(); + let (result, ()) = tokio::join!( + port.prepare(&reference, Architecture::Arm64, None, deadline()), + async { + let initial = daemon.next().await; + assert!(initial.head.contains("/images/")); + initial.reply(404, b"{}").await; + let pull = daemon.next().await; + assert!(pull.head.starts_with("POST /v1.53/images/create?")); + pull.reply(200, b"{\"status\":\"Pull complete\"}\n").await; + let recheck = daemon.next().await; + assert!(recheck.head.starts_with("GET /v1.53/images/")); + recheck.reply(200, b"{\"Id\":\"not-a-digest\"}").await; + } + ); + assert!(result.is_err()); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_not_pull_after_nonmissing_initial_failure_or_unapproved_registry() { + for status in [StatusCode::FORBIDDEN, StatusCode::NOT_FOUND] { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory), + ImageLeaseConfig::default(), + policy("allowed.example"), + ) + .unwrap(); + let port = owner.port(); + let reference = "denied.example/fn:v1".parse().unwrap(); + let (result, ()) = tokio::join!( + port.prepare(&reference, Architecture::Arm64, None, deadline()), + async { + daemon.next().await.reply(status.as_u16(), b"{}").await; + } + ); + assert!(result.is_err()); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; + } +} + +#[tokio::test] +async fn test_should_keep_preparation_owned_until_cancelled_pull_driver_is_joined() { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy("registry.example"), + ) + .unwrap(); + let port = owner.port(); + let task = spawn(async move { + port.prepare( + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + daemon.next().await.reply(404, b"{}").await; + let held = daemon.next().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + drop(held); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_not_treat_platform_metadata_missing_as_initial_root_missing() { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy("registry.example"), + ) + .unwrap(); + let port = owner.port(); + let reference = "registry.example/fn:v1".parse().unwrap(); + let (result, ()) = tokio::join!( + port.prepare(&reference, Architecture::Arm64, None, deadline()), + async { + daemon + .next() + .await + .reply( + 200, + format!("{{\"Id\":\"sha256:{}\"}}", "a".repeat(64)).as_bytes(), + ) + .await; + let selected = daemon.next().await; + assert!(selected.head.starts_with("GET /v1.53/images/sha256")); + assert!(selected.head.contains("platform=")); + selected.reply(404, b"{}").await; + } + ); + assert!(matches!( + result, + Err(DockerError::Image(ArtifactError::Inspection)) + )); + assert!(daemon.requests.try_recv().is_err()); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_cancel_owned_pull_when_preparation_waiter_expires() { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let bound = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let mut owner = ImageLeaseManager::start_with_pull( + bound, + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy("registry.example"), + ) + .unwrap(); + let port = owner.port(); + let task = spawn(async move { + port.prepare( + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + None, + Instant::now() + Duration::from_secs(1), + ) + .await + }); + daemon.next().await.reply(404, b"{}").await; + let held = daemon.next().await; + assert!(matches!( + timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap(), + Err(DockerError::Deadline) + )); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(owner.port().status().pins, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + drop(held); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/pull/tests/real.rs b/crates/rustack-lambda-core/src/docker/pull/tests/real.rs new file mode 100644 index 00000000..1b4de145 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/tests/real.rs @@ -0,0 +1,192 @@ +use std::{collections::BTreeMap, env, fmt, path::Path, time::Duration}; + +use anyhow::{Context, Error, Result, ensure}; +use http::StatusCode; +use rustack_core::settings::ValidatedSettings; +use tempfile::{TempDir, tempdir}; +use tokio::{ + fs, + io::AsyncWriteExt, + sync::watch, + time::{Instant, timeout_at}, +}; + +use crate::{ + artifact::{Architecture, ImageReference}, + docker::{ + DirectoryLease, DockerError, DockerRuntimeSettings, DockerWire, ImageLeaseConfig, + ImageLeaseManager, ImagePinScope, ImagePullPolicy, reconcile_namespace, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(600) +} +fn io_deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +struct Evidence(Option); +impl fmt::Debug for Evidence { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Evidence()") + } +} +impl Evidence { + fn path(&self) -> Result<&Path> { + Ok(self.0.as_ref().context("evidence directory")?.path()) + } +} +impl Drop for Evidence { + fn drop(&mut self) { + if let Some(root) = self.0.take() { + eprintln!("retained pull evidence: {}", root.keep().display()); + } + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker socket, architecture and immutable approved image"] +async fn test_should_pull_approved_immutable_image_prepare_live_pin_and_join_cleanup() -> Result<()> +{ + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let reference: ImageReference = env::var("RUSTACK_PULL_IMAGE") + .context("explicit immutable approved pull image")? + .parse()?; + let mut evidence = Evidence(Some(tempdir()?)); + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, io_deadline()).await?; + let mut images = None; + let result = async { + let directory = DirectoryLease::acquire(evidence.path()?.into(), io_deadline()).await?; + let docker = wire + .port() + .bind_directory(&directory, io_deadline()) + .await?; + ensure!( + reference.digest().is_some() && reference.registry() == Some("public.ecr.aws"), + "fixture requires immutable public.ecr.aws reference" + ); + let cache_miss = match docker.inspect_image(&reference, io_deadline()).await { + Ok(_) => false, + Err(DockerError::Status(StatusCode::NOT_FOUND)) => true, + Err(error) => return Err(error.into()), + }; + let policy = configured_policy(evidence.path()?, &reference).await?; + images = Some(ImageLeaseManager::start_with_pull( + docker.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + policy.clone(), + )?); + let image = images + .as_ref() + .context("image owner")? + .port() + .prepare(&reference, architecture, None, deadline()) + .await?; + ensure!( + image.is_live() && image.unpacked_size().get() > 0, + "prepared image is not live" + ); + let records = directory.image_records(io_deadline()).await?; + ensure!( + records.len() == 1 && records.iter().all(|record| record.committed), + "pin not committed" + ); + let expected = image.resolved().metadata().image_id.clone(); + // Always execute the actual endpoint too, even on a repeated warm run. + let (_cancel, cancel) = watch::channel(()); + let (_stop, stop) = watch::channel(false); + policy + .pull(&docker, &reference, architecture, cancel, stop) + .await?; + ensure!( + docker + .inspect_image(&reference, io_deadline()) + .await? + .id + .as_deref() + == Some(expected.as_str()), + "immutable source changed" + ); + drop(image); + eprintln!( + "approved real pull/preparation cacheMiss={cache_miss}; source cache intentionally \ + retained" + ); + Ok::<(), Error>(()) + } + .await; + let cleanup = match images.as_mut() { + Some(owner) => owner.shutdown(deadline()).await, + None => Ok(()), + }; + if cleanup.is_ok() { + images.take(); + } + let audit = audit_scope(&wire, evidence.path()?).await; + let wire_cleanup = wire.shutdown().await; + result?; + cleanup?; + audit?; + wire_cleanup?; + evidence.0.take(); + Ok(()) +} + +async fn audit_scope(wire: &DockerWire, root: &Path) -> Result<()> { + let directory = DirectoryLease::acquire(root.into(), io_deadline()).await?; + let docker = wire + .port() + .bind_directory(&directory, io_deadline()) + .await?; + let report = reconcile_namespace(&directory, &docker, io_deadline()).await?; + ensure!( + report.containers.remaining == 0 && report.pins.remaining == 0, + "owned resources remain" + ); + Ok(()) +} + +async fn configured_policy(root: &Path, reference: &ImageReference) -> Result { + timeout_at(io_deadline(), async { + let path = fs::canonicalize(root) + .await? + .join("registry-credentials.yaml"); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .await?; + file.write_all(b"{}\n").await?; + drop(file); + let shared = ValidatedSettings::parse( + "budgets:\n lambdaDockerPullSeconds: 600", + &BTreeMap::from([ + ( + "LAMBDA_DOCKER_RELAY_IMAGE".into(), + reference.as_str().to_owned(), + ), + ("LAMBDA_DOCKER_NETWORK".into(), "none".into()), + ("LAMBDA_DOCKER_PULL_POLICY".into(), "ifMissing".into()), + ("LAMBDA_DOCKER_REGISTRIES".into(), "public.ecr.aws".into()), + ( + "LAMBDA_DOCKER_REGISTRY_AUTH_FILE".into(), + path.to_str().context("UTF-8 fixture path")?.to_owned(), + ), + ]), + )?; + let settings = DockerRuntimeSettings::from_lookup( + |key| shared.get(key).map(str::to_owned), + shared.budgets(), + Duration::from_secs(600), + )?; + Ok::<_, Error>(settings.load_pull_policy(io_deadline()).await?) + }) + .await? +} diff --git a/crates/rustack-lambda-core/src/docker/pull/tests/wire.rs b/crates/rustack-lambda-core/src/docker/pull/tests/wire.rs new file mode 100644 index 00000000..0c238918 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pull/tests/wire.rs @@ -0,0 +1,414 @@ +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use http::StatusCode; +use rstest::rstest; +use serde_json::{Value, from_slice}; +use tempfile::{TempDir, tempdir}; +use tokio::{ + spawn, + sync::watch, + time::{Instant, sleep, timeout}, +}; +use tracing::{Level, instrument::WithSubscriber}; + +use super::policy; +use crate::{ + artifact::Architecture, + docker::{ + DirectoryLease, DockerError, DockerPort, DockerWire, ImagePullPolicy, RegistryCredential, + RegistryCredentials, + tests::{Daemon, LogWriter}, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn fixture() -> (TempDir, Daemon, DockerWire, DockerPort) { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + (root, daemon, wire, port) +} + +#[rstest] +#[case("registry.example/team/worker:v1", "v1")] +#[case("registry.example/team/worker", "latest")] +#[case( + "registry.example/team/worker@sha256:\ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +)] +#[tokio::test] +async fn test_should_encode_explicit_tag_digest_platform_and_selected_secret( + #[case] source: &str, + #[case] tag: &str, +) { + let (_root, mut daemon, mut wire, port) = fixture().await; + let host = "registry.example".parse().unwrap(); + let credentials = RegistryCredentials::try_from(vec![( + host, + RegistryCredential::basic("PRIVATE-USER".into(), "PRIVATE-PASSWORD".into()).unwrap(), + )]) + .unwrap(); + let policy = ImagePullPolicy::if_missing( + vec!["registry.example".parse().unwrap()], + credentials, + Duration::from_secs(10), + ) + .unwrap(); + let reference = source.parse().unwrap(); + let (_cancel, cancel) = watch::channel(()); + let (_stop, stop) = watch::channel(false); + let (result, ()) = tokio::join!( + policy.pull(&port, &reference, Architecture::Arm64, cancel, stop), + async { + let request = daemon.next().await; + let target = request + .head + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap(); + let (path, query) = target.split_once('?').unwrap(); + assert_eq!(path, "/v1.53/images/create"); + let query = form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect::>(); + assert_eq!(query.len(), 3); + assert_eq!(query["fromImage"], "registry.example/team/worker"); + assert_eq!(query["tag"], tag); + assert_eq!(query["platform"], "linux/arm64"); + let auth = request + .head + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("x-registry-auth")) + .map(|(_, value)| value.trim()) + }) + .unwrap(); + let credentials: Value = from_slice(&URL_SAFE.decode(auth).unwrap()).unwrap(); + assert_eq!(credentials["username"], "PRIVATE-USER"); + assert_eq!(credentials["password"], "PRIVATE-PASSWORD"); + request + .reply(200, b"{\"status\":\"Pull complete\"}\n") + .await; + } + ); + result.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(200,b"{\"errorDetail\":{\"message\":\"SECRET-PEER\"}}\n".as_slice())] +#[case(200,b"{}".as_slice())] +#[case(200,b"".as_slice())] +#[case(404,b"{\"message\":\"SECRET-PEER\"}".as_slice())] +#[tokio::test] +async fn test_should_reject_failed_empty_or_truncated_streams_without_peer_text( + #[case] status: u16, + #[case] body: &[u8], +) { + let (_root, mut daemon, mut wire, port) = fixture().await; + let policy = policy("registry.example"); + let reference = "registry.example/fn:v1".parse().unwrap(); + let (_cancel, cancel) = watch::channel(()); + let (_stop, stop) = watch::channel(false); + let (result, ()) = tokio::join!( + policy.pull(&port, &reference, Architecture::Arm64, cancel, stop), + async { + daemon.next().await.reply(status, body).await; + } + ); + let error = result.unwrap_err(); + assert!(!format!("{error:?}: {error}").contains("SECRET-PEER")); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_running_and_queued_pulls_without_starving_cleanup() { + let (_root, mut daemon, mut wire, port) = fixture().await; + let policy = policy("registry.example"); + let (cancel, cancelled) = watch::channel(()); + let (_stop, stopped) = watch::channel(false); + let mut tasks = Vec::new(); + let mut held = Vec::new(); + for _ in 0..4 { + let (policy, port, cancel, stop) = ( + policy.clone(), + port.clone(), + cancelled.clone(), + stopped.clone(), + ); + tasks.push(spawn(async move { + policy + .pull( + &port, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancel, + stop, + ) + .await + })); + held.push(daemon.next().await); + } + for _ in 0..8 { + let (policy, port, cancel, stop) = ( + policy.clone(), + port.clone(), + cancelled.clone(), + stopped.clone(), + ); + tasks.push(spawn(async move { + policy + .pull( + &port, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancel, + stop, + ) + .await + })); + } + timeout(Duration::from_secs(3), async { + while port.raw.pulls.capacity() != 0 { + sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + assert!(matches!( + policy + .pull( + &port, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancelled.clone(), + stopped.clone() + ) + .await, + Err(DockerError::Busy) + )); + let key = "a".repeat(64).parse().unwrap(); + let (result, ()) = tokio::join!(port.inspect_container(&key, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains("/containers/")); + request.reply(404, b"{}").await; + }); + assert!(matches!( + result, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + )); + drop(cancel); + drop(held); + for task in tasks { + assert!(task.await.unwrap().is_err()); + } + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_cancel_and_join_an_active_pull_before_terminal_receipt() { + let (root, mut daemon, mut wire, port) = fixture().await; + let policy = policy("registry.example"); + let (cancel, cancelled) = watch::channel(()); + let (_stop, stopped) = watch::channel(false); + let call = port.clone(); + let task = spawn(async move { + policy + .pull( + &call, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancelled, + stopped, + ) + .await + }); + let held = daemon.next().await; + drop(cancel); + assert!(matches!(task.await.unwrap(), Err(DockerError::Cancelled))); + drop(held); + drop(port); + wire.shutdown().await.unwrap(); + drop(wire); + let _next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + daemon.close().await; +} + +#[rstest] +#[case::deadline(false)] +#[case::owner_stop(true)] +#[tokio::test] +async fn test_should_join_an_active_pull_on_deadline_or_owner_stop(#[case] stopping: bool) { + let (root, mut daemon, mut wire, port) = fixture().await; + let policy = ImagePullPolicy::if_missing( + vec!["registry.example".parse().unwrap()], + RegistryCredentials::default(), + Duration::from_secs(1), + ) + .unwrap(); + let (_cancel, cancel) = watch::channel(()); + let (stop, stopped) = watch::channel(false); + let task = spawn(async move { + policy + .pull( + &port, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancel, + stopped, + ) + .await + }); + let held = daemon.next().await; + if stopping { + stop.send_replace(true); + } + let error = timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(if stopping { + matches!(error, DockerError::Stopped) + } else { + matches!(error, DockerError::Deadline) + }); + drop(held); + wire.shutdown().await.unwrap(); + drop(wire); + let _directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + daemon.close().await; +} + +#[rstest] +#[case::abandoned_waiter(false)] +#[case::failed_root(true)] +#[tokio::test] +async fn test_should_retain_accepted_pull_after_waiter_or_root_is_abandoned(#[case] failed: bool) { + let (root, mut daemon, mut wire, port) = fixture().await; + let policy = policy("registry.example"); + let (_cancel, cancel) = watch::channel(()); + let (_stop, stop) = watch::channel(false); + let task = spawn(async move { + policy + .pull( + &port, + &"registry.example/fn:v1".parse().unwrap(), + Architecture::Arm64, + cancel, + stop, + ) + .await + }); + let held = daemon.next().await; + if failed { + wire.owner.task.as_ref().unwrap().abort(); + assert!(task.await.unwrap().is_err()); + } else { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert!( + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .is_err() + ); + } + let result = wire.shutdown().await; + assert!(if failed { + matches!(result, Err(DockerError::Worker)) + } else { + result.is_ok() + }); + drop(held); + drop(wire); + let _directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_filter_pull_auth_target_progress_and_errors_under_trace() { + let capture = LogWriter(Arc::default()); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + async { + tracing::debug!("pull_application_event"); + let (_root, mut daemon, mut wire, port) = fixture().await; + let host = "private-registry.example".parse().unwrap(); + let credential = + RegistryCredential::basic("PRIVATE-USERNAME".into(), "PRIVATE-PASSWORD".into()) + .unwrap(); + let credentials = RegistryCredentials::try_from(vec![(host, credential)]).unwrap(); + let policy = ImagePullPolicy::if_missing( + vec!["private-registry.example".parse().unwrap()], + credentials, + Duration::from_secs(10), + ) + .unwrap(); + let (_cancel, cancel) = watch::channel(()); + let (_stop, stop) = watch::channel(false); + let source = "private-registry.example/private-source:v1" + .parse() + .unwrap(); + let (result, ()) = tokio::join!( + policy.pull(&port, &source, Architecture::Arm64, cancel, stop), + async { + daemon + .next() + .await + .reply( + 200, + b"{\"status\":\"PRIVATE-PROGRESS\"}\n{\"error\":\"PRIVATE-ERROR\"}\n", + ) + .await; + } + ); + assert!(matches!(result, Err(DockerError::PullFailed))); + wire.shutdown().await.unwrap(); + daemon.close().await; + } + .with_subscriber(subscriber) + .await; + let output = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); + assert!(output.contains("pull_application_event")); + for secret in [ + "private-registry.example", + "private-source", + "PRIVATE-USERNAME", + "PRIVATE-PASSWORD", + "PRIVATE-PROGRESS", + "PRIVATE-ERROR", + "X-Registry-Auth", + "x-registry-auth", + ] { + assert!(!output.contains(secret)); + } +} diff --git a/crates/rustack-lambda-core/src/docker/recovery.rs b/crates/rustack-lambda-core/src/docker/recovery.rs new file mode 100644 index 00000000..da939578 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery.rs @@ -0,0 +1,146 @@ +//! Resumable historical container cleanup; unknown absence is never settlement. + +use http::StatusCode; +use tokio::time::Instant; + +pub use self::{ + code::CodeRecovery, + images::{ImagePinRecovery, reconcile_image_pins}, + namespace::{NamespaceRecovery, reconcile_namespace}, +}; +use super::{ + DirectoryLease, DockerError, DockerPort, EnvironmentError, check_deadline, + ownership::ContainerRecord, +}; + +pub(super) mod code; +mod images; +pub(super) mod inventory; +mod namespace; + +/// Container-only recovery accounting, not a complete backend readiness proof. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ContainerRecovery { + /// Records durably erased during this pass. + pub settled: usize, + /// Records still retained, including current-run records never touched here. + pub remaining: usize, +} + +/// Reconcile previous-run recorded containers using exact identity and ID proofs. +/// +/// Unknown404 responses retain the record; unresolved functions retain their +/// relay. Every destructive request follows durable publication of the full ID. +/// Cancellation or errors preserve records for a subsequent pass. This function +/// never creates containers and never touches current-run records. +/// +/// # Errors +/// Returns invalid/expired budgets, unsafe/corrupt/unavailable ownership storage, +/// daemon transport/protocol failures or foreign resource identity. No error +/// grants readiness, and previous successfully settled records remain settled. +/// Requires a port from `DockerPort::bind_directory` for this exact lease. +/// +/// ```no_run +/// # async fn recover(directory: &rustack_lambda_core::docker::DirectoryLease, docker: &rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::EnvironmentError> { +/// let report = rustack_lambda_core::docker::reconcile_containers(directory, docker, +/// tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; +/// assert_eq!(report.remaining, 0); +/// // Image aliases and the other startup proofs still require qualification. +/// # Ok(()) } +/// ``` +pub async fn reconcile_containers( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result { + docker.require_directory(directory)?; + check_deadline(deadline)?; + let mut records = directory.container_records(deadline).await?; + records.sort_by_key(|record| (!record.function(), record.environment)); + let mut report = ContainerRecovery { + settled: 0, + remaining: records.len(), + }; + let mut blocked = Vec::new(); + for record in records { + check_deadline(deadline)?; + if record.run == directory.run_id() { + continue; + } + if !record.function() && blocked.contains(&record.environment) { + continue; + } + if Box::pin(recover(directory, docker, record.clone(), deadline)).await? { + report.settled += 1; + report.remaining -= 1; + } else if record.function() { + blocked.push(record.environment); + } + } + check_deadline(deadline)?; + Ok(report) +} + +async fn recover( + directory: &DirectoryLease, + docker: &DockerPort, + mut record: ContainerRecord, + deadline: Instant, +) -> Result { + let key = record.known_key()?.unwrap_or(record.key()?); + let inspected = match docker.inspect_container(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) if record.known.is_some() => { + directory.settle_container(record, deadline).await?; + return Ok(true); + } + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(false), + result => result?, + }; + let id = record.owned(directory.owner_id(), &inspected)?; + directory + .know_container(record.clone(), id.clone(), deadline) + .await?; + // Re-read the durable identity so subsequent mutation never relies on an + // unpersisted local success or a name which can be reused. + record = directory + .container_records(deadline) + .await? + .into_iter() + .find(|saved| { + saved.environment == record.environment && saved.function() == record.function() + }) + .ok_or(DockerError::Quarantined)?; + let current = match docker.inspect_container(&id, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_container(record, deadline).await?; + return Ok(true); + } + result => result?, + }; + record.owned(directory.owner_id(), ¤t)?; + match current.state.and_then(|state| state.running) { + Some(true) => { + let _ = docker.kill_container(&id, deadline).await; + let exit = docker.wait_container(&id, deadline).await?; + if exit.error.is_some() { + return Err(DockerError::Quarantined.into()); + } + } + Some(false) => {} + None => return Err(DockerError::Protocol.into()), + } + check_deadline(deadline)?; + let _ = docker.remove_container(&id, deadline).await; + match docker.inspect_container(&id, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_container(record, deadline).await?; + Ok(true) + } + Ok(_) => Ok(false), + Err(error) => Err(error.into()), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/recovery/code.rs b/crates/rustack-lambda-core/src/docker/recovery/code.rs new file mode 100644 index 00000000..2c822968 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/code.rs @@ -0,0 +1,224 @@ +//! Historical loader-before-volume retirement; unknown absence retains pins. + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; + +use crate::docker::{ + DirectoryLease, DockerError, DockerPort, Operation, check_deadline, escaped, + ownership::codes::{CodeRecord, LoaderState, VolumeState}, + unit, +}; + +/// Durable code-record accounting; neither resource counts nor readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct CodeRecovery { + /// Records durably erased by this pass after both effects retired. + pub settled: usize, + /// Records still retained, including unresolved issued effects. + pub remaining: usize, +} + +pub(super) async fn reconcile_code( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result { + docker.require_directory(directory)?; + // Namespace recovery calls us only after recorded consumers have joined. + if !directory.container_records(deadline).await?.is_empty() { + return Err(DockerError::Quarantined); + } + let records = directory.code_records(deadline).await?; + if records + .iter() + .any(|record| record.run == directory.run_id()) + { + return Err(DockerError::Quarantined); + } + let mut report = CodeRecovery { + settled: 0, + remaining: records.len(), + }; + for record in records { + check_deadline(deadline)?; + if recover(directory, docker, record, deadline).await? { + report.settled = report.settled.checked_add(1).ok_or(DockerError::Protocol)?; + report.remaining = report + .remaining + .checked_sub(1) + .ok_or(DockerError::Protocol)?; + } + } + check_deadline(deadline)?; + Ok(report) +} +async fn save( + directory: &DirectoryLease, + record: &mut CodeRecord, + next: CodeRecord, + deadline: Instant, +) -> Result<(), DockerError> { + directory + .advance_code(record.clone(), next.clone(), deadline) + .await?; + *record = next; + Ok(()) +} +async fn recover( + directory: &DirectoryLease, + docker: &DockerPort, + mut record: CodeRecord, + deadline: Instant, +) -> Result { + if retire_effects(directory, docker, &mut record, deadline).await? { + directory.settle_code(record, deadline).await?; + Ok(true) + } else { + Ok(false) + } +} +pub(in crate::docker) async fn retire_effects( + directory: &DirectoryLease, + docker: &DockerPort, + record: &mut CodeRecord, + deadline: Instant, +) -> Result { + if record.volume == VolumeState::Planned { + let mut next = record.clone(); + next.volume = VolumeState::Retired; + next.loader = LoaderState::Retired; + save(directory, record, next, deadline).await?; + } + if record.volume == VolumeState::Uncertain { + let inspected = match docker.inspect_volume(&record.volume_key()?, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(false), + result => result?, + }; + record.owned_volume(directory.owner_id(), &inspected)?; + let mut next = record.clone(); + next.volume = VolumeState::Known; + save(directory, record, next, deadline).await?; + } + if !Box::pin(retire_loader(directory, docker, record, deadline)).await? { + return Ok(false); + } + if !retire_volume(directory, docker, record, deadline).await? { + return Ok(false); + } + Ok(true) +} +pub(in crate::docker) async fn retire_loader( + directory: &DirectoryLease, + docker: &DockerPort, + record: &mut CodeRecord, + deadline: Instant, +) -> Result { + match record.loader { + LoaderState::Retired => return Ok(true), + LoaderState::Planned => return loader_retired(directory, record, deadline).await, + LoaderState::Uncertain | LoaderState::Known(_) => {} + } + let inspected = match docker + .inspect_container(&record.loader_key()?, deadline) + .await + { + Err(DockerError::Status(StatusCode::NOT_FOUND)) + if matches!(record.loader, LoaderState::Known(_)) => + { + return loader_retired(directory, record, deadline).await; + } + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(false), + result => result?, + }; + let id = record.owned_loader(directory.owner_id(), &inspected)?; + if record.loader == LoaderState::Uncertain { + let mut next = record.clone(); + next.loader = LoaderState::Known(id); + save(directory, record, next, deadline).await?; + } + let key = record.loader_key()?; + let checked = match docker.inspect_container(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + return loader_retired(directory, record, deadline).await; + } + result => result?, + }; + record.owned_loader(directory.owner_id(), &checked)?; + let _ = async { + unit( + docker + .call( + Operation::Remove, + Method::DELETE, + format!("/containers/{}?force=false", escaped(key.as_str())), + Bytes::new(), + StatusCode::NO_CONTENT, + deadline, + ) + .await?, + deadline, + ) + } + .await; + match docker.inspect_container(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + loader_retired(directory, record, deadline).await + } + Ok(_) => Ok(false), + Err(error) => Err(error), + } +} +async fn loader_retired( + directory: &DirectoryLease, + record: &mut CodeRecord, + deadline: Instant, +) -> Result { + let mut next = record.clone(); + next.loader = LoaderState::Retired; + save(directory, record, next, deadline).await?; + Ok(true) +} +async fn retire_volume( + directory: &DirectoryLease, + docker: &DockerPort, + record: &mut CodeRecord, + deadline: Instant, +) -> Result { + if record.volume == VolumeState::Retired { + return Ok(true); + } + if record.volume != VolumeState::Known || record.loader != LoaderState::Retired { + return Err(DockerError::Quarantined); + } + let key = record.volume_key()?; + let inspected = match docker.inspect_volume(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + return volume_retired(directory, record, deadline).await; + } + result => result?, + }; + record.owned_volume(directory.owner_id(), &inspected)?; + let _ = docker.remove_volume(&key, deadline).await; + match docker.inspect_volume(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + volume_retired(directory, record, deadline).await + } + Ok(_) => Ok(false), + Err(error) => Err(error), + } +} +async fn volume_retired( + directory: &DirectoryLease, + record: &mut CodeRecord, + deadline: Instant, +) -> Result { + let mut next = record.clone(); + next.volume = VolumeState::Retired; + save(directory, record, next, deadline).await?; + Ok(true) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/recovery/code/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/code/tests.rs new file mode 100644 index 00000000..81e11024 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/code/tests.rs @@ -0,0 +1,580 @@ +use std::time::Duration; + +use rstest::rstest; +use serde_json::{Value, json, to_vec}; +use tempfile::TempDir; +use tokio::{ + spawn, + time::{Instant, timeout}, +}; + +use super::*; +use crate::docker::{ + DockerWire, EnvironmentError, ImagePinScope, + ownership::{ + ImageRecord, + codes::tests::{publish, record}, + }, + reconcile_namespace, + tests::{Daemon, IncomingRequest}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn fixture( + volume: VolumeState, + loader: LoaderState, +) -> (TempDir, DirectoryLease, CodeRecord) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let row = record(directory.run_id(), volume, loader); + publish(&directory, vec![row.clone()]).await; + drop(directory); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + (root, directory, row) +} +async fn wire(directory: &DirectoryLease) -> (Daemon, DockerWire, DockerPort) { + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(directory, deadline()) + .await + .unwrap(); + (daemon, wire, port) +} +fn volume_json(directory: &DirectoryLease, row: &CodeRecord) -> Value { + json!({"Name":row.volume_key().unwrap().as_str(),"Driver":"local","Scope":"local","Mountpoint":"/unused","Labels":row.labels(directory.owner_id(),false),"Options":{}}) +} +fn loader_json(directory: &DirectoryLease, row: &CodeRecord) -> Value { + json!({"Id":"cc".repeat(32),"Name":format!("/rustack-code-loader-{}",row.environment),"Image":format!("sha256:{}","aa".repeat(32)),"Config":{"Image":format!("sha256:{}","aa".repeat(32)),"Labels":row.labels(directory.owner_id(),true)},"State":{"Status":"created","Running":false,"StartedAt":"0001-01-01T00:00:00Z"},"Mounts":[{"Type":"volume","Name":row.volume_key().unwrap().as_str(),"Destination":"/var/task","RW":true,"Driver":"local"}]}) +} +async fn reply_json(request: IncomingRequest, value: &Value) { + request.reply(200, &to_vec(value).unwrap()).await; +} +async fn no_more(daemon: &mut Daemon) { + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); +} +async fn audit(daemon: &mut Daemon, directory: &DirectoryLease) { + let request = daemon.next().await; + assert!(request.head.contains("/containers/json?all=true")); + request.reply(200, b"[]").await; + let request = daemon.next().await; + volume_query(&request, directory); + request.reply(200, br#"{"Volumes":[],"Warnings":[]}"#).await; +} +fn volume_query(request: &IncomingRequest, directory: &DirectoryLease) { + let target = request + .head + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap(); + let (path, query) = target.split_once('?').unwrap(); + assert_eq!(path, "/v1.53/volumes"); + let pairs = form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect::>(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].0, "filters"); + assert_eq!( + serde_json::from_str::(&pairs[0].1).unwrap(), + json!({"label":[format!("rustack.lambda.ownerId={}",directory.owner_id())]}) + ); +} + +#[rstest] +#[case(VolumeState::Planned, LoaderState::Planned)] +#[case(VolumeState::Retired, LoaderState::Retired)] +#[tokio::test] +async fn test_should_erase_no_effect_or_retired_records_without_docker_requests( + #[case] volume: VolumeState, + #[case] loader: LoaderState, +) { + let (_root, directory, _) = fixture(volume, loader).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let report = reconcile_code(&directory, &port, deadline()).await.unwrap(); + assert_eq!( + report, + CodeRecovery { + settled: 1, + remaining: 0 + } + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest] +#[case(VolumeState::Uncertain, LoaderState::Planned)] +#[case(VolumeState::Known, LoaderState::Uncertain)] +#[tokio::test] +async fn test_should_retain_unknown_absence_without_volume_or_pin_deletion( + #[case] volume: VolumeState, + #[case] loader: LoaderState, +) { + let (_root, directory, row) = fixture(volume, loader).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + CodeRecovery { + settled: 0, + remaining: 1 + } + ); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + assert!(matches!( + super::super::images::reconcile_image_pins(&directory, &port, deadline()).await, + Err(DockerError::Quarantined) + )); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_erase_known_absent_loader_before_known_absent_volume() { + let (_root, directory, _) = fixture(VolumeState::Known, LoaderState::Known([0xcc; 32])).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains(&"cc".repeat(32))); + request.reply(404, b"{}").await; + let request = daemon.next().await; + assert!(request.head.contains("/volumes/rustack%2Dcode%2D")); + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Retired + ); + request.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + CodeRecovery { + settled: 1, + remaining: 0 + } + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_publish_observed_id_before_delete_and_retire_loader_before_volume() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Uncertain).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let loader = loader_json(&directory, &row); + let volume = volume_json(&directory, &row); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .contains("/containers/rustack%2Dcode%2Dloader%2D") + ); + reply_json(request, &loader).await; + let request = daemon.next().await; + assert!(request.head.contains(&"cc".repeat(32))); + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Known([0xcc; 32]) + ); + reply_json(request, &loader).await; + let request = daemon.next().await; + assert!(request.head.starts_with("DELETE ")); + assert!(request.head.contains("force=false")); + assert!(request.head.contains(&"cc".repeat(32))); + request.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + let request = daemon.next().await; + assert!(request.head.contains("/volumes/rustack%2Dcode%2D")); + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Retired + ); + reply_json(request, &volume).await; + let request = daemon.next().await; + assert!(request.head.starts_with("DELETE ")); + assert!(request.head.contains("force=false")); + assert!(request.head.contains("force=false")); + request.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + CodeRecovery { + settled: 1, + remaining: 0 + } + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_commit_uncertain_volume_before_retiring_planned_loader_and_deleting_volume() { + let (_root, directory, row) = fixture(VolumeState::Uncertain, LoaderState::Planned).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let volume = volume_json(&directory, &row); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &volume).await; + let request = daemon.next().await; + let saved = directory.code_records(deadline()).await.unwrap(); + assert_eq!(saved[0].volume, VolumeState::Known); + assert_eq!(saved[0].loader, LoaderState::Retired); + reply_json(request, &volume).await; + daemon.next().await.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!(report.unwrap().remaining, 0); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest] +#[case("running")] +#[case("started")] +#[case("image")] +#[case("configImage")] +#[case("name")] +#[case("id")] +#[case("knownId")] +#[case("labels")] +#[case("mountName")] +#[case("mountMode")] +#[case("mountType")] +#[case("mountDriver")] +#[case("extraMount")] +#[case("destination")] +#[tokio::test] +async fn test_should_quarantine_conflicting_or_executed_loader_without_delete( + #[case] conflict: &str, +) { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Known([0xcc; 32])).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let mut loader = loader_json(&directory, &row); + match conflict { + "running" => { + loader["State"]["Status"] = json!("running"); + loader["State"]["Running"] = json!(true); + } + "started" => loader["State"]["StartedAt"] = json!("2026-01-01T00:00:00Z"), + "image" => loader["Image"] = json!(format!("sha256:{}", "dd".repeat(32))), + "configImage" => loader["Config"]["Image"] = json!("operator:mutable"), + "name" => loader["Name"] = json!("/foreign"), + "id" => loader["Id"] = json!("CC".repeat(32)), + "knownId" => loader["Id"] = json!("dd".repeat(32)), + "labels" => loader["Config"]["Labels"]["rustack.lambda.unexpected"] = json!("foreign"), + "mountName" => loader["Mounts"][0]["Name"] = json!("foreign"), + "mountMode" => loader["Mounts"][0]["RW"] = json!(false), + "mountType" => loader["Mounts"][0]["Type"] = json!("bind"), + "mountDriver" => loader["Mounts"][0]["Driver"] = json!("foreign"), + "extraMount" => loader["Mounts"] + .as_array_mut() + .unwrap() + .push(json!({"Type":"bind"})), + _ => loader["Mounts"][0]["Destination"] = json!("/wrong"), + } + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &loader).await; + }); + assert!(matches!(report, Err(DockerError::Quarantined))); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest] +#[case("driver")] +#[case("scope")] +#[case("options")] +#[case("labels")] +#[tokio::test] +async fn test_should_quarantine_foreign_volume_without_delete(#[case] conflict: &str) { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Retired).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let mut volume = volume_json(&directory, &row); + match conflict { + "driver" => volume["Driver"] = json!("foreign"), + "scope" => volume["Scope"] = json!("global"), + "options" => volume["Options"] = json!({"device":"/host"}), + _ => volume["Labels"]["unexpected"] = json!("foreign"), + } + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &volume).await; + }); + assert!(matches!(report, Err(DockerError::Quarantined))); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_retain_attached_volume_after_conflicting_delete_until_positive_absence() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Retired).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let volume = volume_json(&directory, &row); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &volume).await; + daemon.next().await.reply(409, b"{}").await; + reply_json(daemon.next().await, &volume).await; + }); + assert_eq!(report.unwrap().remaining, 1); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!(report.unwrap().remaining, 0); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_keep_durable_known_id_after_observer_loss_and_resume_exact_cleanup() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Uncertain).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let loader = loader_json(&directory, &row); + let copy = directory.clone(); + let call = port.clone(); + let task = spawn(async move { reconcile_code(©, &call, deadline()).await }); + reply_json(daemon.next().await, &loader).await; + let held = daemon.next().await; + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Known([0xcc; 32]) + ); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + held.reply(404, b"{}").await; + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains(&"cc".repeat(32))); + request.reply(404, b"{}").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!(report.unwrap().remaining, 0); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_refuse_identity_fence_404_without_settling_resource() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Retired).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + daemon.identity_status.send_replace(404); + assert!(matches!( + reconcile_code(&directory, &port, deadline()).await, + Err(DockerError::IdentityCheck(_)) + )); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_block_namespace_and_pin_retirement_while_code_is_uncertain() { + let (_root, directory, _) = fixture(VolumeState::Known, LoaderState::Uncertain).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + let report = report.unwrap(); + assert_eq!(report.code.remaining, 1); + assert_eq!(report.pins.settled, 0); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_recover_code_before_pin_and_complete_untruncated_volume_audits() { + let root = tempfile::tempdir().unwrap(); + let previous = DirectoryLease::acquire_code(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&previous).await; + publish( + &previous, + vec![record( + previous.run_id(), + VolumeState::Known, + LoaderState::Retired, + )], + ) + .await; + let pin = ImagePinScope::new(previous.clone()).allocate().unwrap(); + let entry = ImageRecord::new( + &pin, + &format!("sha256:{}", "aa".repeat(32)).parse().unwrap(), + ) + .unwrap(); + previous + .prepare_image(entry.clone(), deadline()) + .await + .unwrap(); + previous.commit_image(entry, deadline()).await.unwrap(); + drop(pin); + drop(previous); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains("/volumes/rustack%2Dcode%2D")); + request.reply(404, b"{}").await; + audit(&mut daemon, &directory).await; + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + let request = daemon.next().await; + assert!(request.head.contains("/images/")); + request.reply(404, b"{}").await; + let request = daemon.next().await; + assert!(request.head.contains("/images/json?all=true")); + request.reply(200, b"[]").await; + audit(&mut daemon, &directory).await; + }); + let report = report.unwrap(); + assert_eq!(report.code.settled, 1); + assert_eq!(report.pins.settled, 1); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_refuse_orphan_volume_without_adoption_or_pin_deletion(#[case] late: bool) { + let (_root, directory, row) = fixture(VolumeState::Retired, LoaderState::Retired).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let volume = volume_json(&directory, &row); + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + if late { + audit(&mut daemon, &directory).await; + daemon.next().await.reply(200, b"[]").await; + } + daemon.next().await.reply(200, b"[]").await; + let request = daemon.next().await; + volume_query(&request, &directory); + reply_json(request, &json!({"Volumes":[volume],"Warnings":[]})).await; + }); + assert!(matches!( + report, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + )); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_reject_current_run_code_before_any_namespace_request() { + let (_root, directory, _) = fixture(VolumeState::Retired, LoaderState::Retired).await; + publish( + &directory, + vec![record( + directory.run_id(), + VolumeState::Known, + LoaderState::Retired, + )], + ) + .await; + let (mut daemon, mut wire, port) = wire(&directory).await; + assert!(matches!( + reconcile_namespace(&directory, &port, deadline()).await, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + )); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +mod real; + +#[tokio::test] +async fn test_should_use_nonforce_loader_delete_and_retain_started_race_without_kill() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Known([0xcc; 32])).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let loader = loader_json(&directory, &row); + let mut running = loader.clone(); + running["State"]["Status"] = json!("running"); + running["State"]["Running"] = json!(true); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &loader).await; + reply_json(daemon.next().await, &loader).await; + let request = daemon.next().await; + assert!(request.head.starts_with(&format!( + "DELETE /v1.53/containers/{}?force=false HTTP/", + "cc".repeat(32) + ))); + request.reply(409, b"{}").await; + reply_json(daemon.next().await, &running).await; + }); + assert_eq!(report.unwrap().remaining, 1); + assert_eq!(directory.code_records(deadline()).await.unwrap(), [row]); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &running).await; + }); + assert!(matches!(report, Err(DockerError::Quarantined))); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_recheck_loader_identity_after_durable_publication_before_delete() { + let (_root, directory, row) = fixture(VolumeState::Known, LoaderState::Uncertain).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let loader = loader_json(&directory, &row); + let mut foreign = loader.clone(); + foreign["Id"] = json!("dd".repeat(32)); + let (report, ()) = tokio::join!(reconcile_code(&directory, &port, deadline()), async { + reply_json(daemon.next().await, &loader).await; + let request = daemon.next().await; + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Known([0xcc; 32]) + ); + reply_json(request, &foreign).await; + }); + assert!(matches!(report, Err(DockerError::Quarantined))); + assert_eq!( + directory.code_records(deadline()).await.unwrap()[0].loader, + LoaderState::Known([0xcc; 32]) + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_allow_bounded_inherited_labels_but_reject_unknown_reserved_or_oversized_labels() { + let row = record( + uuid::Uuid::new_v4(), + VolumeState::Known, + LoaderState::Known([0xcc; 32]), + ); + let directory = DirectoryLease::testing(uuid::Uuid::new_v4(), uuid::Uuid::new_v4()); + let mut loader = loader_json(&directory, &row); + loader["Config"]["Labels"]["com.amazonaws.lambda.platform.kernel"] = json!("k510ga"); + let parsed = serde_json::from_value(loader.clone()).unwrap(); + assert!(row.owned_loader(directory.owner_id(), &parsed).is_ok()); + loader["Config"]["Labels"]["com.amazonaws.lambda.platform.kernel"] = json!("x".repeat(257)); + let parsed = serde_json::from_value(loader).unwrap(); + assert!(matches!( + row.owned_loader(directory.owner_id(), &parsed), + Err(DockerError::Quarantined) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/code/tests/real.rs b/crates/rustack-lambda-core/src/docker/recovery/code/tests/real.rs new file mode 100644 index 00000000..98495fe3 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/code/tests/real.rs @@ -0,0 +1,209 @@ +use std::{env, path::Path, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::{ + artifact::{Architecture, ImageReference, ResolvedImage, Sha256Digest}, + docker::{ + DirectoryLease, DockerWire, ImagePinScope, ImageResolver, + models::{ContainerCreateBody, HostConfig, Mount, MountType, MountVolumeOptions}, + ownership::{ + ImageRecord, + codes::{CodeRecord, LoaderState, VolumeState, tests::publish}, + }, + reconcile_namespace, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +fn code_record(run: Uuid, image: &ResolvedImage) -> Result { + let mut base = [0; 32]; + let mut fingerprint = [0; 32]; + hex::decode_to_slice( + image + .metadata() + .image_id + .as_str() + .strip_prefix("sha256:") + .context("base digest")?, + &mut base, + )?; + hex::decode_to_slice( + image + .metadata() + .inspection_hash + .as_str() + .strip_prefix("sha256:") + .context("inspection digest")?, + &mut fingerprint, + )?; + Ok(CodeRecord::builder() + .run(run) + .environment(Uuid::new_v4()) + .resource(Uuid::new_v4()) + .revision(Uuid::new_v4()) + .zip([0x11; 32]) + .base(base) + .fingerprint(fingerprint) + .volume(VolumeState::Uncertain) + .loader(LoaderState::Planned) + .build()) +} +fn loader_body( + row: &CodeRecord, + directory: &DirectoryLease, + parent: &Sha256Digest, +) -> Result { + Ok(ContainerCreateBody { + image: Some(parent.as_str().into()), + user: Some("65534:65534".into()), + entrypoint: Some(vec!["/never-start-this-loader".into()]), + cmd: Some(Vec::new()), + working_dir: Some("/var/task".into()), + labels: Some(row.labels(directory.owner_id(), true).into_iter().collect()), + host_config: Some(HostConfig { + network_mode: Some("none".into()), + readonly_rootfs: Some(true), + memory: Some(128 * 1024 * 1024), + pids_limit: Some(64), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + mounts: Some(vec![Mount { + typ: Some(MountType::VOLUME), + source: Some(row.volume_key()?.as_str().into()), + target: Some("/var/task".into()), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..MountVolumeOptions::default() + }), + ..Mount::default() + }]), + ..HostConfig::default() + }), + ..ContainerCreateBody::default() + }) +} +async fn exercise( + wire: &DockerWire, + root: &Path, + architecture: Architecture, + source: &ImageReference, +) -> Result<()> { + let directory = DirectoryLease::acquire_code(root.into(), deadline()).await?; + let port = wire.port().bind_directory(&directory, deadline()).await?; + let resolved = ImageResolver::new(port.clone()) + .resolve_local(source, architecture, None, deadline()) + .await?; + let parent = resolved.metadata().image_id.clone(); + let pin = ImagePinScope::new(directory.clone()).allocate()?; + let image_record = ImageRecord::new(&pin, &parent)?; + directory + .prepare_image(image_record.clone(), deadline()) + .await?; + port.tag_image_pin(&parent, &pin, deadline()).await?; + directory.commit_image(image_record, deadline()).await?; + let mut row = code_record(directory.run_id(), &resolved)?; + publish(&directory, vec![row.clone()]).await; + let volume_key = row.volume_key()?; + let volume = port + .create_volume( + &volume_key, + &row.labels(directory.owner_id(), false), + deadline(), + ) + .await?; + row.owned_volume(directory.owner_id(), &volume)?; + row.volume = VolumeState::Known; + row.loader = LoaderState::Uncertain; + publish(&directory, vec![row.clone()]).await; + let loader_key = row.loader_key()?; + let body = loader_body(&row, &directory, &parent)?; + let created = port + .create_container(&loader_key, &body, deadline()) + .await?; + let inspected = port + .inspect_container(&created.id.parse()?, deadline()) + .await?; + let id = row.owned_loader(directory.owner_id(), &inspected)?; + row.loader = LoaderState::Known(id); + publish(&directory, vec![row.clone()]).await; + let old_run = directory.run_id(); + let owner = directory.owner_id(); + drop(port); + drop(pin); + drop(directory); + let directory = DirectoryLease::acquire(root.into(), deadline()).await?; + ensure!( + directory.owner_id() == owner && directory.run_id() != old_run, + "immutable owner/fresh run" + ); + let port = wire.port().bind_directory(&directory, deadline()).await?; + let report = reconcile_namespace(&directory, &port, deadline()).await?; + ensure!( + report.code.settled == 1 + && report.code.remaining == 0 + && report.pins.settled == 1 + && report.pins.remaining == 0 + && report.containers.remaining == 0, + "recorded code/pin retirement" + ); + ensure!( + directory.code_records(deadline()).await?.is_empty(), + "code journal remains" + ); + ensure!( + port.inspect_image(source, deadline()).await?.id.as_deref() == Some(parent.as_str()), + "source image changed" + ); + let repeated = reconcile_namespace(&directory, &port, deadline()).await?; + ensure!( + repeated.code.settled == 0 && repeated.code.remaining == 0, + "recovery not idempotent" + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires explicit Docker socket and selected local runtime image"] +async fn test_should_recover_real_code_volume_and_unstarted_loader_across_reacquisition() +-> Result<()> { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let source = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let mut root = tempfile::tempdir()?; + root.disable_cleanup(true); + eprintln!( + "retained code namespace recovery evidence: {}", + root.path().display() + ); + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let result = async { + if let Some(previous) = env::var_os("RUSTACK_CODE_RECOVERY_RESUME") { + let directory = DirectoryLease::acquire(previous.into(), deadline()).await?; + let port = wire.port().bind_directory(&directory, deadline()).await?; + let report = reconcile_namespace(&directory, &port, deadline()).await?; + ensure!( + report.code.remaining == 0 + && report.containers.remaining == 0 + && report.pins.remaining == 0, + "retained evidence still unresolved" + ); + } + exercise(&wire, root.path(), architecture, &source).await + } + .await; + let joined = wire.shutdown().await; + result?; + joined?; + root.disable_cleanup(false); + Ok(()) +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/images.rs b/crates/rustack-lambda-core/src/docker/recovery/images.rs new file mode 100644 index 00000000..ef1bbd55 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/images.rs @@ -0,0 +1,119 @@ +//! Historical alias cleanup only after container ownership has settled. + +use http::StatusCode; +use tokio::time::Instant; + +use super::super::{ + DirectoryLease, DockerError, DockerPort, check_deadline, ownership::ImageRecord, +}; + +/// Pin-only recovery accounting, not full backend readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ImagePinRecovery { + /// Records durably erased during this pass. + pub settled: usize, + /// Records still retained, including current-run pins never touched here. + pub remaining: usize, +} + +/// Retire recorded historical private aliases after historical containers settle. +/// +/// Never replay tag, delete source tags or infer unknown absence as settlement. +/// A positive matching alias is durably committed before removal; an absent +/// committed alias can be erased. Cancellation preserves resumable ownership. +/// +/// # Errors +/// Returns ownership/budget/wire failures, Quarantined for remaining historical +/// containers or mismatching alias identity, and protocol failures. This does +/// not grant readiness or release another owner's live allocations. Requires a +/// port from `DockerPort::bind_directory` for this exact lease; each connection +/// verifies its persisted installation identity before the actual operation. +/// +/// ```no_run +/// # async fn recover(directory: &rustack_lambda_core::docker::DirectoryLease, docker: &rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::DockerError> { +/// let report = rustack_lambda_core::docker::reconcile_image_pins(directory, docker, +/// tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; +/// assert_eq!(report.remaining, 0); +/// # Ok(()) } +/// ``` +pub async fn reconcile_image_pins( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result { + docker.require_directory(directory)?; + if directory + .container_records(deadline) + .await? + .iter() + .any(|record| record.run != directory.run_id()) + { + return Err(DockerError::Quarantined); + } + if !directory.code_records(deadline).await?.is_empty() { + return Err(DockerError::Quarantined); + } + let records = directory.image_records(deadline).await?; + let mut report = ImagePinRecovery { + settled: 0, + remaining: records.len(), + }; + for record in records { + if record.run == directory.run_id() { + continue; + } + if recover(directory, docker, record, deadline).await? { + report.settled = report.settled.checked_add(1).ok_or(DockerError::Protocol)?; + report.remaining = report + .remaining + .checked_sub(1) + .ok_or(DockerError::Protocol)?; + } + } + check_deadline(deadline)?; + Ok(report) +} +async fn recover( + directory: &DirectoryLease, + docker: &DockerPort, + record: ImageRecord, + deadline: Instant, +) -> Result { + let pin = record.pin(directory)?; + let expected = record.expected()?; + let inspected = match docker.inspect_image_pin(&pin, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) if record.committed => { + directory.settle_image(record, deadline).await?; + return Ok(true); + } + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(false), + result => result?, + }; + if inspected.id.as_deref() != Some(expected.as_str()) { + return Err(DockerError::Quarantined); + } + directory.commit_image(record.clone(), deadline).await?; + let checked = match docker.inspect_image_pin(&pin, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_image(record, deadline).await?; + return Ok(true); + } + result => result?, + }; + if checked.id.as_deref() != Some(expected.as_str()) { + return Err(DockerError::Quarantined); + } + let _ = docker.remove_image_pin(&pin, deadline).await; + match docker.inspect_image_pin(&pin, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_image(record, deadline).await?; + Ok(true) + } + Ok(_) => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/recovery/images/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/images/tests.rs new file mode 100644 index 00000000..39a4627d --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/images/tests.rs @@ -0,0 +1,403 @@ +use std::{path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Result as TestResult, ensure}; +use http::StatusCode; +use serde_json::json; +use tokio::time::{Instant, timeout}; + +use super::{ImagePinRecovery, reconcile_image_pins}; +use crate::{ + artifact::Architecture, + docker::{ + DirectoryLease, DockerError, DockerWire, EnvironmentPlan, ImagePin, ImagePinScope, + ImageResolver, + ownership::{ContainerRecord, ImageRecord}, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + tests::Daemon, + }, +}; +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn fixture( + committed: bool, + containers: bool, +) -> (tempfile::TempDir, DirectoryLease, ImageRecord, ImagePin) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let record = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + directory + .prepare_image(record.clone(), deadline()) + .await + .unwrap(); + if committed { + directory + .commit_image(record.clone(), deadline()) + .await + .unwrap(); + } + if containers { + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + directory + .prepare_container( + ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(), + deadline(), + ) + .await + .unwrap(); + } + drop(pin); + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let saved = next.image_records(deadline()).await.unwrap().remove(0); + let pin = saved.pin(&next).unwrap(); + (root, next, saved, pin) +} +async fn matching(daemon: &mut Daemon) { + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&json!({"Id":format!("sha256:{}","a".repeat(64))})).unwrap(), + ) + .await; +} +#[tokio::test] +async fn test_should_retain_unknown_absence_then_commit_and_remove_exact_historical_alias() { + let (_root, directory, record, pin) = fixture(false, false).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (report, ()) = tokio::join!(reconcile_image_pins(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + ImagePinRecovery { + settled: 0, + remaining: 1 + } + ); + assert_eq!( + directory.image_records(deadline()).await.unwrap(), + vec![record] + ); + let (report, ()) = tokio::join!(reconcile_image_pins(&directory, &port, deadline()), async { + matching(&mut daemon).await; + let checked = daemon.next().await; + assert!(directory.image_records(deadline()).await.unwrap()[0].committed); + checked + .reply( + 200, + &serde_json::to_vec(&json!({"Id":format!("sha256:{}","a".repeat(64))})).unwrap(), + ) + .await; + let remove = daemon.next().await; + assert!(remove.head.starts_with("DELETE ")); + assert!( + remove + .head + .contains(&crate::docker::escaped(pin.reference().as_str())) + ); + assert!(remove.head.contains("force=false&noprune=true")); + remove.reply(200, b"[]").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + ImagePinRecovery { + settled: 1, + remaining: 0 + } + ); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_settle_absence_only_after_durable_commitment() { + let (_root, directory, _, _pin) = fixture(true, false).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (report, ()) = tokio::join!(reconcile_image_pins(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + report.unwrap(), + ImagePinRecovery { + settled: 1, + remaining: 0 + } + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_never_delete_a_mismatching_alias(#[case] after_commit: bool) { + let (_root, directory, _, _pin) = fixture(false, false).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (result, ()) = tokio::join!(reconcile_image_pins(&directory, &port, deadline()), async { + if after_commit { + matching(&mut daemon).await; + } + daemon + .next() + .await + .reply( + 200, + &serde_json::to_vec(&json!({"Id":format!("sha256:{}","b".repeat(64))})).unwrap(), + ) + .await; + }); + assert!(matches!(result, Err(DockerError::Quarantined))); + assert_eq!( + directory.image_records(deadline()).await.unwrap()[0].committed, + after_commit + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_block_alias_mutations_while_historical_container_intents_remain() { + let (_root, directory, _, _pin) = fixture(false, true).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + assert!(matches!( + reconcile_image_pins( + &directory, + &wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + deadline() + ) + .await, + Err(DockerError::Quarantined) + )); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_exclude_current_run_aliases() { + let (_root, directory, old, _pin) = fixture(false, false).await; + directory.settle_image(old, deadline()).await.unwrap(); + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + directory + .prepare_image( + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(), + deadline(), + ) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + assert_eq!( + reconcile_image_pins( + &directory, + &wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + deadline() + ) + .await + .unwrap() + .remaining, + 1 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_resume_after_waiter_abandonment_with_persisted_commitment() { + let (_root, directory, _, _pin) = fixture(false, false).await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let retained = directory.clone(); + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let task = + tokio::spawn(async move { reconcile_image_pins(&retained, &port, deadline()).await }); + matching(&mut daemon).await; + let held = daemon.next().await; + assert!(directory.image_records(deadline()).await.unwrap()[0].committed); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + held.reply(200, b"{}").await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (report, ()) = tokio::join!(reconcile_image_pins(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!(report.unwrap().remaining, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[derive(Debug)] +struct RecoveryEvidence(Option); +impl RecoveryEvidence { + fn path(&self) -> TestResult<&Path> { + self.0 + .as_ref() + .map(tempfile::TempDir::path) + .context("missing retained image recovery evidence") + } + fn complete(&mut self) { + drop(self.0.take()); + } +} +impl Drop for RecoveryEvidence { + fn drop(&mut self) { + if let Some(root) = self.0.take() { + eprintln!( + "unsettled image recovery evidence retained at {}", + root.keep().display() + ); + } + } +} + +#[rstest::rstest] +#[case::unknown_visible(false)] +#[case::committed_absent(true)] +#[tokio::test] +#[ignore = "requires explicit Docker socket and matching runtime image"] +async fn test_should_recover_real_private_alias_across_directory_reacquisition( + #[case] committed_absent: bool, +) -> TestResult<()> { + let socket = + std::env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match std::env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown fixture architecture"), + }; + let reference = std::env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let limit = || Instant::now() + Duration::from_secs(60); + let mut evidence = RecoveryEvidence(Some(tempfile::tempdir()?)); + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, limit()).await?; + let mut previous = None; + let mut original = None; + let mut alias = None; + let setup = async { + let directory = DirectoryLease::acquire(evidence.path()?.into(), limit()).await?; + let docker = wire.port().bind_directory(&directory, limit()).await?; + previous = Some(directory.run_id()); + let resolved = ImageResolver::new(docker.clone()) + .resolve_local(&reference, architecture, None, limit()) + .await?; + let expected = resolved.metadata().image_id.clone(); + original = Some(expected.clone()); + let pin = ImagePinScope::new(directory.clone()).allocate()?; + alias = Some(pin.reference().clone()); + let record = ImageRecord::new(&pin, &expected)?; + directory.prepare_image(record.clone(), limit()).await?; + docker.tag_image_pin(&expected, &pin, limit()).await?; + if committed_absent { + directory.commit_image(record, limit()).await?; + docker.remove_image_pin(&pin, limit()).await?; + } + anyhow::Ok(()) + } + .await; + let recovery = async { + let next = DirectoryLease::acquire(evidence.path()?.into(), limit()).await?; + ensure!( + Some(next.run_id()) != previous, + "run identity did not change" + ); + let docker = wire.port().bind_directory(&next, limit()).await?; + let report = reconcile_image_pins(&next, &docker, limit()).await?; + ensure!( + report.settled == 1 && report.remaining == 0, + "historical pin not settled" + ); + ensure!( + next.image_records(limit()).await?.is_empty(), + "durable pin records remain" + ); + let source = docker.inspect_image(&reference, limit()).await?; + ensure!( + source.id.as_deref() + == Some(original.as_ref().context("captured source image")?.as_str()), + "operator source tag changed or disappeared" + ); + ensure!( + matches!( + docker + .inspect_image(alias.as_ref().context("captured private alias")?, limit()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ), + "private alias remains visible" + ); + anyhow::Ok(()) + } + .await; + let wire_cleanup = wire.shutdown().await; + setup?; + recovery?; + wire_cleanup?; + evidence.complete(); + Ok(()) +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/inventory.rs b/crates/rustack-lambda-core/src/docker/recovery/inventory.rs new file mode 100644 index 00000000..e516476a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/inventory.rs @@ -0,0 +1,141 @@ +//! Complete bounded observed inventories, never authority to adopt an effect. + +use std::fmt; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use serde::{Deserialize, Deserializer}; +use tokio::time::Instant; + +use crate::{ + artifact::ImageReference, + docker::{ + DirectoryLease, DockerError, DockerPort, Operation, OwnerFilter, check_deadline, decode, + models::ContainerSummary, + }, +}; + +const MAX_IMAGES: usize = 4096; +const MAX_TAGS: usize = 4096; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImageTags { + #[serde(rename = "RepoTags", deserialize_with = "tags")] + tags: Vec, +} +impl fmt::Debug for ImageTags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ImageTags()") + } +} +fn tags<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Ok(Option::>::deserialize(deserializer)?.unwrap_or_default()) +} + +pub(in crate::docker) async fn containers( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result<(), DockerError> { + docker.require_directory(directory)?; + let filter = OwnerFilter::new(directory.owner_id(), None, None)?; + let rows: Vec = decode( + docker + .call( + Operation::List, + Method::GET, + format!("/containers/json?{}", filter.query(None)?), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + )?; + if rows.len() > 64 { + return Err(DockerError::TooLarge); + } + if !rows.is_empty() { + return Err(DockerError::Quarantined); + } + check_deadline(deadline) +} + +pub(in crate::docker) async fn images( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result<(), DockerError> { + docker.require_directory(directory)?; + let rows: Vec = decode( + docker + .call( + Operation::ListImages, + Method::GET, + "/images/json?all=true".into(), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + )?; + check_tags(rows, directory, deadline) +} +fn check_tags( + rows: Vec, + directory: &DirectoryLease, + deadline: Instant, +) -> Result<(), DockerError> { + if rows.len() > MAX_IMAGES { + return Err(DockerError::TooLarge); + } + let prefix = format!("rustack-internal/lambda-{}/", directory.owner_id().simple()); + let mut count = 0_usize; + for row in rows { + count = count + .checked_add(row.tags.len()) + .ok_or(DockerError::TooLarge)?; + if count > MAX_TAGS { + return Err(DockerError::TooLarge); + } + for tag in row.tags { + check_deadline(deadline)?; + if tag.len() > 1024 { + return Err(DockerError::TooLarge); + } + if tag == ":" { + continue; + } + let familiar = tag + .strip_prefix("docker.io/") + .or_else(|| tag.strip_prefix("index.docker.io/")) + .unwrap_or(&tag); + if familiar.starts_with(&prefix) { + return Err(DockerError::Quarantined); + } + tag.parse::() + .map_err(|_| DockerError::Protocol)?; + } + } + check_deadline(deadline) +} + +#[cfg(test)] +mod tests; + +pub(in crate::docker) async fn volumes( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result<(), DockerError> { + docker.require_directory(directory)?; + if directory.code_journaled() { + let scope = OwnerFilter::new(directory.owner_id(), None, None)?; + if !docker.list_volumes(scope, deadline).await?.is_empty() { + return Err(DockerError::Quarantined); + } + } + check_deadline(deadline) +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/inventory/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/inventory/tests.rs new file mode 100644 index 00000000..f244a70c --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/inventory/tests.rs @@ -0,0 +1,135 @@ +use std::time::Duration; + +use rstest::rstest; +use serde_json::from_str; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ImageTags, MAX_IMAGES, MAX_TAGS, check_tags}; +use crate::docker::{DirectoryLease, DockerError}; + +fn lease() -> DirectoryLease { + DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn row(tag: String) -> ImageTags { + ImageTags { tags: vec![tag] } +} + +#[rstest] +#[case("")] +#[case("docker.io/")] +#[case("index.docker.io/")] +#[test] +fn test_should_refuse_owner_prefix_at_any_depth_or_spelling(#[case] registry: &str) { + let directory = lease(); + let prefix = format!( + "{registry}rustack-internal/lambda-{}/", + directory.owner_id().simple() + ); + for suffix in ["run:pin", "run/deep/path:pin", "not a reference", ""] { + assert!(matches!( + check_tags( + vec![row(format!("{prefix}{suffix}"))], + &directory, + deadline() + ), + Err(DockerError::Quarantined) + )); + } +} + +#[test] +fn test_should_preserve_other_owner_registry_and_lookalike_names() { + let directory = lease(); + let owner = directory.owner_id().simple(); + let tags = vec![ + format!( + "rustack-internal/lambda-{}/run:pin", + Uuid::new_v4().simple() + ), + format!("rustack-internal/lambda-{owner}-other/run:pin"), + format!("registry.example/rustack-internal/lambda-{owner}/run:pin"), + "operator/function:latest".into(), + ":".into(), + ]; + assert!(check_tags(vec![ImageTags { tags }], &directory, deadline()).is_ok()); +} + +#[rstest] +#[case(r#"[{"RepoTags":null}]"#)] +#[case(r#"[{"RepoTags":[]}]"#)] +#[case(r#"[{"RepoTags":[":"]}]"#)] +#[test] +fn test_should_accept_dangling_or_empty_tag_lists(#[case] source: &str) { + let rows = from_str::>(source).unwrap(); + assert!(check_tags(rows, &lease(), deadline()).is_ok()); +} + +#[test] +fn test_should_reject_missing_required_tag_field() { + assert!(from_str::>(r"[{}]").is_err()); +} + +#[rstest] +#[case("https://registry.example/image")] +#[case("bad reference")] +#[case("worker:v1\n")] +#[case("")] +#[test] +fn test_should_reject_malformed_foreign_metadata(#[case] tag: &str) { + assert!(matches!( + check_tags(vec![row(tag.into())], &lease(), deadline()), + Err(DockerError::Protocol) + )); +} + +#[test] +fn test_should_bound_rows_total_tags_and_external_tag_bytes() { + let directory = lease(); + let rows = || { + (0..MAX_IMAGES) + .map(|_| ImageTags { tags: vec![] }) + .collect::>() + }; + assert!(check_tags(rows(), &directory, deadline()).is_ok()); + let mut extra = rows(); + extra.push(ImageTags { tags: vec![] }); + assert!(matches!( + check_tags(extra, &directory, deadline()), + Err(DockerError::TooLarge) + )); + let tags = vec!["worker:v1".into(); MAX_TAGS]; + assert!( + check_tags( + vec![ImageTags { tags: tags.clone() }], + &directory, + deadline() + ) + .is_ok() + ); + assert!(matches!( + check_tags( + vec![ImageTags { tags }, row("worker:v2".into())], + &directory, + deadline() + ), + Err(DockerError::TooLarge) + )); + assert!(matches!( + check_tags(vec![row("a".repeat(1025))], &directory, deadline()), + Err(DockerError::TooLarge) + )); +} + +#[test] +fn test_should_reject_late_classification_and_redact_inventory_debug() { + let rows = vec![row("operator/private:secret".into())]; + assert!(!format!("{rows:?}").contains("secret")); + assert!(matches!( + check_tags(rows, &lease(), Instant::now()), + Err(DockerError::Deadline) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/namespace.rs b/crates/rustack-lambda-core/src/docker/recovery/namespace.rs new file mode 100644 index 00000000..e32dce17 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/namespace.rs @@ -0,0 +1,112 @@ +//! Startup-only composition; inventories cannot settle unproven mutations. + +use std::time::Duration; + +use tokio::time::Instant; + +use super::{ + CodeRecovery, ContainerRecovery, ImagePinRecovery, code::reconcile_code, inventory, + reconcile_containers, reconcile_image_pins, +}; +use crate::docker::{DirectoryLease, DockerError, DockerPort, EnvironmentError, check_deadline}; + +/// Recorded retirement counters; this is not a readiness capability. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct NamespaceRecovery { + /// Historical container records settled/retained by this pass. + pub containers: ContainerRecovery, + /// Historical pin records settled/retained by this pass. + pub pins: ImagePinRecovery, + /// Historical code records settled/retained by this pass. + pub code: CodeRecovery, +} + +/// Drain recorded historical effects and audit the complete observed namespace. +/// +/// Call before starting any runtime producer, under one exclusive coordinator. +/// Unknown effects remain recorded; inventories never authorize adoption or +/// prove absence of unseen pre-journal legacy mutations. Earlier durable cleanup +/// may succeed before a later error; repeating the call resumes that state. +/// +/// # Errors +/// Rejects unbound/foreign ports, current-run work, unrecorded owned resources, +/// inventory bounds, storage/wire/identity failures and expired budgets. +/// +/// ```no_run +/// # async fn recover(directory: &rustack_lambda_core::docker::DirectoryLease, port: &rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::EnvironmentError> { +/// let bound = port.bind_directory(directory, tokio::time::Instant::now() + std::time::Duration::from_secs(10)).await?; +/// let report = rustack_lambda_core::docker::reconcile_namespace(directory, &bound, +/// tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; +/// assert_eq!(report.containers.remaining, 0); +/// assert_eq!(report.pins.remaining, 0); +/// // Startup must still qualify pulls, resources, networking and provenance. +/// # Ok(()) } +/// ``` +pub async fn reconcile_namespace( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result { + docker.require_directory(directory)?; + check_deadline(deadline)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DockerError::Configuration.into()); + } + let containers = directory.container_records(deadline).await?; + let pins = directory.image_records(deadline).await?; + let code = directory.code_records(deadline).await?; + if containers + .iter() + .any(|record| record.run == directory.run_id()) + || pins.iter().any(|record| record.run == directory.run_id()) + || code.iter().any(|record| record.run == directory.run_id()) + { + return Err(DockerError::Quarantined.into()); + } + let containers = reconcile_containers(directory, docker, deadline).await?; + let mut report = NamespaceRecovery { + containers, + code: CodeRecovery { + settled: 0, + remaining: code.len(), + }, + pins: ImagePinRecovery { + settled: 0, + remaining: pins.len(), + }, + }; + if containers.remaining != 0 { + check_deadline(deadline)?; + return Ok(report); + } + report.code = reconcile_code(directory, docker, deadline).await?; + if report.code.remaining != 0 { + check_deadline(deadline)?; + return Ok(report); + } + inventory::containers(directory, docker, deadline).await?; + inventory::volumes(directory, docker, deadline).await?; + report.pins = reconcile_image_pins(directory, docker, deadline).await?; + if report.pins.remaining != 0 { + check_deadline(deadline)?; + return Ok(report); + } + inventory::images(directory, docker, deadline).await?; + inventory::containers(directory, docker, deadline).await?; + inventory::volumes(directory, docker, deadline).await?; + if !directory.container_records(deadline).await?.is_empty() + || !directory.image_records(deadline).await?.is_empty() + || !directory.code_records(deadline).await?.is_empty() + { + return Err(DockerError::Quarantined.into()); + } + check_deadline(deadline)?; + Ok(report) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/recovery/namespace/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests.rs new file mode 100644 index 00000000..c1260dd3 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests.rs @@ -0,0 +1,378 @@ +use std::{sync::Arc, time::Duration}; + +use http::StatusCode; +use rstest::rstest; +use serde_json::{Value, json, to_vec}; +use tempfile::TempDir; +use tokio::{ + spawn, + time::{Instant, timeout}, +}; + +use super::{NamespaceRecovery, reconcile_namespace}; +use crate::docker::{ + DirectoryLease, DockerError, DockerPort, DockerWire, EnvironmentError, EnvironmentPlan, + ImagePinScope, + ownership::{ContainerRecord, ImageRecord}, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + tests::{Daemon, IncomingRequest}, +}; + +mod boundaries; +mod real; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} + +#[derive(Debug, Clone, Copy)] +enum Entry { + None, + Unknown, + Known, +} + +async fn fixture(container: Entry, pin_entry: Entry) -> (TempDir, DirectoryLease) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + if !matches!(container, Entry::None) { + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let record = ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(); + directory + .prepare_container(record.clone(), deadline()) + .await + .unwrap(); + if matches!(container, Entry::Known) { + directory + .know_container(record, "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + } + } + if !matches!(pin_entry, Entry::None) { + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let record = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + directory + .prepare_image(record.clone(), deadline()) + .await + .unwrap(); + if matches!(pin_entry, Entry::Known) { + directory.commit_image(record, deadline()).await.unwrap(); + } + } + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + (root, next) +} + +async fn wire(directory: &DirectoryLease) -> (Daemon, DockerWire, DockerPort) { + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(directory, deadline()) + .await + .unwrap(); + (daemon, wire, port) +} +fn container_query(request: &IncomingRequest, directory: &DirectoryLease) { + let target = request + .head + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap(); + let (path, query) = target.split_once('?').unwrap(); + assert_eq!(path, "/v1.53/containers/json"); + let pairs = form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect::>(); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0], ("all".into(), "true".into())); + assert_eq!( + serde_json::from_str::(&pairs[1].1).unwrap(), + json!({"label":[format!("rustack.lambda.ownerId={}",directory.owner_id())]}) + ); + assert_eq!(pairs[1].0, "filters"); +} +async fn empty_container(daemon: &mut Daemon, directory: &DirectoryLease) { + let request = daemon.next().await; + container_query(&request, directory); + request.reply(200, b"[]").await; +} +async fn image_inventory(daemon: &mut Daemon, body: &[u8]) { + let request = daemon.next().await; + assert!(request.head.starts_with("GET /v1.53/images/json?all=true ")); + request.reply(200, body).await; +} +async fn no_more(daemon: &mut Daemon) { + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn test_should_audit_empty_namespace_repeatedly_without_truncated_queries() { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + for _ in 0..2 { + let (report, ()) = + tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + empty_container(&mut daemon, &directory).await; + image_inventory( + &mut daemon, + br#"[{"RepoTags":null},{"RepoTags":["operator/image:v1"]}]"#, + ) + .await; + empty_container(&mut daemon, &directory).await; + }); + let report = report.unwrap(); + assert_eq!(report.containers.remaining, 0); + assert_eq!(report.pins.remaining, 0); + } + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_settle_recorded_containers_before_pins_and_audit_afterward() { + let (_root, directory) = fixture(Entry::Known, Entry::Known).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains(&"a".repeat(64))); + request.reply(404, b"{}").await; + empty_container(&mut daemon, &directory).await; + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + let request = daemon.next().await; + assert!(request.head.contains("/images/")); + request.reply(404, b"{}").await; + image_inventory(&mut daemon, b"[]").await; + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + empty_container(&mut daemon, &directory).await; + }); + let report = report.unwrap(); + assert_eq!(report.containers.settled, 1); + assert_eq!(report.pins.settled, 1); + assert_eq!(report.containers.remaining, 0); + assert_eq!(report.pins.remaining, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_leave_pins_untouched_while_unknown_containers_remain() { + let (_root, directory) = fixture(Entry::Unknown, Entry::Known).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let before = directory.image_records(deadline()).await.unwrap(); + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + }); + let report = report.unwrap(); + assert_eq!(report.containers.remaining, 1); + assert_eq!(report.pins.remaining, 1); + assert_eq!(report.pins.settled, 0); + assert_eq!(directory.image_records(deadline()).await.unwrap(), before); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_not_use_absent_inventory_to_settle_unknown_pin() { + let (_root, directory) = fixture(Entry::None, Entry::Unknown).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + empty_container(&mut daemon, &directory).await; + daemon.next().await.reply(404, b"{}").await; + }); + let report = report.unwrap(); + assert_eq!(report.pins.remaining, 1); + assert_eq!(report.pins.settled, 0); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(1, false)] +#[case(65, false)] +#[case(1, true)] +#[tokio::test] +async fn test_should_refuse_orphan_or_late_containers_without_adoption( + #[case] count: usize, + #[case] late: bool, +) { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + if late { + empty_container(&mut daemon, &directory).await; + image_inventory(&mut daemon, b"[]").await; + } + let request = daemon.next().await; + container_query(&request, &directory); + request + .reply( + 200, + &to_vec(&vec![json!({"Id":"a".repeat(64)}); count]).unwrap(), + ) + .await; + }); + assert!(matches!( + report, + Err(EnvironmentError::Docker( + DockerError::Quarantined | DockerError::TooLarge + )) + )); + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_refuse_unrecorded_owned_alias_without_image_deletion() { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let tag = format!( + "rustack-internal/lambda-{}/old/deep:pin", + directory.owner_id().simple() + ); + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + empty_container(&mut daemon, &directory).await; + image_inventory(&mut daemon, &to_vec(&json!([{"RepoTags":[tag]}])).unwrap()).await; + }); + assert!(matches!( + report, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + )); + assert_eq!(directory.image_records(deadline()).await.unwrap(), []); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_current_run_or_late_journal_entries(#[case] late: bool) { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let pin = ImagePinScope::new(directory.clone()).allocate().unwrap(); + let record = + ImageRecord::new(&pin, &format!("sha256:{}", "a".repeat(64)).parse().unwrap()).unwrap(); + if !late { + directory + .prepare_image(record.clone(), deadline()) + .await + .unwrap(); + } + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + if late { + empty_container(&mut daemon, &directory).await; + image_inventory(&mut daemon, b"[]").await; + let request = daemon.next().await; + directory.prepare_image(record, deadline()).await.unwrap(); + request.reply(200, b"[]").await; + } + }); + assert!(matches!( + report, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + )); + assert_eq!(directory.image_records(deadline()).await.unwrap().len(), 1); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(false, 404)] +#[case(false, 500)] +#[case(true, 404)] +#[tokio::test] +async fn test_should_preserve_list_status_origin_instead_of_empty_success( + #[case] images: bool, + #[case] status: u16, +) { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + if images { + empty_container(&mut daemon, &directory).await; + } + daemon.next().await.reply(status, b"{}").await; + }); + assert!( + matches!(report, Err(EnvironmentError::Docker(DockerError::Status(actual))) if actual.as_u16()==status) + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_resume_recorded_cleanup_after_abandoned_observer() { + let (_root, directory) = fixture(Entry::Known, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let copy = directory.clone(); + let call = port.clone(); + let task = spawn(async move { reconcile_namespace(©, &call, deadline()).await }); + let held = daemon.next().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + held.reply(404, b"{}").await; + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 1 + ); + let (report, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + daemon.next().await.reply(404, b"{}").await; + empty_container(&mut daemon, &directory).await; + image_inventory(&mut daemon, b"[]").await; + empty_container(&mut daemon, &directory).await; + }); + let NamespaceRecovery { + containers, pins, .. + } = report.unwrap(); + assert_eq!(containers.settled, 1); + assert_eq!(pins.remaining, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_identity_404_without_auditing_or_claiming_absence() { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + daemon.identity_status.send_replace(404); + assert!( + matches!(reconcile_namespace(&directory, &port, deadline()).await, + Err(EnvironmentError::Docker(DockerError::IdentityCheck(source))) if matches!(source.as_ref(),DockerError::Status(StatusCode::NOT_FOUND))) + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/boundaries.rs b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/boundaries.rs new file mode 100644 index 00000000..34e38b6e --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/boundaries.rs @@ -0,0 +1,165 @@ +use std::{sync::Arc, time::Duration}; + +use rstest::rstest; +use serde_json::{json, to_vec}; +use tokio::{io::AsyncWriteExt, time::Instant}; + +use super::{Entry, deadline, empty_container, fixture, no_more, wire}; +use crate::docker::{ + DirectoryLease, DockerError, EnvironmentError, EnvironmentPlan, MAX_BODY_BYTES, + ownership::ContainerRecord, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + reconcile_namespace, +}; + +#[rstest] +#[case::root_shape(0)] +#[case::tag_type(1)] +#[case::duplicate_fields(2)] +#[case::tag_bytes(3)] +#[case::row_members(4)] +#[case::tag_members(5)] +#[case::declared_body(6)] +#[case::exact_body(7)] +#[case::chunked_body(8)] +#[case::missing_tags(9)] +#[tokio::test] +async fn test_should_bound_actual_image_inventory_before_clean_result(#[case] mode: u8) { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let (result, ()) = tokio::join!(reconcile_namespace(&directory, &port, deadline()), async { + empty_container(&mut daemon, &directory).await; + let mut request = daemon.next().await; + assert!(request.head.starts_with("GET /v1.53/images/json?all=true ")); + match mode { + 6 => { + request + .raw( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + MAX_BODY_BYTES + 1 + ) + .as_bytes(), + ) + .await; + } + 8 => { + let mut response = format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: \ + close\r\n\r\n{:x}\r\n", + MAX_BODY_BYTES + 1 + ) + .into_bytes(); + response.extend(vec![b' '; MAX_BODY_BYTES + 1]); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + let _ = request.stream.write_all(&response).await; + } + _ => { + let body = match mode { + 0 => b"{}".to_vec(), + 1 => br#"[{"RepoTags":[42]}]"#.to_vec(), + 2 => br#"[{"RepoTags":[],"RepoTags":[]}]"#.to_vec(), + 3 => to_vec(&json!([{"RepoTags":["a".repeat(1025)]}])).unwrap(), + 4 => to_vec(&vec![json!({}); 4097]).unwrap(), + 5 => to_vec(&json!([{"RepoTags":vec!["worker:v1";4097]}])).unwrap(), + 9 => b"[{}]".to_vec(), + _ => { + let mut bytes = b"[]".to_vec(); + bytes.resize(MAX_BODY_BYTES, b' '); + bytes + } + }; + request.reply(200, &body).await; + if mode == 7 { + empty_container(&mut daemon, &directory).await; + } + } + } + }); + if mode == 7 { + let report = result.unwrap(); + assert_eq!(report.containers.remaining, 0); + assert_eq!(report.pins.remaining, 0); + } else { + assert!(result.is_err()); + } + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_invalid_budget_without_daemon_work(#[case] overlong: bool) { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let limit = if overlong { + Instant::now() + Duration::from_secs(61) + } else { + Instant::now() + }; + assert!(matches!( + reconcile_namespace(&directory, &port, limit).await, + Err(EnvironmentError::Docker( + DockerError::Deadline | DockerError::Configuration + )) + )); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_require_exact_directory_binding_before_reading_or_mutation() { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let other_root = tempfile::tempdir().unwrap(); + let other = DirectoryLease::acquire(other_root.path().into(), deadline()) + .await + .unwrap(); + assert!(matches!( + reconcile_namespace(&other, &port, deadline()).await, + Err(EnvironmentError::Docker(DockerError::DaemonBinding)) + )); + assert!(matches!( + reconcile_namespace(&directory, &wire.port(), deadline()).await, + Err(EnvironmentError::Docker(DockerError::DaemonBinding)) + )); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_refuse_current_run_container_before_any_recovery_io() { + let (_root, directory) = fixture(Entry::None, Entry::None).await; + let (mut daemon, mut wire, port) = wire(&directory).await; + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let record = ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(); + directory + .prepare_container(record.clone(), deadline()) + .await + .unwrap(); + assert!(matches!( + reconcile_namespace(&directory, &port, deadline()).await, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + )); + assert_eq!( + directory.container_records(deadline()).await.unwrap(), + [record] + ); + no_more(&mut daemon).await; + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/real.rs b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/real.rs new file mode 100644 index 00000000..f5fa8904 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/namespace/tests/real.rs @@ -0,0 +1,479 @@ +use std::{collections::BTreeSet, env, fmt, path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Ok as anyhow_ok, Result, ensure}; +use http::StatusCode; +use percent_encoding::percent_decode_str; +use rstest::rstest; +use tempfile::{TempDir, tempdir}; +use tokio::time::Instant; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::{ + artifact::{Architecture, DeploymentArtifact, ImageReference, Sha256Digest}, + docker::{ + DirectoryLease, DockerError, DockerPort, DockerWire, EnvironmentError, EnvironmentPlan, + ImageLeaseConfig, ImageLeaseManager, ImagePin, ImagePinScope, + ownership::{ContainerRecord, ImageRecord}, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + reconcile_namespace, + tests::Daemon, + }, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} + +struct Evidence(Option); +impl fmt::Debug for Evidence { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Evidence()") + } +} +impl Evidence { + fn path(&self) -> Result<&Path> { + Ok(self.0.as_ref().context("retained evidence")?.path()) + } + fn complete(&mut self) { + self.0.take(); + } +} +impl Drop for Evidence { + fn drop(&mut self) { + if let Some(directory) = self.0.take() { + eprintln!( + "retained namespace qualification evidence: {}", + directory.keep().display() + ); + } + } +} + +#[derive(Debug, Clone, Copy)] +enum Mode { + Recorded, + OrphanContainer, + OrphanAlias, +} + +#[derive(TypedBuilder)] +struct Fixture { + wire: DockerWire, + evidence: Evidence, + #[builder(default)] + images: Option, + #[builder(default)] + planned: Vec, + #[builder(default)] + alias: Option<(Uuid, Uuid)>, + #[builder(default)] + previous_run: Option, + #[builder(default)] + source: Option, + #[builder(default)] + parent: Option, +} +impl fmt::Debug for Fixture { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Fixture()") + } +} +impl Fixture { + async fn setup( + &mut self, + architecture: Architecture, + reference: ImageReference, + mode: Mode, + ) -> Result<()> { + let directory = DirectoryLease::acquire(self.evidence.path()?.into(), deadline()).await?; + let docker = self + .wire + .port() + .bind_directory(&directory, deadline()) + .await?; + self.previous_run = Some(directory.run_id()); + self.source = Some(reference.clone()); + self.images = Some(ImageLeaseManager::start( + docker.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + )?); + let image = self + .images + .as_ref() + .context("image owner")? + .port() + .prepare(&reference, architecture, None, deadline()) + .await?; + self.parent = Some(image.resolved().metadata().image_id.clone()); + let mut revision = revision(); + revision.architecture = architecture; + revision.artifact = DeploymentArtifact::Image(image.clone()); + let plan = EnvironmentPlan::new(directory.clone(), image, Arc::new(revision), policy())?; + if !matches!(mode, Mode::OrphanAlias) { + for role in [ContainerRole::Relay, ContainerRole::Function] { + let record = ContainerRecord::from_plan(&plan, role)?; + self.planned.push(record.clone()); + if matches!(mode, Mode::Recorded) { + directory.prepare_container(record, deadline()).await?; + } + let created = match role { + ContainerRole::Relay => plan.create_relay(&docker, deadline()).await?, + ContainerRole::Function => plan.create_function(&docker, deadline()).await?, + }; + if matches!(role, ContainerRole::Relay) { + docker + .start_container(&created.id.parse()?, deadline()) + .await?; + } + } + } + if !matches!(mode, Mode::OrphanContainer) { + let pin = ImagePinScope::new(directory.clone()).allocate()?; + self.alias = Some((pin.run, pin.id)); + let expected = self.parent.as_ref().context("parent image")?; + if matches!(mode, Mode::Recorded) { + directory + .prepare_image(ImageRecord::new(&pin, expected)?, deadline()) + .await?; + } + docker.tag_image_pin(expected, &pin, deadline()).await?; + } + Ok(()) + } + async fn close_images(&mut self) -> Result<()> { + if let Some(images) = &mut self.images { + images.shutdown(deadline()).await?; + } + self.images.take(); + Ok(()) + } + async fn exercise(&self, mode: Mode) -> Result<()> { + let directory = DirectoryLease::acquire(self.evidence.path()?.into(), deadline()).await?; + ensure!( + Some(directory.run_id()) != self.previous_run, + "run identity did not change" + ); + let docker = self + .wire + .port() + .bind_directory(&directory, deadline()) + .await?; + let result = reconcile_namespace(&directory, &docker, deadline()).await; + match mode { + Mode::Recorded => { + let report = result?; + ensure!( + report.containers.settled == 2 + && report.containers.remaining == 0 + && report.pins.settled == 1 + && report.pins.remaining == 0, + "recorded namespace not fully settled" + ); + } + Mode::OrphanContainer => { + ensure!( + matches!( + result, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + ), + "unrecorded container accepted" + ); + for record in &self.planned { + let inspected = docker.inspect_container(&record.key()?, deadline()).await?; + record.owned(directory.owner_id(), &inspected)?; + } + } + Mode::OrphanAlias => { + ensure!( + matches!( + result, + Err(EnvironmentError::Docker(DockerError::Quarantined)) + ), + "unrecorded alias accepted" + ); + let (run, id) = self.alias.context("captured private alias")?; + let pin = ImagePin::historical(directory.clone(), run, id)?; + ensure!( + docker + .inspect_image_pin(&pin, deadline()) + .await? + .id + .as_deref() + == Some(self.parent.as_ref().context("parent")?.as_str()), + "orphan alias was changed" + ); + } + } + Ok(()) + } + async fn cleanup(&self) -> Result<()> { + let directory = DirectoryLease::acquire(self.evidence.path()?.into(), deadline()).await?; + let docker = self + .wire + .port() + .bind_directory(&directory, deadline()) + .await?; + // Fixture-only cleanup uses identities captured before its own creates; + // production namespace recovery never adopts unrecorded effects. + let mut result = Ok(()); + let mut blocked = BTreeSet::new(); + for record in self.planned.iter().rev() { + if !record.function() && blocked.contains(&record.environment) { + continue; + } + let retired = cleanup_container(&directory, &docker, record).await; + if retired.is_err() && record.function() { + blocked.insert(record.environment); + } + preserve_error(&mut result, retired); + } + if let Some((run, id)) = self.alias.filter(|_| result.is_ok()) { + let retired = async { + let pin = ImagePin::historical(directory.clone(), run, id)?; + cleanup_alias( + &directory, + &docker, + &pin, + self.parent.as_ref().context("parent")?, + ) + .await + } + .await; + preserve_error(&mut result, retired); + } + let audit = async { + let report = reconcile_namespace(&directory, &docker, deadline()).await?; + ensure!( + report.containers.remaining == 0 && report.pins.remaining == 0, + "cleanup retains unresolved ownership" + ); + anyhow_ok(()) + } + .await; + preserve_error(&mut result, audit); + let source = async { + let source = docker + .inspect_image(self.source.as_ref().context("source tag")?, deadline()) + .await?; + ensure!( + source.id.as_deref() == Some(self.parent.as_ref().context("parent")?.as_str()), + "operator source tag changed" + ); + anyhow_ok(()) + } + .await; + preserve_error(&mut result, source); + result + } +} +fn preserve_error(result: &mut Result<()>, next: Result<()>) { + if result.is_ok() { + *result = next; + } +} +async fn cleanup_container( + directory: &DirectoryLease, + docker: &DockerPort, + record: &ContainerRecord, +) -> Result<()> { + let inspected = match docker.inspect_container(&record.key()?, deadline()).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(()), + result => result?, + }; + let id = record.owned(directory.owner_id(), &inspected)?; + if directory + .container_records(deadline()) + .await? + .iter() + .any(|saved| { + saved.environment == record.environment && saved.function() == record.function() + }) + { + directory + .know_container(record.clone(), id.clone(), deadline()) + .await?; + } + if inspected.state.and_then(|state| state.running) == Some(true) { + docker.kill_container(&id, deadline()).await?; + ensure!( + docker + .wait_container(&id, deadline()) + .await? + .error + .is_none(), + "container wait failed" + ); + } + docker.remove_container(&id, deadline()).await?; + ensure!( + matches!( + docker.inspect_container(&id, deadline()).await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ), + "fixture container remains" + ); + Ok(()) +} +async fn cleanup_alias( + directory: &DirectoryLease, + docker: &DockerPort, + pin: &ImagePin, + parent: &Sha256Digest, +) -> Result<()> { + match docker.inspect_image_pin(pin, deadline()).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(()), + Ok(image) => ensure!( + image.id.as_deref() == Some(parent.as_str()), + "foreign private alias" + ), + Err(error) => return Err(error.into()), + } + let expected = ImageRecord::new(pin, parent)?; + if directory + .image_records(deadline()) + .await? + .iter() + .any(|saved| { + saved + .pin(directory) + .is_ok_and(|saved| saved.reference() == pin.reference()) + }) + { + directory.commit_image(expected, deadline()).await?; + } + ensure!( + docker + .inspect_image_pin(pin, deadline()) + .await? + .id + .as_deref() + == Some(parent.as_str()), + "alias identity changed" + ); + docker.remove_image_pin(pin, deadline()).await?; + ensure!( + matches!( + docker.inspect_image_pin(pin, deadline()).await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ), + "fixture alias remains" + ); + Ok(()) +} + +#[rstest] +#[case::recorded(Mode::Recorded)] +#[case::orphan_container(Mode::OrphanContainer)] +#[case::orphan_alias(Mode::OrphanAlias)] +#[tokio::test] +#[ignore = "requires explicit Docker socket and matching runtime image"] +async fn test_should_reconcile_real_namespace_and_preserve_unrecorded_observations( + #[case] mode: Mode, +) -> Result<()> { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let source = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let evidence = Evidence(Some(tempdir()?)); + let wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let mut fixture = Fixture::builder().wire(wire).evidence(evidence).build(); + let setup = fixture.setup(architecture, source, mode).await; + let images = fixture.close_images().await; + let exercise = if setup.is_ok() && images.is_ok() { + fixture.exercise(mode).await + } else { + Ok(()) + }; + let cleanup = fixture.cleanup().await; + let wire = fixture.wire.shutdown().await; + setup?; + images?; + exercise?; + cleanup?; + wire?; + fixture.evidence.complete(); + Ok(()) +} + +#[tokio::test] +async fn test_should_attempt_independent_cleanup_and_audit_after_first_failure() { + let root = tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let mut planned = Vec::new(); + for _ in 0..2 { + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + for role in [ContainerRole::Relay, ContainerRole::Function] { + planned.push(ContainerRecord::from_plan(&plan, role).unwrap()); + } + } + let failure = planned[3].key().unwrap().as_str().to_owned(); + let independent = [planned[1].key().unwrap(), planned[0].key().unwrap()]; + drop(directory); + let mut daemon = Daemon::new(); + let wire = daemon.wire().await; + let parent: Sha256Digest = format!("sha256:{}", "a".repeat(64)).parse().unwrap(); + let mut fixture = Fixture::builder() + .wire(wire) + .evidence(Evidence(Some(root))) + .planned(planned) + .source(Some("operator/image:v1".parse().unwrap())) + .parent(Some(parent.clone())) + .build(); + let (result, ()) = tokio::join!(fixture.cleanup(), async { + let request = daemon.next().await; + ensure_fixture_target(&request.head, &failure); + request.reply(500, b"{}").await; + for key in independent { + let request = daemon.next().await; + ensure_fixture_target(&request.head, key.as_str()); + request.reply(404, b"{}").await; + } + for target in [ + "/containers/json?", + "/images/json?all=true", + "/containers/json?", + ] { + let request = daemon.next().await; + assert!(request.head.contains(target)); + request.reply(200, b"[]").await; + } + let request = daemon.next().await; + assert!(request.head.contains("/images/operator%2Fimage%3Av1/json")); + request + .reply(200, format!(r#"{{"Id":"{}"}}"#, parent.as_str()).as_bytes()) + .await; + }); + let cleanup = fixture.wire.shutdown().await; + daemon.close().await; + assert!(matches!( + result.unwrap_err().downcast_ref::(), + Some(DockerError::Status(StatusCode::INTERNAL_SERVER_ERROR)) + )); + cleanup.unwrap(); + fixture.evidence.complete(); +} +fn ensure_fixture_target(head: &str, key: &str) { + assert!( + percent_decode_str(head) + .decode_utf8() + .unwrap() + .contains(&format!("/containers/{key}/json")) + ); +} diff --git a/crates/rustack-lambda-core/src/docker/recovery/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/tests.rs new file mode 100644 index 00000000..22b71276 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/tests.rs @@ -0,0 +1,382 @@ +use std::{sync::Arc, time::Duration}; + +use serde_json::{Value, json}; +use tokio::time::timeout; + +use super::*; +use crate::docker::{ + EnvironmentPlan, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + tests::Daemon, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn fixture() -> ( + tempfile::TempDir, + DirectoryLease, + Vec, + Vec, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + Daemon::bind_fixture(&directory).await; + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let mut records = Vec::new(); + let mut inspected = Vec::new(); + for role in [ContainerRole::Function, ContainerRole::Relay] { + let record = ContainerRecord::from_plan(&plan, role).unwrap(); + directory + .prepare_container(record.clone(), deadline()) + .await + .unwrap(); + let role = if record.function() { + "function" + } else { + "relay" + }; + let id = if record.function() { "a" } else { "b" }.repeat(64); + inspected.push(json!({"Id": id,"Name":format!("/{}",record.key().unwrap().as_str()),"Config":{"Image":plan.image_id(if record.function() {ContainerRole::Function} else {ContainerRole::Relay}),"Labels":{ + "rustack.lambda.ownerId":directory.owner_id().to_string(),"rustack.lambda.runId":directory.run_id().to_string(),"rustack.lambda.environmentId":plan.environment_id().to_string(),"rustack.lambda.functionResourceId":plan.revision().resource_id.to_string(),"rustack.lambda.role":role}},"State":{"Running":false}})); + records.push(record); + } + drop(plan); + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + (root, next, records, inspected) +} +async fn respond(daemon: &mut Daemon, value: &Value) { + daemon + .next() + .await + .reply(200, &serde_json::to_vec(value).unwrap()) + .await; +} +#[tokio::test] +async fn test_should_keep_unknown_absence_and_block_relay_then_recover_late_commit() { + let (_root, directory, records, inspected) = fixture().await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let recover = reconcile_containers(&directory, &port, deadline()); + let serve = async { + let request = daemon.next().await; + assert!(request.head.contains("function")); + request.reply(404, b"{}").await; + }; + let (result, ()) = tokio::join!(recover, serve); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 0, + remaining: 2 + } + ); + assert_eq!( + directory.container_records(deadline()).await.unwrap(), + records + ); + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let recover = reconcile_containers(&directory, &port, deadline()); + let serve = async { + for value in &inspected { + respond(&mut daemon, value).await; + // The full ID must already be durable before any destructive call. + let request = daemon.next().await; + let saved = directory.container_records(deadline()).await.unwrap(); + assert!(saved.iter().any(|record| record.known.is_some())); + assert!(request.head.contains(value["Id"].as_str().unwrap())); + request + .reply(200, &serde_json::to_vec(value).unwrap()) + .await; + let remove = daemon.next().await; + assert!(remove.head.starts_with("DELETE ")); + remove.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + } + }; + let (result, ()) = tokio::join!(recover, serve); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 2, + remaining: 0 + } + ); + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_settle_only_durable_known_absence() { + let (_root, directory, records, _) = fixture().await; + for record in &records { + directory + .know_container(record.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + } + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (result, ()) = tokio::join!(reconcile_containers(&directory, &port, deadline()), async { + for _ in 0..2 { + let request = daemon.next().await; + assert!(request.head.contains(&"a".repeat(64))); + request.reply(404, b"{}").await; + } + }); + assert_eq!(result.unwrap().remaining, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case("/Name")] +#[case("/Id")] +#[case("/Config/Image")] +#[case("/Config/Labels/rustack.lambda.ownerId")] +#[case("/Config/Labels/rustack.lambda.runId")] +#[case("/Config/Labels/rustack.lambda.environmentId")] +#[case("/Config/Labels/rustack.lambda.functionResourceId")] +#[case("/Config/Labels/rustack.lambda.role")] +#[tokio::test] +async fn test_should_never_mutate_foreign_or_malformed_resources(#[case] field: &str) { + let (_root, directory, _, mut inspected) = fixture().await; + *inspected[0].pointer_mut(field).unwrap() = json!("foreign"); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (result, ()) = tokio::join!( + reconcile_containers(&directory, &port, deadline()), + respond(&mut daemon, &inspected[0]) + ); + assert!(result.is_err()); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 2 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_never_reconcile_current_run_records() { + let (_root, directory, records, _) = fixture().await; + for record in records { + directory + .settle_container(record, deadline()) + .await + .unwrap(); + } + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + directory + .prepare_container( + ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(), + deadline(), + ) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + assert_eq!( + reconcile_containers( + &directory, + &wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(), + deadline() + ) + .await + .unwrap() + .remaining, + 1 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +#[ignore = "requires explicit Docker socket and matching runtime image"] +async fn test_should_reconcile_real_previous_run_containers_from_durable_intents() +-> anyhow::Result<()> { + use anyhow::{Context, ensure}; + + use crate::{ + artifact::{Architecture, DeploymentArtifact}, + docker::{DockerWire, ImageLeaseConfig, ImageLeaseManager, ImagePinScope}, + }; + let socket = + std::env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match std::env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown fixture architecture"), + }; + let image = std::env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let limit = || Instant::now() + Duration::from_secs(60); + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, limit()).await?; + let root = tempfile::tempdir()?; + let directory = DirectoryLease::acquire(root.path().into(), limit()).await?; + let docker = wire.port().bind_directory(&directory, limit()).await?; + let previous_run = directory.run_id(); + let mut images = ImageLeaseManager::start( + docker.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + )?; + let setup = async { + let image = images + .port() + .prepare(&image, architecture, None, limit()) + .await?; + let mut revision = revision(); + revision.architecture = architecture; + revision.artifact = DeploymentArtifact::Image(image.clone()); + let plan = EnvironmentPlan::new(directory.clone(), image, Arc::new(revision), policy())?; + for role in [ContainerRole::Relay, ContainerRole::Function] { + directory + .prepare_container(ContainerRecord::from_plan(&plan, role)?, limit()) + .await?; + let created = match role { + ContainerRole::Relay => plan.create_relay(&docker.clone(), limit()).await?, + ContainerRole::Function => plan.create_function(&docker.clone(), limit()).await?, + }; + if matches!(role, ContainerRole::Relay) { + docker + .start_container(&created.id.parse()?, limit()) + .await?; + } + } + anyhow::Ok(()) + } + .await; + let image_cleanup = images.shutdown(limit()).await; + drop(images); + drop(docker); + drop(directory); + let recovery = async { + let next = DirectoryLease::acquire(root.path().into(), limit()).await?; + ensure!(next.run_id() != previous_run, "run identity did not change"); + let docker = wire.port().bind_directory(&next, limit()).await?; + let report = reconcile_containers(&next, &docker, limit()).await?; + ensure!( + next.container_records(limit()).await?.is_empty(), + "durable records remain" + ); + anyhow::Ok(report) + } + .await; + if !recovery.as_ref().is_ok_and(|report| report.remaining == 0) + || image_cleanup.is_err() + || setup.is_err() + { + // Any post-intent failure retains the evidence, including failed lease + // reacquisition while an image recovery owner still holds exclusion. + let _ = root.keep(); + } + let wire_cleanup = wire.shutdown().await; + setup?; + image_cleanup?; + let report = recovery?; + ensure!( + report.settled == 2 && report.remaining == 0, + "historical containers were not both settled" + ); + wire_cleanup?; + Ok(()) +} + +#[tokio::test] +async fn test_should_resume_known_identity_after_recovery_waiter_is_aborted() { + let (_root, directory, _, inspected) = fixture().await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory.clone(); + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let task = tokio::spawn(async move { reconcile_containers(&lease, &port, deadline()).await }); + respond(&mut daemon, &inspected[0]).await; + let held = daemon.next().await; + assert!(held.head.contains(&"a".repeat(64))); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + held.reply(200, &serde_json::to_vec(&inspected[0]).unwrap()) + .await; + let saved = directory.container_records(deadline()).await.unwrap(); + assert!( + saved + .iter() + .any(|record| record.function() && record.known.is_some()) + ); + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let (result, ()) = tokio::join!(reconcile_containers(&directory, &port, deadline()), async { + let known = daemon.next().await; + assert!(known.head.contains(&"a".repeat(64))); + known.reply(404, b"{}").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 1, + remaining: 1 + } + ); + assert!(!directory.container_records(deadline()).await.unwrap()[0].function()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/sdk.rs b/crates/rustack-lambda-core/src/docker/sdk.rs new file mode 100644 index 00000000..cdbf31d1 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/sdk.rs @@ -0,0 +1,273 @@ +//! Explicit local handler credentials, qualified affiliation and bounded environment. +//! +//! Only private startup qualification issues contexts. Context identity separates +//! warm plans; the retained proof and exact scope, not that identity, authorize use. + +use std::{ + collections::BTreeMap, + fmt, + num::NonZeroU64, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, +}; + +use secrecy::{ExposeSecret, SecretString}; + +use super::{ + ContainerPolicy, DirectoryLease, DockerError, DockerExecutorConfig, DockerGatewayEndpoint, + DockerReadiness, EnvironmentPort, +}; +use crate::artifact::PinnedImage; + +const MAX_AUGMENTATION_BYTES: usize = 4096; +const MAX_ENV_ENTRIES: usize = 320; +const MAX_ENV_BYTES: usize = 80 * 1024; +const MAX_ENV_ENTRY_BYTES: usize = 4097; +const UNSET: [&str; 16] = [ + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_SDK_LOAD_CONFIG", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_ACCESS_KEY", + "AWS_SECRET_KEY", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_ROLE_ARN", + "AWS_ROLE_SESSION_NAME", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_EC2_METADATA_SERVICE_ENDPOINT", + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", +]; +static NEXT_CONTEXT: AtomicU64 = AtomicU64::new(1); + +#[derive(Clone)] +pub(super) struct LocalSdkCredentials { + access: SecretString, + secret: SecretString, +} +impl fmt::Debug for LocalSdkCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LocalSdkCredentials()") + } +} +impl TryFrom<(String, String)> for LocalSdkCredentials { + type Error = DockerError; + fn try_from((access, secret): (String, String)) -> Result { + if !(1..=128).contains(&access.len()) + || !access + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)) + || !(1..=256).contains(&secret.len()) + || !secret.bytes().all(|b| (33..=126).contains(&b)) + { + return Err(DockerError::Configuration); + } + Ok(Self { + access: access.into(), + secret: secret.into(), + }) + } +} +impl LocalSdkCredentials { + pub(super) fn parse( + access: Option, + secret: Option, + ) -> Result, DockerError> { + match (access, secret) { + (None, None) => Ok(None), + (Some(access), Some(secret)) => (access, secret).try_into().map(Some), + _ => Err(DockerError::Configuration), + } + } +} + +#[derive(Debug, Clone)] +pub(super) struct SdkAffiliation { + directory: DirectoryLease, + relay: PinnedImage, + policy: ContainerPolicy, + environments: EnvironmentPort, + readiness: Arc, +} +impl SdkAffiliation { + pub(super) fn new( + directory: DirectoryLease, + relay: PinnedImage, + policy: ContainerPolicy, + environments: EnvironmentPort, + readiness: Arc, + ) -> Self { + Self { + directory, + relay, + policy, + environments, + readiness, + } + } + pub(super) fn live(&self) -> Result<(), DockerError> { + self.readiness.check().map_err(|_| DockerError::Stopped) + } + fn matches(&self, config: &DockerExecutorConfig) -> bool { + let expected = self.relay.resolved().metadata(); + let actual = config.relay.resolved().metadata(); + Arc::ptr_eq(&self.readiness, &config.readiness) + && self.directory.same_instance(&config.directory) + && self.environments.same_service(&config.environments) + && self.policy == config.policy + && expected.image_id == actual.image_id + && expected.inspection_hash == actual.inspection_hash + && self.relay.is_live() + && config.relay.is_live() + && self.readiness.check().is_ok() + } +} + +/// Opaque qualified local SDK environment; credentials never enter saved revisions. +/// +/// Only the private startup issuer can construct this context from actual proof. +/// Clones retain the same nonsecret warm identity and live qualification holder. +/// +/// ```compile_fail +/// let _ = serde_json::from_str::("{}"); +/// ``` +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerSdkEnvironment { + credentials: LocalSdkCredentials, + endpoint: DockerGatewayEndpoint, + affiliation: SdkAffiliation, + identity: NonZeroU64, +} +impl fmt::Debug for DockerSdkEnvironment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DockerSdkEnvironment()") + } +} +impl DockerSdkEnvironment { + pub(super) fn new( + credentials: LocalSdkCredentials, + endpoint: DockerGatewayEndpoint, + affiliation: SdkAffiliation, + ) -> Result { + affiliation + .readiness + .check() + .map_err(|_| DockerError::Stopped)?; + Ok(Self { + credentials, + endpoint, + affiliation, + identity: next_identity(&NEXT_CONTEXT)?, + }) + } + pub(super) fn matches(&self, config: &DockerExecutorConfig) -> bool { + self.affiliation.matches(config) + } + pub(super) fn liveness(&self) -> SdkAffiliation { + self.affiliation.clone() + } + pub(super) fn identity(&self) -> NonZeroU64 { + self.identity + } + pub(super) fn environment(&self, entries: Vec) -> Result, DockerError> { + environment(&self.credentials, &self.endpoint, entries) + } +} + +fn next_identity(counter: &AtomicU64) -> Result { + let value = counter + .try_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + NonZeroU64::new(value).and_then(|_| value.checked_add(1)) + }) + .map_err(|_| DockerError::Configuration)?; + NonZeroU64::new(value).ok_or(DockerError::Configuration) +} + +fn environment( + credentials: &LocalSdkCredentials, + endpoint: &DockerGatewayEndpoint, + entries: Vec, +) -> Result, DockerError> { + if entries.len() > MAX_ENV_ENTRIES { + return Err(DockerError::Configuration); + } + let mut values = BTreeMap::new(); + let mut input_bytes = 0usize; + for entry in entries { + input_bytes = add_bytes(input_bytes, entry.len(), MAX_ENV_BYTES)?; + if entry.len() > MAX_ENV_ENTRY_BYTES || entry.contains('\0') { + return Err(DockerError::Configuration); + } + let (key, value) = entry + .split_once('=') + .map_or((entry.as_str(), None), |(k, v)| (k, Some(v.to_owned()))); + values.insert(key.to_owned(), value); + } + for (key, value) in &mut values { + if key.starts_with("AWS_ENDPOINT_URL_") || UNSET.contains(&key.as_str()) { + *value = None; + } + } + let fixed = [ + ("AWS_ACCESS_KEY_ID", credentials.access.expose_secret()), + ("AWS_SECRET_ACCESS_KEY", credentials.secret.expose_secret()), + ("AWS_ENDPOINT_URL", endpoint.as_str()), + ("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false"), + ("AWS_EC2_METADATA_DISABLED", "true"), + ("AWS_CONFIG_FILE", "/dev/null"), + ("AWS_SHARED_CREDENTIALS_FILE", "/dev/null"), + ]; + let mut augmentation = 0usize; + for key in UNSET { + augmentation = add_bytes(augmentation, key.len(), MAX_AUGMENTATION_BYTES)?; + values.insert(key.to_owned(), None); + } + for (key, value) in fixed { + let length = key + .len() + .checked_add(1) + .and_then(|n| n.checked_add(value.len())) + .ok_or(DockerError::Configuration)?; + augmentation = add_bytes(augmentation, length, MAX_AUGMENTATION_BYTES)?; + values.insert(key.to_owned(), Some(value.to_owned())); + } + if values.len() > MAX_ENV_ENTRIES { + return Err(DockerError::Configuration); + } + let mut output = Vec::with_capacity(values.len()); + let mut bytes = 0usize; + for (key, value) in values { + let length = match &value { + Some(value) => key + .len() + .checked_add(1) + .and_then(|n| n.checked_add(value.len())) + .ok_or(DockerError::Configuration)?, + None => key.len(), + }; + if length > MAX_ENV_ENTRY_BYTES { + return Err(DockerError::Configuration); + } + bytes = add_bytes(bytes, length, MAX_ENV_BYTES)?; + output.push(match value { + Some(value) => format!("{key}={value}"), + None => key, + }); + } + Ok(output) +} +fn add_bytes(current: usize, extra: usize, maximum: usize) -> Result { + current + .checked_add(extra) + .filter(|value| *value <= maximum) + .ok_or(DockerError::Configuration) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/sdk/tests.rs b/crates/rustack-lambda-core/src/docker/sdk/tests.rs new file mode 100644 index 00000000..33108c7a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/sdk/tests.rs @@ -0,0 +1,117 @@ +use rstest::rstest; + +use super::*; + +fn credentials() -> LocalSdkCredentials { + ("local-sdk-key".to_owned(), "private-sdk-value".to_owned()) + .try_into() + .unwrap() +} +fn endpoint() -> DockerGatewayEndpoint { + "http://127.0.0.1:4566".parse().unwrap() +} +#[rstest] +#[case("", "value")] +#[case("key", "")] +#[case("bad/key", "value")] +#[case("bad key", "value")] +#[case("é", "value")] +#[case("key", "bad value")] +#[case("key", "bad\nvalue")] +#[case("key", "bad\0value")] +#[case("key", "é")] +fn test_should_reject_invalid_sdk_credentials(#[case] access: &str, #[case] secret: &str) { + assert!(LocalSdkCredentials::try_from((access.into(), secret.into())).is_err()); +} +#[test] +fn test_should_bound_pair_and_never_fabricate_a_partial_sdk_pair() { + assert!(LocalSdkCredentials::parse(None, None).unwrap().is_none()); + assert!(LocalSdkCredentials::parse(Some("key".into()), None).is_err()); + assert!(LocalSdkCredentials::parse(None, Some("secret".into())).is_err()); + for (access, secret, accepted) in [ + ("k".repeat(128), "s".repeat(256), true), + ("k".repeat(129), "s".into(), false), + ("k".into(), "s".repeat(257), false), + ] { + assert_eq!( + LocalSdkCredentials::try_from((access, secret)).is_ok(), + accepted + ); + } + let value = credentials(); + assert_eq!(format!("{value:?}"), "LocalSdkCredentials()"); +} +#[test] +fn test_should_allocate_checked_nonzero_context_ids_without_reuse_or_wrap() { + let counter = AtomicU64::new(1); + assert_eq!(next_identity(&counter).unwrap().get(), 1); + assert_eq!(next_identity(&counter).unwrap().get(), 2); + for invalid in [0, u64::MAX] { + let counter = AtomicU64::new(invalid); + assert!(next_identity(&counter).is_err()); + assert_eq!(counter.load(Ordering::Relaxed), invalid); + } +} +#[test] +fn test_should_override_sdk_values_and_retain_all_explicit_unset_markers() { + let mut entries = vec![ + "BUSINESS=preserved".into(), + "AWS_ACCESS_KEY_ID=poison".into(), + "AWS_SECRET_ACCESS_KEY=poison".into(), + "AWS_ENDPOINT_URL=http://127.0.0.1:9".into(), + "AWS_ENDPOINT_URL_S3=http://127.0.0.1:9".into(), + "AWS_ENDPOINT_URL_DYNAMODB=http://127.0.0.1:9".into(), + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=true".into(), + ]; + entries.extend(UNSET.map(|key| format!("{key}=poison"))); + let result = environment(&credentials(), &endpoint(), entries).unwrap(); + for key in UNSET + .into_iter() + .chain(["AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL_DYNAMODB"]) + { + assert_eq!( + result + .iter() + .filter(|entry| entry.split('=').next() == Some(key)) + .collect::>(), + vec![key] + ); + } + for expected in [ + "BUSINESS=preserved", + "AWS_ACCESS_KEY_ID=local-sdk-key", + "AWS_SECRET_ACCESS_KEY=private-sdk-value", + "AWS_ENDPOINT_URL=http://127.0.0.1:4566/", + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=false", + "AWS_EC2_METADATA_DISABLED=true", + "AWS_CONFIG_FILE=/dev/null", + "AWS_SHARED_CREDENTIALS_FILE=/dev/null", + ] { + assert!(result.iter().any(|entry| entry == expected), "{expected}"); + } + assert!(result.windows(2).all(|pair| pair.first() < pair.get(1))); + assert!(result.iter().all(|entry| !entry.contains("poison"))); +} +#[test] +fn test_should_preserve_maximum_valid_user_entry_and_bound_final_sdk_environment() { + let maximum = format!("ZZ={}", "v".repeat(4094)); + let result = environment(&credentials(), &endpoint(), vec![maximum.clone()]).unwrap(); + assert!(result.contains(&maximum)); + for entries in [ + vec!["x".repeat(MAX_ENV_ENTRY_BYTES + 1)], + (0..21) + .map(|i| format!("KEY_{i}={}", "v".repeat(4090))) + .collect(), + (0..298).map(|i| format!("KEY_{i}=v")).collect(), + vec!["AA=bad\0value".into()], + ] { + assert!(environment(&credentials(), &endpoint(), entries).is_err()); + } + let result = environment( + &credentials(), + &endpoint(), + (0..297).map(|i| format!("KEY_{i}=v")).collect(), + ) + .unwrap(); + assert_eq!(result.len(), MAX_ENV_ENTRIES); +} diff --git a/crates/rustack-lambda-core/src/docker/startup.rs b/crates/rustack-lambda-core/src/docker/startup.rs new file mode 100644 index 00000000..fac34b2a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/startup.rs @@ -0,0 +1,499 @@ +//! Actual startup receipts and one ordered cleanup owner, never request-supplied readiness. + +mod qualify; +use std::{ + fmt, + path::PathBuf, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use serde::Serialize; +use thiserror::Error; +use tokio::{ + runtime::Handle, + sync::{Semaphore, SemaphorePermit}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + CodeVolumeManager, DirectoryLease, DirectoryLeaseError, DockerCpuAllocation, DockerError, + DockerExecutor, DockerExecutorError, DockerPort, DockerReadiness, DockerRuntimeSettings, + DockerSdkEnvironment, DockerWire, DockerZipRuntime, EndpointProbe, EndpointProbeConfig, + EnvironmentCapacity, EnvironmentError, EnvironmentManager, EnvironmentProbe, + EnvironmentProbeConfig, EnvironmentProbeError, GatewayInstanceId, ImageLeaseManager, + ImageLeasePort, RegistryAuthError, + ownership::{DataDirectory, DirectoryAcquisition}, + sdk::{LocalSdkCredentials, SdkAffiliation}, +}; +use crate::{ + artifact::{Architecture, PinnedImage}, + config::LambdaConfig, + deployment::ImageDeploymentConfig, + error::LambdaServiceError, + executor::{Executor, ExecutorBackend, ExecutorError}, + invocation::CleanupTask, + provider::RustackLambda, + storage::FunctionStore, +}; + +static STARTUPS: Semaphore = Semaphore::const_new(4); + +/// Static startup errors retain typed sources without displaying operator values. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum DockerStartupError { + /// Invalid or mismatching operator inputs. + #[error("invalid Docker startup configuration")] + Configuration, + /// This attempt was already begun, failed, abandoned or stopped. + #[error("Docker startup attempt is unavailable")] + Unavailable, + /// Global startup or cleanup observation expired. + #[error("Docker startup observation deadline exceeded")] + Deadline, + /// Current runtime or bounded startup admission is unavailable. + #[error("Docker startup supervision unavailable")] + Worker, + /// Unsafe, legacy, busy or damaged namespace. + #[error("Docker startup ownership failed")] + Directory(#[source] Arc), + /// Bounded daemon or image failure. + #[error("Docker startup daemon failed: {0}")] + Docker(#[from] DockerError), + /// The selected relay could not be prepared. + #[error("Docker startup relay preparation failed: {0}")] + Relay(#[source] DockerError), + /// The configured endpoint failed its actual container-origin proof. + #[error("Docker startup endpoint qualification failed: {0}")] + Endpoint(#[source] EnvironmentProbeError), + /// Actual resource qualification failed. + #[error("Docker startup probe failed: {0}")] + Probe(#[from] EnvironmentProbeError), + /// Retained runtime allocation failed. + #[error("Docker startup environment failed: {0}")] + Environment(#[from] EnvironmentError), + /// Private credential loading failed. + #[error("Docker startup credentials failed: {0}")] + Credentials(#[from] RegistryAuthError), + /// Executor/provider construction failed. + #[error("Docker startup installation failed")] + Installation(#[source] Arc), + /// Runtime graph has not confirmed physical cleanup. + #[error("Docker runtime cleanup is not confirmed")] + Cleanup(#[source] ExecutorError), +} +impl From for DockerStartupError { + fn from(error: DirectoryLeaseError) -> Self { + Self::Directory(Arc::new(error)) + } +} +impl From for DockerStartupError { + fn from(error: DockerExecutorError) -> Self { + Self::Cleanup(error.into()) + } +} + +/// Validated immutable production settings, including the exact provider configuration. +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerStartupConfig { + settings: DockerRuntimeSettings, + directory: PathBuf, + instance: GatewayInstanceId, + lambda: LambdaConfig, +} +impl fmt::Debug for DockerStartupConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DockerStartupConfig()") + } +} +impl DockerStartupConfig { + /// Validate one selected image runtime before any listener or filesystem IO. + /// # Errors + /// Rejects non-Docker settings, invalid directory, init/idle/warm limits or a + /// configured endpoint paired with disconnected networking. + /// ```no_run + /// # fn config(settings:rustack_lambda_core::docker::DockerRuntimeSettings,lambda:rustack_lambda_core::config::LambdaConfig,id:rustack_lambda_core::docker::GatewayInstanceId)->Result<(),rustack_lambda_core::docker::DockerStartupError>{ + /// let config=rustack_lambda_core::docker::DockerStartupConfig::new(settings,"/private/data".into(),id,lambda)?; + /// # Ok(()) } + /// ``` + pub fn new( + settings: DockerRuntimeSettings, + directory: PathBuf, + instance: GatewayInstanceId, + lambda: LambdaConfig, + ) -> Result { + DataDirectory::try_from(directory.clone())?; + if settings.zip_image().is_some() && settings.budgets().lambda_docker_image_pins < 2 { + return Err(DockerStartupError::Configuration); + } + + if lambda.executor != ExecutorBackend::Docker + || lambda.max_warm_instances != 1 + || lambda.init_timeout.subsec_nanos() != 0 + || !(1..=900).contains(&lambda.init_timeout.as_secs()) + || lambda.idle_timeout != settings.container_policy().idle() + || (settings.gateway_endpoint().is_some() + && settings.container_policy().network.value() == "none") + { + return Err(DockerStartupError::Configuration); + } + Ok(Self { + settings, + directory, + instance, + lambda, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Phase { + New, + Qualifying, + Ready, + Failed, + Stopping, +} + +#[derive(Debug, TypedBuilder)] +struct Evidence { + base: EnvironmentProbeConfig, + resource: EnvironmentProbe, + endpoint: Option<(EndpointProbeConfig, EndpointProbe)>, + images: ImageLeasePort, + docker: DockerPort, + architecture: Architecture, + #[builder(default)] + zip: Option, +} +#[derive(Debug, Default)] +struct Qualification { + installed: AtomicBool, + revoked: AtomicBool, + evidence: OnceLock, +} +impl Qualification { + fn sdk_environment( + self: &Arc, + credentials: LocalSdkCredentials, + ) -> Result { + self.check().map_err(|_| DockerStartupError::Unavailable)?; + let evidence = self.evidence.get().ok_or(DockerStartupError::Unavailable)?; + let (endpoint, _) = evidence + .endpoint + .as_ref() + .ok_or(DockerStartupError::Configuration)?; + let readiness: Arc = self.clone(); + Ok(DockerSdkEnvironment::new( + credentials, + endpoint.gateway_endpoint().clone(), + SdkAffiliation::new( + evidence.base.directory.clone(), + evidence.base.relay.clone(), + evidence.base.policy.clone(), + evidence.base.environments.clone(), + readiness, + ), + )?) + } +} +impl DockerReadiness for Qualification { + fn check(&self) -> Result<(), DockerExecutorError> { + if self.revoked.load(Ordering::Acquire) { + return Err(DockerExecutorError::Unavailable); + } + let evidence = self + .evidence + .get() + .ok_or(DockerExecutorError::Unavailable)?; + let environment = evidence.base.environments.status(); + if evidence.base.directory.require_journaled().is_err() + || evidence + .docker + .require_directory(&evidence.base.directory) + .is_err() + || *evidence.docker.raw.stop.borrow() + || evidence.images.available().is_err() + || evidence + .zip + .as_ref() + .is_some_and(|zip| zip.available().is_err()) + || !environment.accepting + || environment.worker_failed + || !evidence.resource.matches(&evidence.base) + || evidence + .endpoint + .as_ref() + .is_some_and(|(config, proof)| !proof.matches(config)) + { + return Err(DockerExecutorError::Unavailable); + } + Ok(()) + } +} + +/// Read-only live backend diagnostics; cloning never transfers shutdown authority. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct DockerRuntimeStatus(Arc); +/// Bounded capability data, separate from private startup receipts. +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct DockerCapabilities { + /// Requested/effective CPU pair from retained kernel-qualified policy. + #[serde(skip_serializing_if = "Option::is_none")] + pub cpu: Option, + /// Actual retained allocation/memory charges, absent before qualification. + #[serde(skip_serializing_if = "Option::is_none")] + pub capacity: Option, + /// Actual installed owner health and receipt validity. + pub ready: bool, + /// Qualified Linux platform, absent before successful qualification. + pub platform: Option<&'static str>, + /// Whether a configured gateway passed the actual endpoint proof. + pub sdk_gateway: bool, + /// Qualified provided.al2023 execution through the installed code owner. + pub zip: bool, +} +impl DockerRuntimeStatus { + /// Observe current receipt/owner health without IO or peer data. + /// ```no_run + /// # fn status(status:&rustack_lambda_core::docker::DockerRuntimeStatus){let capabilities=status.capabilities();} + /// ``` + #[must_use] + pub fn capabilities(&self) -> DockerCapabilities { + let evidence = self.0.evidence.get(); + DockerCapabilities { + cpu: evidence.map(|evidence| evidence.base.policy.cpu_allocation()), + capacity: evidence.map(|evidence| evidence.base.environments.status()), + ready: self.0.installed.load(Ordering::Acquire) && self.0.check().is_ok(), + platform: evidence.map(|e| match e.architecture { + Architecture::Arm64 => "linux/arm64", + Architecture::X86_64 => "linux/amd64", + }), + sdk_gateway: evidence.is_some_and(|e| e.endpoint.is_some()), + zip: self.0.installed.load(Ordering::Acquire) + && self.0.check().is_ok() + && evidence.is_some_and(|e| e.zip.is_some()), + } + } +} + +/// Qualified provider installation, issued only by the private startup owner. +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerInstallation { + lambda: LambdaConfig, + executor: Arc, + images: ImageLeasePort, + deployment: ImageDeploymentConfig, +} +impl fmt::Debug for DockerInstallation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DockerInstallation()") + } +} +impl DockerInstallation { + /// Install the exact validated provider over its real executor and image owner. + /// # Errors + /// Rejects revoked startup evidence or failed deployment construction. + /// ```no_run + /// # fn install(installation:&rustack_lambda_core::docker::DockerInstallation,store:rustack_lambda_core::storage::FunctionStore)->Result<(),rustack_lambda_core::docker::DockerStartupError>{ + /// let provider=installation.provider(store)?; + /// # Ok(()) } + /// ``` + pub fn provider(&self, store: FunctionStore) -> Result { + self.executor + .image_available() + .map_err(DockerStartupError::Cleanup)?; + RustackLambda::with_executor(store, self.lambda.clone(), self.executor.clone()) + .with_image_preparation(self.images.clone(), self.deployment) + .map_err(|error| DockerStartupError::Installation(Arc::new(error))) + } +} + +#[derive(Debug, TypedBuilder)] +struct State { + config: DockerStartupConfig, + _admission: SemaphorePermit<'static>, + #[builder(default)] + directory: Option, + #[builder(default)] + acquiring: Option, + #[builder(default)] + connecting: Option>>, + #[builder(default)] + fault: Option, + #[builder(default)] + wire: Option, + #[builder(default)] + bound: Option, + #[builder(default)] + producing: bool, + #[builder(default)] + images: Option, + #[builder(default)] + code: Option, + #[builder(default)] + zip: Option, + #[builder(default)] + environments: Option, + #[builder(default)] + relay: Option, + #[builder(default)] + executor: Option>, +} +/// Unique startup/shutdown authority for the exact production runtime graph. +#[derive(Debug)] +#[non_exhaustive] +pub struct DockerRuntime { + state: Option, + phase: Phase, + qualification: Arc, + runtime: Handle, + cleanup: CleanupTask, +} +impl DockerRuntime { + /// Reserve a bounded startup owner without IO; qualification is one attempt. + /// # Errors + /// Rejects absent runtime or four already-retained runtime owners. + /// ```no_run + /// # fn begin(config:rustack_lambda_core::docker::DockerStartupConfig)->Result<(),rustack_lambda_core::docker::DockerStartupError>{ + /// let owner=rustack_lambda_core::docker::DockerRuntime::new(config)?; + /// # Ok(()) } + /// ``` + pub fn new(config: DockerStartupConfig) -> Result { + let runtime = Handle::try_current().map_err(|_| DockerStartupError::Worker)?; + let admission = STARTUPS + .try_acquire() + .map_err(|_| DockerStartupError::Worker)?; + Ok(Self { + state: Some( + State::builder() + .config(config) + ._admission(admission) + .build(), + ), + phase: Phase::New, + qualification: Arc::new(Qualification::default()), + runtime, + cleanup: CleanupTask::default(), + }) + } + /// Consume actual receipts once under the supplied absolute process deadline. + /// # Errors + /// Returns typed qualification failure. Dropping this observer cannot replay + /// the attempt; retain this owner for shutdown, or Drop starts that same cleanup. + /// ```no_run + /// # async fn qualify(owner:&mut rustack_lambda_core::docker::DockerRuntime){ + /// let result=owner.qualify(tokio::time::Instant::now()+std::time::Duration::from_secs(600)).await; + /// # } + /// ``` + pub async fn qualify(&mut self, deadline: Instant) -> Result<(), DockerStartupError> { + if self.phase != Phase::New { + return Err(DockerStartupError::Unavailable); + } + qualify::deadline(deadline)?; + self.phase = Phase::Qualifying; + let state = self.state.as_mut().ok_or(DockerStartupError::Unavailable)?; + let result = Box::pin(state.qualify(&self.qualification, deadline)).await; + self.phase = if result.is_ok() { + self.qualification.installed.store(true, Ordering::Release); + Phase::Ready + } else { + self.qualification.revoked.store(true, Ordering::Release); + Phase::Failed + }; + result + } + /// Obtain a qualified installation without granting alternate configuration. + /// # Errors + /// Rejects incomplete, failed, abandoned, stopped or revoked qualification. + /// ```no_run + /// # fn installation(owner:&rustack_lambda_core::docker::DockerRuntime){let installation=owner.installation();} + /// ``` + pub fn installation(&self) -> Result { + if self.phase != Phase::Ready { + return Err(DockerStartupError::Unavailable); + } + self.qualification + .check() + .map_err(|_| DockerStartupError::Unavailable)?; + let state = self.state.as_ref().ok_or(DockerStartupError::Unavailable)?; + let budgets = state.config.settings.budgets(); + Ok(DockerInstallation { + lambda: state.config.lambda.clone(), + executor: state + .executor + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .clone(), + images: state + .images + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .port(), + deployment: ImageDeploymentConfig::new( + budgets + .lambda_docker_deployment_jobs + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + Duration::from_secs(budgets.lambda_docker_deployment_seconds), + ) + .map_err(|error| DockerStartupError::Installation(Arc::new(error)))?, + }) + } + /// Retain live read-only capability observations, without startup authority. + /// ```no_run + /// # fn status(owner:&rustack_lambda_core::docker::DockerRuntime){let status=owner.status();} + /// ``` + #[must_use] + pub fn status(&self) -> DockerRuntimeStatus { + DockerRuntimeStatus(self.qualification.clone()) + } + fn start_cleanup(&mut self) { + self.phase = Phase::Stopping; + self.qualification.revoked.store(true, Ordering::Release); + if let Some(state) = self.state.take() { + self.cleanup + .start_on(&self.runtime, move || state.cleanup()); + } + } + /// Revoke admission and observe the same retained ordered cleanup operation. + /// # Errors + /// Returns expiry or failed cleanup observation without releasing unresolved + /// ownership. A later call resumes observation; Drop keeps that operation alive. + /// ```no_run + /// # async fn stop(owner:&mut rustack_lambda_core::docker::DockerRuntime){let result=owner.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await;} + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), DockerStartupError> { + self.start_cleanup(); + if Instant::now() >= deadline { + return Err(DockerStartupError::Deadline); + } + timeout_at(deadline, self.cleanup.wait()) + .await + .map_err(|_| DockerStartupError::Deadline)? + .map_err(DockerStartupError::Cleanup)?; + if Instant::now() >= deadline { + return Err(DockerStartupError::Deadline); + } + Ok(()) + } +} +impl Drop for DockerRuntime { + fn drop(&mut self) { + self.start_cleanup(); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/startup/qualify.rs b/crates/rustack-lambda-core/src/docker/startup/qualify.rs new file mode 100644 index 00000000..e96c4253 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/startup/qualify.rs @@ -0,0 +1,496 @@ +//! Dependency-ordered qualification and physical reverse-order retirement. + +use std::{env::consts::ARCH, sync::Arc, time::Duration}; + +use tokio::time::{Instant, sleep, timeout_at}; + +use super::{DockerStartupError, Evidence, Qualification, State}; +use crate::{ + artifact::{Architecture, PinnedImage}, + docker::{ + CodeVolumeConfig, CodeVolumeManager, DirectoryLease, DockerError, DockerExecutor, + DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, DockerPort, + DockerReadiness, DockerWire, DockerZipRuntime, EndpointProbeConfig, EnvironmentManager, + EnvironmentProbeConfig, EnvironmentSettings, ImageLeaseConfig, ImageLeaseManager, + ImageLeasePort, ImagePinScope, PoolSettings, ProbeSettings, + code_volume::validate_base, + ownership::DirectoryAcquisition, + probe_endpoint, probe_environment, reconcile_namespace, + recovery::inventory::{containers, images, volumes}, + }, + executor::{Executor, ExecutorError}, +}; + +pub(super) fn deadline(at: Instant) -> Result<(), DockerStartupError> { + let remaining = at + .checked_duration_since(Instant::now()) + .ok_or(DockerStartupError::Deadline)?; + if remaining.is_zero() { + return Err(DockerStartupError::Deadline); + } + if remaining > Duration::from_secs(3600) { + return Err(DockerStartupError::Configuration); + } + Ok(()) +} +fn step(at: Instant, seconds: u64) -> Instant { + at.min(Instant::now() + Duration::from_secs(seconds)) +} + +#[derive(Debug)] +struct PreparedImages { + directory: DirectoryLease, + docker: DockerPort, + architecture: Architecture, + images: ImageLeasePort, + relay: PinnedImage, +} + +impl State { + async fn acquire_directory( + &mut self, + at: Instant, + ) -> Result { + self.acquiring = Some(if self.config.settings.zip_image().is_some() { + DirectoryAcquisition::begin_code(self.config.directory.clone(), step(at, 60))? + } else { + DirectoryAcquisition::begin(self.config.directory.clone(), step(at, 60))? + }); + let acquired = self + .acquiring + .as_mut() + .ok_or(DockerStartupError::Unavailable)? + .wait(at) + .await?; + self.acquiring.take(); + self.directory = Some(acquired?); + let directory = self + .directory + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .clone(); + directory.require_journaled()?; + Ok(directory) + } + async fn prepare_images(&mut self, at: Instant) -> Result { + let settings = self.config.settings.clone(); + let budgets = settings.budgets(); + let directory = self.acquire_directory(at).await?; + self.connecting = Some(tokio::spawn(DockerWire::connect( + settings.socket().clone(), + step(at, budgets.lambda_docker_io_seconds), + ))); + let connected = timeout_at( + at, + self.connecting + .as_mut() + .ok_or(DockerStartupError::Unavailable)?, + ) + .await + .map_err(|_| DockerStartupError::Deadline)?; + self.connecting.take(); + self.wire = Some(connected.map_err(|_| DockerStartupError::Worker)??); + let docker = self + .wire + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .port() + .bind_directory(&directory, step(at, budgets.lambda_docker_io_seconds)) + .await?; + self.bound = Some(docker.clone()); + let recovery = reconcile_namespace(&directory, &docker, step(at, 60)).await?; + if recovery.containers.remaining != 0 + || recovery.pins.remaining != 0 + || recovery.code.remaining != 0 + { + return Err(DockerError::Quarantined.into()); + } + let info = docker + .info(step(at, budgets.lambda_docker_io_seconds)) + .await?; + let architecture = platform( + info.os_type.as_deref(), + info.architecture.as_deref(), + settings.cross_architecture(), + )?; + let pull = settings.load_pull_policy(step(at, 60)).await?; + self.producing = true; + self.images = Some(ImageLeaseManager::start_qualified( + docker.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::new( + budgets + .lambda_docker_image_pins + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + budgets + .lambda_docker_deployment_jobs + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + Duration::from_secs(budgets.lambda_docker_io_seconds), + )?, + pull, + architecture, + )?); + let images = self + .images + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .port(); + self.relay = Some( + images + .prepare( + settings.relay_image(), + architecture, + None, + step(at, budgets.lambda_docker_deployment_seconds), + ) + .await + .map_err(DockerStartupError::Relay)?, + ); + let relay = self + .relay + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .clone(); + Ok(PreparedImages { + directory, + docker, + architecture, + images, + relay, + }) + } + async fn prepare_zip( + &mut self, + prepared: &PreparedImages, + at: Instant, + ) -> Result<(), DockerStartupError> { + let settings = self.config.settings.clone(); + let budgets = settings.budgets(); + if let Some(reference) = settings.zip_image() { + let base = prepared + .images + .prepare( + reference, + prepared.architecture, + None, + step(at, budgets.lambda_docker_deployment_seconds), + ) + .await?; + validate_base( + &base, + &prepared.docker, + step(at, budgets.lambda_docker_io_seconds), + ) + .await?; + let volumes: u8 = budgets + .lambda_docker_environments + .try_into() + .map_err(|_| DockerStartupError::Configuration)?; + self.code = Some(CodeVolumeManager::start( + prepared.docker.clone(), + prepared.directory.clone(), + CodeVolumeConfig::new( + volumes, + volumes.min(2), + Duration::from_secs(budgets.lambda_docker_io_seconds), + Duration::from_secs(budgets.lambda_docker_code_upload_seconds), + )?, + )?); + self.zip = Some(DockerZipRuntime::new( + &prepared.directory, + base, + self.code + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .port(), + )?); + } + Ok(()) + } + async fn prepare_probe( + &mut self, + prepared: &PreparedImages, + at: Instant, + ) -> Result { + let settings = self.config.settings.clone(); + let budgets = settings.budgets(); + let directory = prepared.directory.clone(); + let docker = prepared.docker.clone(); + let relay = prepared.relay.clone(); + let network = settings + .container_policy() + .network + .qualify(&docker, step(at, budgets.lambda_docker_io_seconds)) + .await?; + let policy = network.apply(settings.container_policy())?; + self.environments = Some(EnvironmentManager::start( + docker.clone(), + directory.clone(), + EnvironmentSettings::builder() + .environments( + budgets + .lambda_docker_environments + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + ) + .memory_bytes(budgets.lambda_docker_memory_bytes) + .io(Duration::from_secs(budgets.lambda_docker_io_seconds)) + .start(Duration::from_secs(budgets.lambda_docker_start_seconds)) + .init(self.config.lambda.init_timeout) + .idle(self.config.lambda.idle_timeout) + .log_ring_bytes(settings.log_ring_bytes()) + .build() + .try_into()?, + )?); + let environments = self + .environments + .as_ref() + .ok_or(DockerStartupError::Unavailable)? + .port(); + let base = EnvironmentProbeConfig::builder() + .directory(directory.clone()) + .relay(relay.clone()) + .environments(environments.clone()) + .policy(policy.clone()) + .network(network) + .settings(ProbeSettings::new( + Duration::from_secs(budgets.lambda_docker_start_seconds), + Duration::from_secs(budgets.lambda_docker_cleanup_seconds), + )?) + .build(); + Ok(base) + } + pub(super) async fn qualify( + &mut self, + qualification: &Arc, + at: Instant, + ) -> Result<(), DockerStartupError> { + let prepared = self.prepare_images(at).await?; + self.prepare_zip(&prepared, at).await?; + let base = self.prepare_probe(&prepared, at).await?; + let settings = self.config.settings.clone(); + let budgets = settings.budgets(); + let resource = probe_environment(base.clone(), at).await?; + let endpoint = if let Some(endpoint) = settings.gateway_endpoint() { + let config = EndpointProbeConfig::new( + base.clone(), + resource.clone(), + endpoint.clone(), + self.config.instance, + ) + .map_err(DockerStartupError::Endpoint)?; + let proof = probe_endpoint(config.clone(), at) + .await + .map_err(DockerStartupError::Endpoint)?; + Some((config, proof)) + } else { + None + }; + deadline(at)?; + let evidence = Evidence::builder() + .base(base.clone()) + .resource(resource) + .endpoint(endpoint) + .images(prepared.images) + .docker(prepared.docker) + .architecture(prepared.architecture) + .zip(self.zip.clone()) + .build(); + qualification + .evidence + .set(evidence) + .map_err(|_| DockerStartupError::Unavailable)?; + // Installing the exact proof is separate from constructing dispatch. The + // final live/clock fence still precedes any installation publication. + qualification + .check() + .map_err(|_| DockerStartupError::Unavailable)?; + self.executor = Some(Arc::new(DockerExecutor::new( + DockerExecutorConfig::builder() + .directory(base.directory) + .relay(base.relay) + .policy(base.policy) + .environments(base.environments) + .readiness(qualification.clone()) + .zip(self.zip.clone()) + .sdk( + settings + .sdk_credentials() + .cloned() + .map(|credentials| qualification.sdk_environment(credentials)) + .transpose()?, + ) + .settings(DockerExecutorSettings::new( + PoolSettings::builder() + .warm( + budgets + .lambda_docker_environments + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + ) + .warm_per_revision( + self.config + .lambda + .max_warm_instances + .try_into() + .map_err(|_| DockerStartupError::Configuration)?, + ) + .build() + .try_into()?, + Duration::from_secs(budgets.lambda_docker_cleanup_seconds), + )?) + .build(), + )?)); + deadline(at)?; + qualification + .check() + .map_err(|_| DockerStartupError::Unavailable) + } + pub(super) async fn cleanup(mut self) -> Result<(), ExecutorError> { + let mut delay = Duration::from_secs(1); + loop { + if self.cleanup_attempt().await.is_ok() { + return match self.fault.take() { + Some(error) => Err(error), + None => Ok(()), + }; + } + tracing::warn!("Docker cleanup remains owned; retrying without releasing namespace"); + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + fn retain_fault(&mut self, error: DockerStartupError) { + self.fault + .get_or_insert_with(|| DockerExecutorError::Owner(Arc::new(error)).into()); + } + async fn pending(&mut self) -> Result<(), DockerStartupError> { + if let Some(acquiring) = &mut self.acquiring { + acquiring.cancel(); + let result = acquiring + .wait(Instant::now() + Duration::from_secs(60)) + .await?; + self.acquiring.take(); + if let Ok(directory) = result { + self.directory = Some(directory); + } + } + if let Some(connecting) = &mut self.connecting { + let result = timeout_at(Instant::now() + Duration::from_secs(60), connecting) + .await + .map_err(|_| DockerStartupError::Deadline)?; + self.connecting.take(); + match result { + Ok(Ok(wire)) => self.wire = Some(wire), + Ok(Err(_)) => {} + Err(_) => self.retain_fault(DockerStartupError::Worker), + } + } + Ok(()) + } + async fn cleanup_attempt(&mut self) -> Result<(), DockerStartupError> { + self.pending().await?; + if let Some(executor) = &self.executor { + let result = executor.shutdown().await; + let settled = executor.physically_settled(); + if let Err(error) = result { + self.retain_fault(DockerStartupError::Cleanup(error)); + } + if !settled { + return Err(DockerStartupError::Unavailable); + } + } + if let Some(environments) = &mut self.environments { + let result = environments + .shutdown(Instant::now() + Duration::from_secs(60)) + .await; + let settled = environments.physically_settled(); + if let Err(error) = result { + self.retain_fault(error.into()); + } + if !settled { + return Err(DockerStartupError::Unavailable); + } + } + if let Some(code) = &mut self.code { + let result = code + .shutdown(Instant::now() + Duration::from_secs(60)) + .await; + let settled = code.physically_settled(); + if let Err(error) = result { + self.retain_fault(error.into()); + } + if !settled { + return Err(DockerStartupError::Unavailable); + } + } + if let Some(images) = &mut self.images { + let result = images + .shutdown(Instant::now() + Duration::from_secs(60)) + .await; + let settled = images.physically_settled(); + if let Err(error) = result { + self.retain_fault(error.into()); + } + if !settled { + return Err(DockerStartupError::Unavailable); + } + } + if self.producing { + let directory = self + .directory + .as_ref() + .ok_or(DockerStartupError::Unavailable)?; + let docker = self.bound.as_ref().ok_or(DockerStartupError::Unavailable)?; + let at = Instant::now() + Duration::from_secs(60); + containers(directory, docker, at).await?; + images(directory, docker, at).await?; + volumes(directory, docker, at).await?; + if !directory.container_records(at).await?.is_empty() + || !directory.image_records(at).await?.is_empty() + || !directory.code_records(at).await?.is_empty() + { + return Err(DockerError::Quarantined.into()); + } + self.producing = false; + } + if let Some(wire) = &mut self.wire { + let result = timeout_at(Instant::now() + Duration::from_secs(60), wire.shutdown()) + .await + .map_err(|_| DockerStartupError::Deadline)?; + let settled = wire.physically_settled(); + if let Err(error) = result { + self.retain_fault(error.into()); + } + if !settled { + return Err(DockerStartupError::Unavailable); + } + } + Ok(()) + } +} +fn platform( + os: Option<&str>, + architecture: Option<&str>, + cross: bool, +) -> Result { + if os != Some("linux") { + return Err(DockerStartupError::Configuration); + } + let selected = match architecture { + Some("aarch64" | "arm64") => Architecture::Arm64, + Some("x86_64" | "amd64") => Architecture::X86_64, + _ => return Err(DockerStartupError::Configuration), + }; + if !cross + && !matches!( + (ARCH, selected), + ("aarch64", Architecture::Arm64) | ("x86_64", Architecture::X86_64) + ) + { + return Err(DockerStartupError::Configuration); + } + Ok(selected) +} diff --git a/crates/rustack-lambda-core/src/docker/startup/tests.rs b/crates/rustack-lambda-core/src/docker/startup/tests.rs new file mode 100644 index 00000000..380fac95 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/startup/tests.rs @@ -0,0 +1,529 @@ +use std::{path::PathBuf, time::Duration}; + +use rstest::rstest; +use rustack_core::settings::RuntimeBudgets; +use tokio::time::Instant; + +use super::*; +use crate::docker::{EnvironmentConfig, ImageLeaseConfig}; + +fn settings() -> DockerRuntimeSettings { + DockerRuntimeSettings::from_lookup( + |key| match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + "LAMBDA_DOCKER_SOCKET" => { + Some("unix:///private/rustack-missing-fixture/docker.sock".into()) + } + _ => None, + }, + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap() +} +fn lambda() -> LambdaConfig { + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + } +} +async fn begin(config: DockerStartupConfig) -> DockerRuntime { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + match DockerRuntime::new(config.clone()) { + Ok(owner) => return owner, + Err(DockerStartupError::Worker) => tokio::task::yield_now().await, + Err(error) => panic!("unexpected startup constructor: {error}"), + } + } + }) + .await + .unwrap() +} + +fn config(directory: PathBuf) -> DockerStartupConfig { + DockerStartupConfig::new( + settings(), + directory, + GatewayInstanceId::generate().unwrap(), + lambda(), + ) + .unwrap() +} +#[rstest] +#[case(0)] +#[case(1)] +#[case(2)] +#[case(3)] +#[case(4)] +#[test] +fn test_should_reject_mismatching_startup_inputs_before_io(#[case] case: u8) { + let mut lambda = lambda(); + match case { + 0 => lambda.executor = ExecutorBackend::Native, + 1 => lambda.max_warm_instances = 2, + 2 => lambda.init_timeout = Duration::ZERO, + 3 => lambda.init_timeout = Duration::from_secs(901), + _ => lambda.idle_timeout = Duration::from_secs(599), + } + assert!(matches!( + DockerStartupConfig::new( + settings(), + "/private/data".into(), + GatewayInstanceId::generate().unwrap(), + lambda + ), + Err(DockerStartupError::Configuration) + )); +} +#[test] +fn test_should_reject_disconnected_configured_endpoint_before_startup() { + let settings = DockerRuntimeSettings::from_lookup( + |key| match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + "LAMBDA_DOCKER_GATEWAY_ENDPOINT" => Some("http://localhost:4566".into()), + _ => None, + }, + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap(); + assert!( + DockerStartupConfig::new( + settings, + "/private/data".into(), + GatewayInstanceId::generate().unwrap(), + lambda() + ) + .is_err() + ); +} +#[tokio::test] +async fn test_should_retain_failed_attempt_without_replay_and_release_after_owned_shutdown() { + let directory = tempfile::tempdir().unwrap(); + let mut owner = begin(config(directory.path().into())).await; + let status = owner.status(); + assert!(!status.capabilities().ready && !status.capabilities().sdk_gateway); + assert!(status.capabilities().capacity.is_none()); + assert!( + serde_json::to_value(status.capabilities()) + .unwrap() + .get("capacity") + .is_none() + ); + assert!(owner.installation().is_err()); + assert!( + owner + .qualify(Instant::now() + Duration::from_secs(10)) + .await + .is_err() + ); + assert_eq!(owner.phase, Phase::Failed); + assert!(matches!( + owner + .qualify(Instant::now() + Duration::from_secs(10)) + .await, + Err(DockerStartupError::Unavailable) + )); + assert!(matches!( + DirectoryLease::acquire( + directory.path().into(), + Instant::now() + Duration::from_secs(5) + ) + .await, + Err(DirectoryLeaseError::InUse) + )); + owner + .shutdown(Instant::now() + Duration::from_secs(10)) + .await + .unwrap(); + owner + .shutdown(Instant::now() + Duration::from_secs(10)) + .await + .unwrap(); + assert!(!status.capabilities().ready); + let lease = DirectoryLease::acquire( + directory.path().into(), + Instant::now() + Duration::from_secs(5), + ) + .await + .unwrap(); + lease.require_journaled().unwrap(); +} +#[tokio::test] +async fn test_should_reject_expired_qualification_without_publishing_or_touching_directory() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("absent"); + let mut owner = begin(config(path.clone())).await; + assert!(matches!( + owner.qualify(Instant::now()).await, + Err(DockerStartupError::Deadline) + )); + assert!(!tokio::fs::try_exists(path).await.unwrap()); + assert!(owner.installation().is_err()); + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} +#[test] +fn test_should_redact_startup_configuration_and_error_diagnostics() { + let config = config("/private/OPERATOR-SECRET".into()); + assert!(!format!("{config:?}").contains("OPERATOR-SECRET")); + assert_eq!( + config.settings.container_policy().idle(), + Duration::from_secs(600) + ); + assert!(DockerRuntime::new(config).is_err()); +} + +#[tokio::test] +async fn test_should_keep_abandoned_qualification_io_and_directory_until_same_cleanup_joins() { + let root = tempfile::tempdir().unwrap(); + let mut daemon = crate::docker::tests::Daemon::new(); + let socket = daemon.socket_uri(); + let settings = DockerRuntimeSettings::from_lookup( + |key| match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + "LAMBDA_DOCKER_SOCKET" => Some(socket.clone()), + _ => None, + }, + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap(); + let config = DockerStartupConfig::new( + settings, + root.path().into(), + GatewayInstanceId::generate().unwrap(), + lambda(), + ) + .unwrap(); + let mut owner = begin(config).await; + let mut qualification = Box::pin(owner.qualify(Instant::now() + Duration::from_secs(20))); + let request = tokio::select! { + request=daemon.next()=>request, + result=&mut qualification=>panic!("qualification completed before inventory: {result:?}"), + }; + assert!(request.head.contains("/containers/json?")); + drop(qualification); + assert_eq!(owner.phase, Phase::Qualifying); + assert!(matches!( + owner + .qualify(Instant::now() + Duration::from_secs(10)) + .await, + Err(DockerStartupError::Unavailable) + )); + assert!(matches!( + owner.shutdown(Instant::now()).await, + Err(DockerStartupError::Deadline) + )); + drop(request); + owner + .shutdown(Instant::now() + Duration::from_secs(10)) + .await + .unwrap(); + daemon.close().await; + DirectoryLease::acquire(root.path().into(), Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_should_keep_cancelled_version_connect_owned_until_response_and_repeated_shutdown() { + use tokio::net::UnixListener; + let root = tempfile::tempdir().unwrap(); + let socket_root = tempfile::tempdir().unwrap(); + let socket = socket_root.path().join("docker.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let settings_lookup = std::collections::BTreeMap::from([ + ("LAMBDA_DOCKER_RELAY_IMAGE", "relay:v1".to_owned()), + ("LAMBDA_DOCKER_NETWORK", "none".to_owned()), + ( + "LAMBDA_DOCKER_SOCKET", + format!("unix://{}", socket.display()), + ), + ]); + let settings = DockerRuntimeSettings::from_lookup( + |key| settings_lookup.get(key).cloned(), + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .unwrap(); + let config = DockerStartupConfig::new( + settings, + root.path().into(), + GatewayInstanceId::generate().unwrap(), + lambda(), + ) + .unwrap(); + let mut owner = begin(config).await; + let mut work = Box::pin(owner.qualify(Instant::now() + Duration::from_secs(10))); + let (stream, _) = tokio::select! {connection=listener.accept()=>connection.unwrap(),result=&mut work=>panic!("early connect {result:?}")}; + let request = crate::docker::tests::read_request(stream).await.unwrap(); + assert!(request.head.starts_with("GET /version ")); + drop(work); + assert!(matches!( + owner.shutdown(Instant::now()).await, + Err(DockerStartupError::Deadline) + )); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), Instant::now() + Duration::from_secs(3)).await, + Err(DirectoryLeaseError::InUse) + )); + request + .reply(200, br#"{"ApiVersion":"1.53","MinAPIVersion":"1.44"}"#) + .await; + owner + .shutdown(Instant::now() + Duration::from_secs(10)) + .await + .unwrap(); + owner + .shutdown(Instant::now() + Duration::from_secs(10)) + .await + .unwrap(); + DirectoryLease::acquire(root.path().into(), Instant::now() + Duration::from_secs(3)) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_cache_connect_worker_failure_and_never_repoll_completed_join() { + let root = tempfile::tempdir().unwrap(); + let mut owner = begin(config(root.path().into())).await; + owner.state.as_mut().unwrap().connecting = Some(tokio::spawn(async { + panic!("injected connect worker panic") + })); + for _ in 0..2 { + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await, + Err(DockerStartupError::Cleanup(_)) + )); + } +} +#[tokio::test] +async fn test_should_continue_physical_image_and_wire_cleanup_after_sticky_environment_fault() { + use uuid::Uuid; + + use crate::docker::{EnvironmentManager, ImageLeaseManager, ImagePinScope, tests::Daemon}; + let root = tempfile::tempdir().unwrap(); + let mut owner = begin(config(root.path().into())).await; + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire.port(); + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let environments = EnvironmentManager::start( + port.clone(), + directory.clone(), + EnvironmentConfig::default(), + ) + .unwrap(); + environments.abort_test_worker(); + let images = ImageLeaseManager::start( + port.clone(), + ImagePinScope::new(directory), + ImageLeaseConfig::default(), + ) + .unwrap(); + let image_status = images.port(); + let state = owner.state.as_mut().unwrap(); + state.wire = Some(wire); + state.images = Some(images); + state.environments = Some(environments); + for _ in 0..2 { + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await, + Err(DockerStartupError::Cleanup(_)) + )); + } + assert!(!image_status.status().accepting); + assert_eq!(image_status.status().pins, 0); + assert!(matches!( + port.info(Instant::now() + Duration::from_secs(3)).await, + Err(DockerError::Stopped) + )); + daemon.close().await; +} + +async fn synthetic_ready( + owner: &mut DockerRuntime, +) -> (crate::docker::tests::Daemon, crate::docker::DockerPort) { + use uuid::Uuid; + + use crate::docker::{ + DockerExecutorConfig, EnvironmentProbeConfig, ImageLeaseManager, ImagePinScope, + NetworkSelection, plan::tests::image_with, probe::tests::synthetic_resource, tests::Daemon, + }; + + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let port = wire.port(); + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let environments = EnvironmentManager::start( + port.clone(), + directory.clone(), + EnvironmentConfig::default(), + ) + .unwrap(); + let images = ImageLeaseManager::start( + port.clone(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + ) + .unwrap(); + let network: NetworkSelection = "none".parse().unwrap(); + let base = EnvironmentProbeConfig::builder() + .directory(directory.clone()) + .relay(image_with("1000", vec![])) + .environments(environments.port()) + .policy( + owner + .state + .as_ref() + .unwrap() + .config + .settings + .container_policy() + .clone(), + ) + .network( + network + .qualify(&port, Instant::now() + Duration::from_secs(5)) + .await + .unwrap(), + ) + .build(); + owner + .qualification + .evidence + .set( + Evidence::builder() + .base(base.clone()) + .resource(synthetic_resource(&base)) + .endpoint(None) + .images(images.port()) + .docker(port.clone()) + .architecture(Architecture::Arm64) + .build(), + ) + .unwrap(); + let executor = Arc::new( + DockerExecutor::new( + DockerExecutorConfig::builder() + .directory(directory.clone()) + .relay(base.relay) + .policy(base.policy) + .environments(base.environments) + .readiness(owner.qualification.clone()) + .build(), + ) + .unwrap(), + ); + let state = owner.state.as_mut().unwrap(); + state.directory = Some(directory); + state.environments = Some(environments); + state.images = Some(images); + state.wire = Some(wire); + state.executor = Some(executor); + owner.phase = Phase::Ready; + (daemon, port) +} +#[tokio::test] +async fn test_should_refuse_late_installation_after_dependency_revocation_and_join_faulted_graph() { + let root = tempfile::tempdir().unwrap(); + let mut owner = begin(config(root.path().into())).await; + let (daemon, port) = synthetic_ready(&mut owner).await; + owner.phase = Phase::Ready; + assert!(owner.installation().is_ok()); + let environments = owner.state.as_mut().unwrap().environments.as_mut().unwrap(); + environments.abort_test_worker(); + assert!( + environments + .shutdown(Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + assert!(matches!( + owner.installation(), + Err(DockerStartupError::Unavailable) + )); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_secs(5)) + .await, + Err(DockerStartupError::Cleanup(_)) + )); + assert!(matches!( + port.info(Instant::now() + Duration::from_secs(5)).await, + Err(DockerError::Stopped) + )); + daemon.close().await; +} +#[test] +fn test_should_reject_mutable_zip_selection_before_startup() { + assert!( + DockerRuntimeSettings::from_lookup( + |key| match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + "LAMBDA_DOCKER_ZIP_IMAGE" => Some("zip:v1".into()), + _ => None, + }, + &RuntimeBudgets::default(), + Duration::from_secs(600), + ) + .is_err() + ); +} + +#[rstest] +#[case::insufficient_pins(1, false)] +#[case::separate_source_and_relay(2, true)] +#[test] +fn test_should_accept_pinned_zip_only_with_separate_source_pin_capacity( + #[case] pins: usize, + #[case] accepted: bool, +) { + let mut budgets = RuntimeBudgets::default(); + budgets.lambda_docker_image_pins = pins; + budgets.lambda_docker_deployment_jobs = 1; + let settings = DockerRuntimeSettings::from_lookup( + |key| match key { + "LAMBDA_DOCKER_RELAY_IMAGE" => Some("relay:v1".into()), + "LAMBDA_DOCKER_NETWORK" => Some("none".into()), + "LAMBDA_DOCKER_ZIP_IMAGE" => Some(format!("base@sha256:{}", "a".repeat(64))), + _ => None, + }, + &budgets, + Duration::from_secs(600), + ) + .unwrap(); + assert_eq!( + DockerStartupConfig::new( + settings, + "/private/data".into(), + GatewayInstanceId::generate().unwrap(), + lambda() + ) + .is_ok(), + accepted + ); +} + +#[test] +fn test_should_never_issue_handler_sdk_context_without_qualified_endpoint_evidence() { + let qualification = Arc::new(Qualification::default()); + let credentials: LocalSdkCredentials = ("local-key".into(), "local-secret".into()) + .try_into() + .unwrap(); + assert!(matches!( + qualification.sdk_environment(credentials), + Err(DockerStartupError::Unavailable) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/tests.rs b/crates/rustack-lambda-core/src/docker/tests.rs new file mode 100644 index 00000000..7e7b1cb5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/tests.rs @@ -0,0 +1,990 @@ +use std::{ + future::{Future, pending}, + task::{Context, Waker}, +}; + +use tempfile::TempDir; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, duplex}, + net::{UnixListener, UnixStream}, +}; + +use super::*; +use crate::artifact::{Architecture, ArtifactError}; + +#[derive(Debug, Clone)] +pub(super) struct LogWriter(pub(super) std::sync::Arc>>); +impl Write for LogWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self.0.lock().unwrap(); + if output.len().saturating_add(bytes.len()) > 2 * MAX_BODY_BYTES { + return Err(io::Error::other("capture exceeded limit")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(super) fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +pub(super) struct IncomingRequest { + pub(super) head: String, + pub(super) body: Vec, + pub(super) stream: UnixStream, +} +impl IncomingRequest { + pub(super) async fn reply(mut self, status: u16, body: &[u8]) { + let head = format!( + "HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + timeout_at(deadline(), async { + self.stream.write_all(head.as_bytes()).await?; + self.stream.write_all(body).await + }) + .await + .unwrap() + .unwrap(); + } + pub(super) async fn raw(mut self, response: &[u8]) { + timeout_at(deadline(), self.stream.write_all(response)) + .await + .unwrap() + .unwrap(); + } +} + +pub(super) const DAEMON_ID: &str = "44ca5378-c28c-4bac-b6ed-2b298f02ecb5"; + +pub(super) struct Daemon { + directory: TempDir, + pub(super) requests: mpsc::Receiver, + stop: watch::Sender, + task: JoinHandle<()>, + pub(super) identity: watch::Sender, + pub(super) identity_status: watch::Sender, +} +impl Daemon { + pub(super) async fn bind_fixture(directory: &DirectoryLease) { + directory + .bind_daemon( + binding::DaemonIdentity::try_from(DAEMON_ID).unwrap(), + deadline(), + ) + .await + .unwrap(); + } + + pub(super) fn new() -> Self { + Self::with_version("1.53") + } + fn with_version(version: &'static str) -> Self { + let directory = tempfile::tempdir().unwrap(); + let listener = UnixListener::bind(directory.path().join("docker.sock")).unwrap(); + let (requests, received) = mpsc::channel(32); + let (stop, mut stopped) = watch::channel(false); + let (identity, daemon_identity) = watch::channel(DAEMON_ID.to_owned()); + let (identity_status, daemon_status) = watch::channel(200); + let task = tokio::spawn(async move { + let mut jobs = JoinSet::new(); + loop { + tokio::select! { + biased; + () = connection::stopped(&mut stopped) => break, + job = jobs.join_next(), if !jobs.is_empty() => { job.unwrap().unwrap(); }, + stream = listener.accept(), if jobs.len() < 16 => { + let requests = requests.clone(); + let daemon_identity = daemon_identity.clone(); + let daemon_status = daemon_status.clone(); + let (stream, _) = stream.unwrap(); + jobs.spawn(async move { + // Closing a fixture connection during shutdown is + // normal. Panics still fail the joined server task. + let mut stream = stream; + while let Ok(mut incoming) = read_request(stream).await { + if incoming.head.starts_with("GET /version ") { + incoming.reply(200, format!(r#"{{"ApiVersion":"{version}","MinAPIVersion":"1.44"}}"#).as_bytes()).await; + break; + } else if incoming.head.starts_with("GET /info ") { + let body = serde_json::to_vec(&serde_json::json!({"ID":daemon_identity.borrow().clone()})).unwrap(); + let status = *daemon_status.borrow(); + let head = format!("HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nConnection: keep-alive\r\n\r\n", body.len()); + if incoming.stream.write_all(head.as_bytes()).await.is_err() + || incoming.stream.write_all(&body).await.is_err() { break; } + stream = incoming.stream; + } else { let _ = requests.send(incoming).await; break; } + } + }); + } + } + } + jobs.abort_all(); + while let Some(result) = jobs.join_next().await { + if let Err(error) = result { + assert!(error.is_cancelled()); + } + } + }); + Self { + directory, + requests: received, + stop, + task, + identity, + identity_status, + } + } + pub(super) fn socket_uri(&self) -> String { + format!( + "unix://{}", + self.directory.path().join("docker.sock").display() + ) + } + pub(super) fn socket(&self) -> DockerSocket { + self.socket_uri().parse().unwrap() + } + pub(super) async fn wire(&self) -> DockerWire { + match DockerWire::connect(self.socket(), deadline()).await { + Ok(wire) => wire, + Err(DockerError::Http(error)) => panic!("fixture HTTP startup failed: {error:?}"), + Err(error) => panic!("fixture startup failed: {error}"), + } + } + pub(super) async fn next(&mut self) -> IncomingRequest { + timeout_at(deadline(), self.requests.recv()) + .await + .unwrap() + .unwrap() + } + pub(super) async fn close(self) { + self.stop.send_replace(true); + self.task.await.unwrap(); + } +} + +pub(super) async fn read_request(mut stream: UnixStream) -> io::Result { + timeout_at(deadline(), async { + let mut input = Vec::new(); + let boundary = loop { + if let Some(index) = input.windows(4).position(|value| value == b"\r\n\r\n") { + break index + 4; + } + if input.len() > 16 * 1024 { + return Err(io::Error::other("fixture header exceeded cap")); + } + let mut block = [0; 512]; + let count = stream.read(&mut block).await?; + if count == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + input.extend_from_slice(block.get(..count).unwrap()); + }; + let head = + String::from_utf8(input.get(..boundary).unwrap().to_vec()).map_err(io::Error::other)?; + let length = head + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + let limit = if head.starts_with("PUT ") && head.contains("/archive?") { + 4 * MAX_BODY_BYTES + } else { + MAX_BODY_BYTES + }; + if length > limit { + return Err(io::Error::other("fixture body exceeded cap")); + } + let mut body = input.get(boundary..).unwrap().to_vec(); + let missing = length + .checked_sub(body.len()) + .ok_or_else(|| io::Error::other("unexpected pipelining"))?; + let mut remainder = vec![0; missing]; + stream.read_exact(&mut remainder).await?; + body.extend(remainder); + Ok(IncomingRequest { head, body, stream }) + }) + .await + .map_err(|_| io::Error::other("fixture deadline"))? +} + +fn record(channel: u8, bytes: &[u8]) -> Vec { + let mut output = vec![channel, 0, 0, 0]; + output.extend_from_slice(&u32::try_from(bytes.len()).unwrap().to_be_bytes()); + output.extend_from_slice(bytes); + output +} + +#[tokio::test] +async fn test_should_negotiate_bound_metadata_encode_paths_and_preserve_status() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + assert_eq!(port.api_version().to_string(), "1.53"); + let (info, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + assert!(request.head.starts_with("GET /v1.53/info ")); + request + .reply(200, br#"{"ID":"daemon-id","OSType":"linux"}"#) + .await; + }); + assert_eq!(info.unwrap().id.as_deref(), Some("daemon-id")); + let image: ImageReference = "localhost:5000/team/worker:v1".parse().unwrap(); + let (result, ()) = tokio::join!(port.inspect_image(&image, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("GET /v1.53/images/localhost%3A5000%2Fteam%2Fworker%3Av1/json ") + ); + request.reply(404, b"secret daemon error").await; + }); + let error = result.unwrap_err(); + assert!(matches!(error, DockerError::Status(StatusCode::NOT_FOUND))); + assert!(!format!("{error:?} {error}").contains("secret")); + wire.shutdown().await.unwrap(); + assert!(matches!( + port.info(deadline()).await, + Err(DockerError::Stopped) + )); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_oversized_truncated_chunked_trailer_and_duplicate_metadata() { + for mode in 0..6 { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + match mode { + 0 => request.raw(b"HTTP/1.1 200 OK\r\nContent-Length: 1048577\r\n\r\n").await, + 1 => request.raw(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n{}").await, + 2 => request.raw(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\n{}\r\n0\r\nSecret: trailer\r\n\r\n").await, + 3 => request.reply(200, br#"{"ID":"one","ID":"two"}"#).await, + 4 => request.raw(format!("HTTP/1.1 200 OK\r\nSecret: {}\r\nContent-Length: 2\r\n\r\n{{}}", "x".repeat(4097)).as_bytes()).await, + _ => { + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + response.extend_from_slice(b"100001\r\n"); + response.extend(std::iter::repeat_n(b'x', MAX_BODY_BYTES + 1)); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + // The client closes as soon as its byte cap is reached. + let mut stream = request.stream; + let _ = timeout_at(deadline(), stream.write_all(&response)).await; + } + } + }); + assert!(result.is_err(), "case {mode}"); + wire.shutdown().await.unwrap(); + daemon.close().await; + } +} + +#[tokio::test] +async fn test_should_send_typed_container_create_and_management_operations() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key: DockerKey = "owned-test".parse().unwrap(); + let body = ContainerCreateBody { + image: Some("sha256:fixture".into()), + env: Some(vec!["SECRET=value".into()]), + ..Default::default() + }; + let (created, ()) = tokio::join!(port.create_container(&key, &body, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("POST /v1.53/containers/create?name=owned-test ") + ); + let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + assert_eq!(body["Env"], serde_json::json!(["SECRET=value"])); + request + .reply(201, br#"{"Id":"owned-test","Warnings":[]}"#) + .await; + }); + assert_eq!(created.unwrap().id, "owned-test"); + for operation in 0..5 { + let (result, ()) = tokio::join!( + async { + match operation { + 0 => port.start_container(&key, deadline()).await, + 1 => port + .inspect_container(&key, deadline()) + .await + .map(|inspect| assert_eq!(inspect.id.as_deref(), Some("owned-test"))), + 2 => port.kill_container(&key, deadline()).await, + 3 => port + .wait_container(&key, deadline()) + .await + .map(|wait| assert_eq!(wait.status_code, 0)), + _ => port.remove_container(&key, deadline()).await, + } + }, + async { + let request = daemon.next().await; + assert!(request.head.contains("/containers/owned%2Dtest")); + match operation { + 1 => request.reply(200, br#"{"Id":"owned-test"}"#).await, + 3 => request.reply(200, br#"{"StatusCode":0}"#).await, + _ => request.reply(204, b"").await, + } + } + ); + result.unwrap(); + } + let filter = OwnerFilter::new(Uuid::new_v4(), Some(Uuid::new_v4()), None).unwrap(); + let (found, ()) = tokio::join!(port.list_containers(filter, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains("rustack.lambda.ownerId")); + request.reply(200, b"[]").await; + }); + assert_eq!(found.unwrap(), Vec::::new()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_request_serialization_and_redact_request_debug() { + let body = ContainerCreateBody { + env: Some(vec!["TOP_SECRET=value".into()]), + ..Default::default() + }; + let request = RawRequest { + archive: None, + pull: None, + binding: None, + mutation: None, + operation: Operation::Create, + method: Method::POST, + target: "/secret-target".into(), + body: encode(&body).unwrap(), + expected: StatusCode::CREATED, + }; + let shown = format!("{request:?}"); + assert!(!shown.contains("TOP_SECRET") && !shown.contains("secret-target")); + let oversized = ContainerCreateBody { + env: Some(vec!["x".repeat(MAX_BODY_BYTES)]), + ..Default::default() + }; + assert!(matches!(encode(&oversized), Err(DockerError::TooLarge))); + assert!( + !format!( + "{:?}", + Encoded { + bytes: b"secret".to_vec(), + failure: None + } + ) + .contains("secret") + ); +} + +#[tokio::test] +async fn test_should_upgrade_handoff_and_decode_fragmented_binary_output_without_a_management_slot() +{ + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key: DockerKey = "relay-test".parse().unwrap(); + let (attached, mut server) = tokio::join!(port.attach(&key, deadline()), async { + let mut request = daemon.next().await; + assert!(request.head.contains("logs=false")); + request + .stream + .write_all( + b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n", + ) + .await + .unwrap(); + request.stream + }); + let attach = attached.unwrap(); + assert!(!format!("{attach:?}").contains("secret")); + let (mut reader, mut writer) = attach.split(Duration::from_secs(1)).unwrap(); + let mut records = record(1, b"binary\0secret"); + records.extend(record(2, b"stderr secret")); + let (chunks, ()) = tokio::join!( + async { + let first = reader.next(deadline()).await.unwrap().unwrap(); + let second = reader.next(deadline()).await.unwrap().unwrap(); + (first, second) + }, + async { + for byte in records { + server.write_all(&[byte]).await.unwrap(); + } + } + ); + assert_eq!(chunks.0.bytes.as_ref(), b"binary\0secret"); + assert_eq!(chunks.1.channel, OutputChannel::Stderr); + assert!(!format!("{:?} {reader:?}", chunks.0).contains("secret")); + writer.write_all(b"raw stdin").await.unwrap(); + let mut stdin = [0; 9]; + server.read_exact(&mut stdin).await.unwrap(); + assert_eq!(&stdin, b"raw stdin"); + // Exported attach has no management driver left to block shutdown. + wire.shutdown().await.unwrap(); + drop(reader); + drop(writer); + drop(server); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_cleanup_capacity_while_business_calls_and_mailbox_are_full() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let mut calls = JoinSet::new(); + for _ in 0..6 { + let port = port.clone(); + calls.spawn(async move { port.info(deadline()).await }); + } + let mut held = Vec::new(); + for _ in 0..6 { + held.push(daemon.next().await); + } + let mut queued = Vec::new(); + for _ in 0..32 { + let (reply, response) = oneshot::channel(); + port.raw + .business + .try_send(Envelope { + upload_completion: None, + request: RawRequest { + archive: None, + pull: None, + binding: None, + mutation: None, + operation: Operation::Info, + method: Method::GET, + target: "/v1.53/info".into(), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline: deadline(), + reply, + }) + .unwrap(); + queued.push(response); + } + assert!(matches!( + port.info(deadline()).await, + Err(DockerError::Busy) + )); + let key: DockerKey = "owned-container".parse().unwrap(); + let (removed, ()) = tokio::join!(port.remove_container(&key, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("DELETE /v1.53/containers/owned%2Dcontainer?") + ); + request.reply(204, b"").await; + }); + removed.unwrap(); + wire.shutdown().await.unwrap(); + for response in queued { + assert!(matches!(response.await.unwrap(), Err(DockerError::Stopped))); + } + while let Some(result) = calls.join_next().await { + assert!(result.unwrap().is_err()); + } + for mut request in held { + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + } + daemon.close().await; +} + +#[tokio::test] +async fn test_should_admit_kill_while_two_lifecycle_waits_are_blocked() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let mut calls = JoinSet::new(); + let key: DockerKey = "running-container".parse().unwrap(); + for _ in 0..2 { + let port = port.clone(); + let key = key.clone(); + calls.spawn(async move { port.wait_container(&key, deadline()).await }); + } + let mut held = Vec::new(); + for _ in 0..2 { + let request = daemon.next().await; + assert!(request.head.contains("/wait?condition=not-running")); + held.push(request); + } + let (killed, ()) = tokio::join!( + port.kill_container(&key, Instant::now() + Duration::from_secs(1)), + async { + let request = daemon.next().await; + assert!(request.head.contains("/kill?signal=SIGKILL")); + request.reply(204, b"").await; + } + ); + killed.unwrap(); + wire.shutdown().await.unwrap(); + while let Some(result) = calls.join_next().await { + assert!(result.unwrap().is_err()); + } + for mut request in held { + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + } + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_inflight_calls_after_waiter_drop_and_join_after_cancelled_shutdown() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let caller = tokio::spawn(async move { port.info(deadline()).await }); + let request = daemon.next().await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + // The accepted request still owns its driver even though its response + // waiter is gone. Finish the body before stopping the retained wire owner. + request.reply(200, b"{}").await; + let mut shutdown = Box::pin(wire.shutdown()); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(shutdown); + wire.shutdown().await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_filter_secret_requests_responses_and_partial_trailers_under_trace() { + let capture = LogWriter(std::sync::Arc::default()); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + async { + tracing::debug!("application_event_is_preserved"); + let dispatch = runtime_dispatch(); + tracing::dispatcher::with_default(&dispatch, || { + tracing::trace!(target: "bollard::docker", "secret_constructed_bollard"); + }); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key = "secret-request-target".parse().unwrap(); + let body = ContainerCreateBody { env: Some(vec!["SECRET_ENV=secret_request_body".into()]), ..Default::default() }; + let (created, ()) = tokio::join!(port.create_container(&key, &body, deadline()), async { + let request = daemon.next().await; + request.reply(201, br#"{"Id":"known","Warnings":["secret_response_body"]}"#).await; + }); + assert_eq!(created.unwrap().id, "known"); + let (result, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + let mut stream = request.stream; + stream.write_all(b"HTTP/1.1 200 OK\r\nSecret: secret_response_header\r\nTransfer-Encoding: chunked\r\n\r\n2\r\n{}\r\n0\r\nSecret: secret_partial_trailer").await.unwrap(); + // Force at least one partial trailer-buffer parse before finishing. + tokio::task::yield_now().await; + stream.write_all(b"\r\n\r\n").await.unwrap(); + }); + assert!(result.is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; + }.with_subscriber(subscriber).await; + let shown = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); + assert!(shown.contains("application_event_is_preserved")); + for secret in [ + "secret_constructed_bollard", + "secret-request-target", + "secret_request_body", + "secret_response_body", + "secret_response_header", + "secret_partial_trailer", + ] { + assert!( + !shown.contains(secret), + "transport secret escaped the filter" + ); + } +} + +#[tokio::test] +async fn test_should_stream_large_records_in_bounded_chunks_and_reject_bad_outer_framing() { + let data = record(1, &vec![7; 100_000]); + let mut reader = DockerOutputReader::new(data.as_slice(), Duration::from_secs(1)).unwrap(); + let mut total = 0; + while let Some(chunk) = reader.next(deadline()).await.unwrap() { + assert!(chunk.bytes.len() <= 16 * 1024); + total += chunk.bytes.len(); + } + assert_eq!(total, 100_000); + assert!(reader.next(deadline()).await.unwrap().is_none()); + for bytes in [ + vec![0, 0, 0, 0, 0, 0, 0, 0], + vec![3, 0, 0, 0, 0, 0, 0, 0], + vec![1, 1, 0, 0, 0, 0, 0, 0], + vec![1, 0], + vec![1, 0, 0, 0, 0xff, 0xff, 0xff, 0xff], + vec![1, 0, 0, 0, 0, 0, 0, 1], + ] { + let mut reader = DockerOutputReader::new(bytes.as_slice(), Duration::from_secs(1)).unwrap(); + assert!(reader.next(deadline()).await.is_err()); + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Poisoned) + )); + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_not_extend_partial_record_deadlines_across_chunks_or_restart_cancelled_reads() +{ + let data = record(1, &vec![9; 32 * 1024]); + let mut reader = DockerOutputReader::new(data.as_slice(), Duration::from_secs(1)).unwrap(); + assert_eq!( + reader.next(deadline()).await.unwrap().unwrap().bytes.len(), + 16 * 1024 + ); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Deadline) + )); + let (mut input, source) = duplex(8); + input.write_all(&[1]).await.unwrap(); + let mut reader = DockerOutputReader::new(source, Duration::from_secs(1)).unwrap(); + let mut waiting = Box::pin(reader.next(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Poisoned) + )); +} + +#[test] +fn test_should_validate_explicit_sockets_identifiers_and_version_negotiation() { + for socket in [ + "/var/run/docker.sock", + "tcp://localhost:2375", + "unix://relative", + "unix:///bad\0socket", + ] { + assert!(socket.parse::().is_err()); + } + for key in ["", "../bad", "path/name", "-bad", "space name"] { + assert!(key.parse::().is_err()); + } + assert!("a".repeat(129).parse::().is_err()); + assert_eq!(ApiVersion::negotiate("1.99", "1.44").unwrap().minor(), 53); + assert!(ApiVersion::negotiate("1.43", "1.24").is_err()); + assert!(ApiVersion::negotiate("1.99", "1.54").is_err()); + for version in ["1.053", "2.53", "1.-1", "1.1000"] { + assert!(version.parse::().is_err()); + } + assert!(OwnerFilter::new(Uuid::nil(), None, None).is_err()); + assert!(OwnerFilter::new(Uuid::new_v4(), None, Some(Uuid::new_v4())).is_err()); +} + +#[tokio::test] +async fn test_should_close_a_timed_out_body_driver_without_stopping_the_wire() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!(port.info(Instant::now() + Duration::from_secs(1)), async { + let mut request = daemon.next().await; + request + .stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n") + .await + .unwrap(); + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + assert!(matches!(result, Err(DockerError::Deadline))); + let (next, ()) = tokio::join!(port.info(deadline()), async { + daemon.next().await.reply(200, b"{}").await; + }); + next.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_cache_supervision_failure_when_priority_stop_wins() { + let (_business, requests) = mpsc::channel(32); + let (_cleanup, priority) = mpsc::channel(8); + let (_waits, waits) = mpsc::channel(8); + let (_pulls, pulls) = mpsc::channel(8); + let (control, _) = watch::channel(true); + let emergency = Arc::new(CleanupTask::default()); + let mut actor = Actor::builder() + .runtime(Handle::current()) + .emergency(emergency.clone()) + .socket(PathBuf::from("/unused-fixture-socket")) + .business(requests) + .cleanup(priority) + .waits(waits) + .pulls(pulls) + .control(control.clone()) + .build(); + let failed = actor + .jobs + .spawn(async { panic!("controlled wire worker failure") }); + while !failed.is_finished() { + tokio::task::yield_now().await; + } + let task = tokio::spawn(actor.run()); + let mut owner = Owner { + emergency, + stop: control, + task: Some(task), + primary: None, + settlement: None, + }; + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); +} + +#[tokio::test] +async fn test_should_reject_expired_and_excessive_deadlines_before_metadata_publication() { + assert!(matches!( + check_deadline(Instant::now()), + Err(DockerError::Deadline) + )); + assert!(matches!( + check_deadline(Instant::now() + Duration::from_secs(601)), + Err(DockerError::Configuration) + )); + let response = RawReply::Body { + status: StatusCode::OK, + bytes: Bytes::from_static(b"{}"), + }; + let decoded: Result = decode(response, Instant::now()); + assert!(matches!(decoded, Err(DockerError::Deadline))); +} + +#[rstest::rstest] +#[case("1.44", false)] +#[case("1.48", false)] +#[case("1.49", true)] +#[case("1.53", true)] +#[tokio::test] +async fn test_should_resolve_preloaded_images_with_versioned_platform_selection( + #[case] version: &'static str, + #[case] platform: bool, +) { + use crate::artifact::Architecture; + let mut daemon = Daemon::with_version(version); + let mut wire = daemon.wire().await; + let resolver = ImageResolver::new(wire.port()); + let reference = "registry.test:5000/team/worker:v1".parse().unwrap(); + let invalid_override = rustack_lambda_model::types::ImageConfig { + working_directory: Some("relative".into()), + ..Default::default() + }; + assert!(matches!( + resolver + .resolve_local( + &reference, + Architecture::Arm64, + Some(&invalid_override), + deadline() + ) + .await, + Err(DockerError::Configuration) + )); + assert!(daemon.requests.try_recv().is_err()); + let (image, ()) = tokio::join!( + resolver.resolve_local(&reference, Architecture::Arm64, None, deadline()), + async { + let request = daemon.next().await; + let target = request.head.split_whitespace().nth(1).unwrap(); + assert_eq!( + target, + format!("/v{version}/images/registry%2Etest%3A5000%2Fteam%2Fworker%3Av1/json") + ); + let id = format!("sha256:{}", "a".repeat(64)); + let body = serde_json::json!({"Id":id,"Os":"linux","Architecture":"arm64","Size":1024,"Config":{"Cmd":["/start"]},"GraphDriver":{"Name":"overlay2","Data":{}}}); + request + .reply(200, &serde_json::to_vec(&body).unwrap()) + .await; + if platform { + let request = daemon.next().await; + let target = request.head.split_whitespace().nth(1).unwrap(); + let (path, query) = target.split_once('?').unwrap(); + assert_eq!( + path, + format!("/v{version}/images/sha256%3A{}/json", "a".repeat(64)) + ); + let pairs: Vec<_> = form_urlencoded::parse(query.as_bytes()).collect(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs.first().unwrap().0, "platform"); + assert_eq!( + serde_json::from_str::(&pairs.first().unwrap().1).unwrap(), + serde_json::json!({"os":"linux","architecture":"arm64"}) + ); + let mut selected = body; + selected["Id"] = serde_json::json!(format!("sha256:{}", "b".repeat(64))); + selected["Descriptor"] = serde_json::json!({"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":format!("sha256:{}","b".repeat(64))}); + request + .reply(200, &serde_json::to_vec(&selected).unwrap()) + .await; + } else { + let request = daemon.next().await; + assert!(request.head.starts_with(&format!("GET /v{version}/info "))); + request + .reply( + 200, + br#"{"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"]]}"#, + ) + .await; + } + } + ); + let image = image.unwrap(); + assert_eq!(image.metadata().architecture, Architecture::Arm64); + assert_eq!( + image.metadata().image_id.as_str(), + format!("sha256:{}", "a".repeat(64)) + ); + assert_eq!( + image.metadata().execution.plan(None).unwrap().entrypoint(), + [""] + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::missing_reference(false, false, 404)] +#[case::missing_selected_platform(true, false, 404)] +#[case::other_selected_failure(true, false, 500)] +#[case::identity_not_found(true, true, 404)] +#[tokio::test] +async fn test_should_distinguish_selected_platform_absence_from_reference_and_identity_failures( + #[case] captured: bool, + #[case] identity_failure: bool, + #[case] status: u16, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire + .port() + .bind_directory(&directory, deadline()) + .await + .unwrap(); + let resolver = ImageResolver::new(port); + let reference = "worker:v1".parse().unwrap(); + let (result, ()) = tokio::join!( + resolver.resolve_local(&reference, Architecture::Arm64, None, deadline()), + async { + let request = daemon.next().await; + assert!(request.head.contains("/images/worker%3Av1/json ")); + if !captured { + request.reply(status, b"{}").await; + return; + } + if identity_failure { + daemon.identity_status.send_replace(status); + } + let body = serde_json::json!({"Id":format!("sha256:{}", "a".repeat(64))}); + request + .reply(200, &serde_json::to_vec(&body).unwrap()) + .await; + if !identity_failure { + let selected = daemon.next().await; + assert!(selected.head.contains("/json?platform=")); + selected.reply(status, b"{}").await; + } + } + ); + if identity_failure { + assert!(matches!(result, Err(DockerError::IdentityCheck(source)) + if matches!(source.as_ref(), DockerError::Status(StatusCode::NOT_FOUND)))); + } else if captured && status == 404 { + assert!(matches!( + result, + Err(DockerError::Image(ArtifactError::Inspection)) + )); + } else { + assert!(matches!(result, Err(DockerError::Status(code)) if code.as_u16() == status)); + } + assert!(daemon.requests.try_recv().is_err()); + wire.shutdown().await.unwrap(); + drop(directory); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_failed_wire_root_across_abandoned_emergency_shutdown_waiter() { + let emergency = Arc::new(CleanupTask::default()); + let (release, wait) = oneshot::channel(); + assert!(emergency.start(move || async move { + wait.await + .map_err(|_| ExecutorError::CleanupFailed("fixture release closed".into())) + })); + let (stop, _) = watch::channel(false); + let task = tokio::spawn(pending::>()); + task.abort(); + let mut owner = Owner { + emergency, + stop, + task: Some(task), + primary: None, + settlement: None, + }; + assert!( + tokio::time::timeout(Duration::from_millis(25), owner.shutdown()) + .await + .is_err() + ); + assert!(owner.task.is_none()); + assert!(matches!(owner.primary, Some(Err(DockerError::Worker)))); + release.send(()).unwrap(); + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); +} diff --git a/crates/rustack-lambda-core/src/docker/types.rs b/crates/rustack-lambda-core/src/docker/types.rs new file mode 100644 index 00000000..73e5cb87 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/types.rs @@ -0,0 +1,238 @@ +//! Validated local Docker identities, versions and redacted errors. + +use std::{fmt, io, os::unix::fs::FileTypeExt, path::PathBuf, str::FromStr, sync::Arc}; + +use http::StatusCode; +use rustack_core::json::JsonError; +use tokio::time::{Instant, timeout_at}; + +use super::{DirectoryLeaseError, check_deadline}; +use crate::artifact::ArtifactError; + +/// Docker boundary failure classes; no peer response body is displayed. +#[derive(Clone, thiserror::Error)] +#[non_exhaustive] +pub enum DockerError { + /// Invalid local configuration or identifier. + #[error("invalid Docker configuration")] + Configuration, + /// ZIP bytes or the provided-runtime ELF bootstrap violate the code profile. + #[error("invalid Docker ZIP code")] + InvalidCode, + /// The runtime port cannot prove the persisted daemon/directory association. + #[error("Docker daemon binding failed")] + DaemonBinding, + /// Pulling is not authorized for this exact operator-selected registry. + #[error("Docker registry pull is not allowed")] + PullDenied, + /// The daemon reported an in-band pull failure; peer text is discarded. + #[error("Docker image pull failed")] + PullFailed, + /// The connection identity stage failed before the requested operation. + #[error("Docker connection identity check failed")] + IdentityCheck(#[source] Arc), + /// The local daemon's API range is incompatible. + #[error("unsupported Docker API version")] + Version, + /// A declared/actual byte or collection allowance was exceeded. + #[error("Docker byte budget exceeded")] + TooLarge, + /// A framing/header/response is malformed. + #[error("invalid Docker protocol")] + Protocol, + /// A cancelled partial stream cannot be restarted. + #[error("Docker stream is invalidated")] + Poisoned, + /// Absolute IO deadline expired. + #[error("Docker deadline exceeded")] + Deadline, + /// Wire service has stopped admission. + #[error("Docker wire is stopped")] + Stopped, + /// A bounded mailbox has no capacity. + #[error("Docker wire is busy")] + Busy, + /// An attached preparation caller cancelled before receiving an artifact. + #[error("Docker preparation cancelled")] + Cancelled, + /// An owned image mutation/cleanup is not yet proven settled. + #[error("Docker image cleanup is quarantined")] + Quarantined, + /// An owned task panicked or failed to join. + #[error("Docker supervision failed")] + Worker, + /// Persistent mutation ownership could not be read or durably settled. + #[error("persistent Docker ownership failed")] + Ownership(#[source] Arc), + /// A bounded output allocation could not be reserved. + #[error("Docker allocation failed")] + Allocation, + /// Request/model encoding failed without exposing input text. + #[error("invalid Docker request schema")] + Schema, + /// Non-success HTTP response; daemon error text is intentionally excluded. + #[error("Docker response status {0}")] + Status(StatusCode), + /// Local socket/stream IO failed. + #[error("Docker IO failed")] + Io(#[source] Arc), + /// HTTP transport failed. + #[error("Docker HTTP failed")] + Http(#[source] Arc), + /// Bounded metadata validation failed. + #[error(transparent)] + Json(#[from] JsonError), + /// Image inspection cannot establish a valid executable artifact. + #[error(transparent)] + Image(#[from] ArtifactError), +} + +impl fmt::Debug for DockerError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} +impl From for DockerError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} +impl From for DockerError { + fn from(error: DirectoryLeaseError) -> Self { + Self::Ownership(Arc::new(error)) + } +} +impl From for DockerError { + fn from(error: hyper::Error) -> Self { + Self::Http(Arc::new(error)) + } +} + +/// Explicit local Unix socket, independent of ambient Docker environment. +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerSocket(PathBuf); + +impl fmt::Debug for DockerSocket { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerSocket()") + } +} +impl FromStr for DockerSocket { + type Err = DockerError; + fn from_str(value: &str) -> Result { + let path = value + .strip_prefix("unix://") + .ok_or(DockerError::Configuration)?; + if !path.starts_with('/') || path.len() > 4096 || path.chars().any(char::is_control) { + return Err(DockerError::Configuration); + } + Ok(Self(PathBuf::from(path))) + } +} +impl DockerSocket { + pub(super) async fn verified(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + let path = timeout_at(deadline, tokio::fs::canonicalize(&self.0)) + .await + .map_err(|_| DockerError::Deadline)??; + let metadata = timeout_at(deadline, tokio::fs::metadata(&path)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if !metadata.file_type().is_socket() { + return Err(DockerError::Configuration); + } + Ok(path) + } +} + +/// Validated Docker object name or content identifier used as a path component. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct DockerKey(String); + +impl fmt::Debug for DockerKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerKey()") + } +} +impl FromStr for DockerKey { + type Err = DockerError; + fn from_str(value: &str) -> Result { + if value.len() > 128 + || !value + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_alphanumeric()) + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) + { + return Err(DockerError::Configuration); + } + Ok(Self(value.to_owned())) + } +} +impl DockerKey { + /// Borrow the validated spelling; it is still encoded when placed in a URI. + /// + /// ``` + /// let key: rustack_lambda_core::docker::DockerKey = "rustack-relay-123".parse()?; + /// assert_eq!(key.as_str(), "rustack-relay-123"); + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// A bounded Docker API version; negotiations select the local1.44..1.53 range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub struct ApiVersion(u16); + +impl FromStr for ApiVersion { + type Err = DockerError; + fn from_str(value: &str) -> Result { + let minor = value.strip_prefix("1.").ok_or(DockerError::Version)?; + if minor.is_empty() || minor.len() > 3 || !minor.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(DockerError::Version); + } + let parsed = minor.parse::().map_err(|_| DockerError::Version)?; + if parsed.to_string() != minor { + return Err(DockerError::Version); + } + Ok(Self(parsed)) + } +} +impl ApiVersion { + pub(super) fn negotiate(maximum: &str, minimum: &str) -> Result { + let selected = maximum.parse::()?.min(Self(53)); + let minimum: Self = minimum.parse()?; + if selected < Self(44) || selected < minimum { + return Err(DockerError::Version); + } + Ok(selected) + } + + /// Numeric API minor version, without its major version or URI prefix. + /// + /// ``` + /// let version: rustack_lambda_core::docker::ApiVersion = "1.53".parse()?; + /// assert_eq!(version.minor(), 53); + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn minor(self) -> u16 { + self.0 + } +} +impl fmt::Display for ApiVersion { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "1.{}", self.0) + } +} diff --git a/crates/rustack-lambda-core/src/docker/volumes.rs b/crates/rustack-lambda-core/src/docker/volumes.rs new file mode 100644 index 00000000..73a21bae --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/volumes.rs @@ -0,0 +1,293 @@ +//! Local volumes and typed code upload; callers retain mutation recovery. + +use std::collections::BTreeMap; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; + +use super::{ + DockerCodeArchive, DockerError, DockerKey, DockerPort, Operation, OwnerFilter, RawRequest, + decode, encode, escaped, + models::{Volume, VolumeCreateRequest, VolumeListResponse, VolumeScopeEnum}, + mutation::MutationGuard, + unit, +}; + +impl DockerPort { + /// Upload checked code to fixed `/var/task`, retaining buffer admission. + /// + /// Requires an existing writable code volume. Creates/starts no container. + /// + /// # Errors + /// Returns normal wire failures. Cancellation/error does not prove no writes. + /// + /// ```no_run + /// # async fn upload(p: &rustack_lambda_core::docker::DockerPort, c: &rustack_lambda_core::docker::DockerKey, a: &rustack_lambda_core::docker::DockerCodeArchive) { + /// let result = p.upload_code(c, a, tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await; + /// # } + /// ``` + pub async fn upload_code( + &self, + container: &DockerKey, + code: &DockerCodeArchive, + deadline: Instant, + ) -> Result<(), DockerError> { + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("path", "/var/task") + .append_pair("noOverwriteDirNonDir", "true") + .append_pair("copyUIDGID", "false") + .finish(); + unit( + self.raw + .request( + RawRequest { + archive: Some(code.clone()), + pull: None, + binding: None, + mutation: None, + operation: Operation::UploadCode, + method: Method::PUT, + target: format!( + "/v{}/containers/{}/archive?{query}", + self.version, + escaped(container.as_str()) + ), + body: code.bytes().clone(), + expected: StatusCode::OK, + }, + deadline, + ) + .await?, + deadline, + ) + } + + /// Create a named local volume without driver, bind or cluster options. + /// + /// Caller must first journal intent and retain it through uncertain creation. + /// + /// # Errors + /// Returns invalid labels, mismatched reply metadata or normal wire failures. + /// No error authorizes deletion/retry; a matching reply is not journal authority. + /// + /// ```no_run + /// # async fn create(p: &rustack_lambda_core::docker::DockerPort, n: &rustack_lambda_core::docker::DockerKey) { + /// let labels = std::collections::BTreeMap::new(); + /// let result = p.create_volume(n, &labels, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_volume( + &self, + name: &DockerKey, + labels: &BTreeMap, + deadline: Instant, + ) -> Result { + self.create_volume_tracked(name, labels, deadline, None) + .await + } + pub(super) async fn create_volume_tracked( + &self, + name: &DockerKey, + labels: &BTreeMap, + deadline: Instant, + mutation: Option, + ) -> Result { + validate_labels( + labels + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())), + )?; + let body = VolumeCreateRequest { + name: Some(name.as_str().to_owned()), + driver: Some("local".into()), + labels: Some( + labels + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ), + ..VolumeCreateRequest::default() + }; + let volume: Volume = decode( + self.raw + .request( + RawRequest { + archive: None, + pull: None, + binding: None, + mutation, + operation: Operation::CreateVolume, + method: Method::POST, + target: format!("/v{}/volumes/create", self.version), + body: encode(&body)?, + expected: StatusCode::CREATED, + }, + deadline, + ) + .await?, + deadline, + )?; + validate_volume(&volume)?; + if volume.name != name.as_str() + || volume.driver != "local" + || volume.scope != Some(VolumeScopeEnum::LOCAL) + || !volume.options.is_empty() + || volume.labels.len() != labels.len() + || !labels + .iter() + .all(|(key, value)| volume.labels.get(key) == Some(value)) + { + return Err(DockerError::Protocol); + } + Ok(volume) + } + + /// Inspect one volume through reserved cleanup admission. + /// + /// # Errors + /// Returns invalid/bounded metadata or wire failures. Observation grants no + /// destructive authority; the caller must verify its durable identity. + /// + /// ```no_run + /// # async fn inspect(p: &rustack_lambda_core::docker::DockerPort, n: &rustack_lambda_core::docker::DockerKey) { + /// let result = p.inspect_volume(n, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_volume( + &self, + name: &DockerKey, + deadline: Instant, + ) -> Result { + let volume: Volume = decode( + self.call( + Operation::InspectVolume, + Method::GET, + format!("/volumes/{}", escaped(name.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + )?; + validate_volume(&volume)?; + if volume.name != name.as_str() { + return Err(DockerError::Protocol); + } + Ok(volume) + } + + /// List a complete owner-label volume scope, without a positive result limit. + /// + /// # Errors + /// Rejects warnings, missing collections, over1024 rows, invalid metadata and + /// wire failures; no incomplete response is returned as an absence proof. + /// + /// ```no_run + /// # async fn list(p: &rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let scope = rustack_lambda_core::docker::OwnerFilter::new(uuid::Uuid::new_v4(), None, None)?; + /// let result = p.list_volumes(scope, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # Ok(()) } + /// ``` + pub async fn list_volumes( + &self, + scope: OwnerFilter, + deadline: Instant, + ) -> Result, DockerError> { + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("filters", &scope.encoded_labels()?) + .finish(); + let response: VolumeListResponse = decode( + self.call( + Operation::ListVolumes, + Method::GET, + format!("/volumes?{query}"), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + )?; + if response + .warnings + .is_some_and(|warnings| !warnings.is_empty()) + { + return Err(DockerError::Protocol); + } + let volumes = response.volumes.ok_or(DockerError::Protocol)?; + if volumes.len() > 1024 { + return Err(DockerError::TooLarge); + } + for volume in &volumes { + validate_volume(volume)?; + } + Ok(volumes) + } + + /// Remove a named volume with force disabled and reserved cleanup admission. + /// + /// # Errors + /// Returns normal wire failures, including attached-volume conflict. Caller + /// must prove ownership/consumer absence first, then inspect absence; neither + ///404 nor this response settles a journal here. + /// + /// ```no_run + /// # async fn remove(p: &rustack_lambda_core::docker::DockerPort, n: &rustack_lambda_core::docker::DockerKey) { + /// let result = p.remove_volume(n, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn remove_volume( + &self, + name: &DockerKey, + deadline: Instant, + ) -> Result<(), DockerError> { + unit( + self.call( + Operation::RemoveVolume, + Method::DELETE, + format!("/volumes/{}?force=false", escaped(name.as_str())), + Bytes::new(), + StatusCode::NO_CONTENT, + deadline, + ) + .await?, + deadline, + ) + } +} + +fn validate_labels<'a>( + labels: impl Iterator, +) -> Result<(), DockerError> { + for (index, (key, value)) in labels.enumerate() { + if index >= 16 + || key.is_empty() + || key.len() > 128 + || value.len() > 256 + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) + || value.chars().any(char::is_control) + { + return Err(DockerError::Configuration); + } + } + Ok(()) +} +fn validate_volume(volume: &Volume) -> Result<(), DockerError> { + let _: DockerKey = volume.name.parse()?; + if volume.driver.len() > 256 || volume.mountpoint.len() > 4096 || volume.options.len() > 16 { + return Err(DockerError::TooLarge); + } + validate_labels( + volume + .labels + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())), + ) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/volumes/tests.rs b/crates/rustack-lambda-core/src/docker/volumes/tests.rs new file mode 100644 index 00000000..8e501704 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/volumes/tests.rs @@ -0,0 +1,267 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::sync::Semaphore; + +use super::*; +use crate::{ + artifact::Architecture, + docker::{ + DirectoryLease, + code::tests::{elf, prepare, prepare_with, source, zip}, + tests::{Daemon, deadline}, + }, +}; + +fn volume(name: &str, labels: &BTreeMap) -> Value { + json!({"Name":name,"Driver":"local","Mountpoint":"/daemon/private","Labels":labels,"Scope":"local","Options":{}}) +} + +#[tokio::test] +async fn test_should_create_observe_list_and_remove_only_explicit_local_volumes() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let name: DockerKey = "rustack-code-fixture".parse().unwrap(); + let labels = BTreeMap::from([("rustack.lambda.ownerId".into(), "recorded-owner".into())]); + let pending = tokio::spawn({ + let (port, name, labels) = (port.clone(), name.clone(), labels.clone()); + async move { port.create_volume(&name, &labels, deadline()).await } + }); + let request = daemon.requests.recv().await.unwrap(); + assert!(request.head.starts_with("POST /v1.53/volumes/create ")); + let body: Value = serde_json::from_slice(&request.body).unwrap(); + assert_eq!( + body, + json!({"Name":name.as_str(),"Driver":"local","Labels":labels}) + ); + request + .reply( + 201, + &serde_json::to_vec(&volume(name.as_str(), &labels)).unwrap(), + ) + .await; + assert_eq!(pending.await.unwrap().unwrap().name, name.as_str()); + let pending = tokio::spawn({ + let (port, name) = (port.clone(), name.clone()); + async move { port.inspect_volume(&name, deadline()).await } + }); + let request = daemon.requests.recv().await.unwrap(); + assert!( + request + .head + .starts_with("GET /v1.53/volumes/rustack%2Dcode%2Dfixture ") + ); + request + .reply( + 200, + &serde_json::to_vec(&volume(name.as_str(), &labels)).unwrap(), + ) + .await; + pending.await.unwrap().unwrap(); + let filter = OwnerFilter::new(uuid::Uuid::new_v4(), None, None).unwrap(); + let pending = tokio::spawn({ + let port = port.clone(); + async move { port.list_volumes(filter, deadline()).await } + }); + let request = daemon.requests.recv().await.unwrap(); + assert!(request.head.starts_with("GET /v1.53/volumes?filters=")); + assert!(!request.head.contains("limit=")); + assert!(!request.head.contains("all=")); + request.reply(200, br#"{"Volumes":[],"Warnings":[]}"#).await; + assert_eq!(pending.await.unwrap().unwrap(), [] as [Volume; 0]); + let pending = tokio::spawn({ + let (port, name) = (port.clone(), name.clone()); + async move { port.remove_volume(&name, deadline()).await } + }); + let request = daemon.requests.recv().await.unwrap(); + assert!( + request + .head + .starts_with("DELETE /v1.53/volumes/rustack%2Dcode%2Dfixture?force=false ") + ); + request.reply(204, b"").await; + pending.await.unwrap().unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_upload_large_typed_tar_without_relaxing_json_or_response_limits() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let bootstrap = elf(Architecture::Arm64); + let data = vec![b'x'; 2 * 1024 * 1024]; + let (_root, source) = source(zip(&[ + ("bootstrap", &bootstrap, 0o755), + ("large", &data, 0o644), + ])) + .await; + let code = prepare(source).await; + let expected = code.bytes().clone(); + let key: DockerKey = "loader".parse().unwrap(); + let pending = tokio::spawn({ + let port = port.clone(); + async move { port.upload_code(&key, &code, deadline()).await } + }); + let request = daemon.requests.recv().await.unwrap(); + assert!(request.head.starts_with( + "PUT /v1.53/containers/loader/archive?path=%2Fvar%2Ftask&noOverwriteDirNonDir=true&\ + copyUIDGID=false " + )); + assert!( + request + .head + .to_ascii_lowercase() + .contains("content-type: application/x-tar") + ); + assert_eq!(request.body, expected); + request.reply(200, b"").await; + pending.await.unwrap().unwrap(); + assert!(matches!( + port.call( + Operation::CreateVolume, + Method::POST, + "/volumes/create".into(), + Bytes::from(vec![0; 1024 * 1024 + 1]), + StatusCode::CREATED, + deadline() + ) + .await, + Err(DockerError::TooLarge) + )); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_unsafe_labels_before_any_wire_request() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let name = "volume".parse().unwrap(); + for labels in [ + BTreeMap::from([("bad/key".into(), "value".into())]), + BTreeMap::from([("valid".into(), "bad\nvalue".into())]), + BTreeMap::from([("valid".into(), "a".repeat(257))]), + ] { + assert!(matches!( + port.create_volume(&name, &labels, deadline()).await, + Err(DockerError::Configuration) + )); + } + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_accepted_upload_admission_after_its_caller_is_abandoned() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let admission = Arc::new(Semaphore::new(1)); + let bootstrap = elf(Architecture::Arm64); + let (_root, source) = source(zip(&[("bootstrap", &bootstrap, 0o755)])).await; + let code = prepare_with(source, admission.clone()).await; + let pending = tokio::spawn(async move { + let key = "held-loader".parse().unwrap(); + port.upload_code(&key, &code, deadline()).await + }); + let request = daemon.next().await; + assert!(request.head.starts_with("PUT ")); + pending.abort(); + assert!(pending.await.unwrap_err().is_cancelled()); + assert_eq!(admission.available_permits(), 0); + request.reply(200, b"").await; + wire.shutdown().await.unwrap(); + assert_eq!(admission.available_permits(), 1); + daemon.close().await; +} + +#[rstest] +#[case::missing(json!({"Warnings":[]}))] +#[case::null(json!({"Volumes":null}))] +#[case::warning(json!({"Volumes":[],"Warnings":["partial result"]}))] +#[case::row_kind(json!({"Volumes":[false]}))] +#[tokio::test] +async fn test_should_reject_incomplete_or_malformed_volume_inventories(#[case] response: Value) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let scope = OwnerFilter::new(uuid::Uuid::new_v4(), None, None).unwrap(); + let pending = tokio::spawn(async move { port.list_volumes(scope, deadline()).await }); + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&response).unwrap()) + .await; + assert!(pending.await.unwrap().is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case::name("Name",json!("foreign"))] +#[case::driver("Driver",json!("plugin"))] +#[case::scope("Scope",json!("global"))] +#[case::options("Options",json!({"device":"/host/source"}))] +#[case::labels("Labels",json!({"foreign":"SECRET_VALUE"}))] +#[tokio::test] +async fn test_should_reject_volume_create_identity_or_policy_substitution( + #[case] field: &str, + #[case] value: Value, +) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let name: DockerKey = "volume".parse().unwrap(); + let pending = tokio::spawn(async move { + port.create_volume(&name, &BTreeMap::new(), deadline()) + .await + }); + let mut response = volume("volume", &BTreeMap::new()); + response[field] = value; + daemon + .next() + .await + .reply(201, &serde_json::to_vec(&response).unwrap()) + .await; + let error = pending.await.unwrap().unwrap_err(); + assert!(matches!(error, DockerError::Protocol)); + assert!(!format!("{error:?}").contains("SECRET_VALUE")); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_response_caps_and_connection_identity_fences_on_volume_operations() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let name: DockerKey = "volume".parse().unwrap(); + let pending = tokio::spawn({ + let port = port.clone(); + let name = name.clone(); + async move { port.inspect_volume(&name, deadline()).await } + }); + daemon + .next() + .await + .raw(b"HTTP/1.1 200 OK\r\nContent-Length: 1048577\r\nConnection: close\r\n\r\n") + .await; + assert!(matches!(pending.await.unwrap(), Err(DockerError::TooLarge))); + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().to_owned(), deadline()) + .await + .unwrap(); + let bound = port.bind_directory(&directory, deadline()).await.unwrap(); + daemon.identity_status.send_replace(404); + let error = bound.inspect_volume(&name, deadline()).await.unwrap_err(); + assert!(matches!(error, DockerError::IdentityCheck(_))); + assert!(daemon.requests.try_recv().is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/zip.rs b/crates/rustack-lambda-core/src/docker/zip.rs new file mode 100644 index 00000000..e2a8e217 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/zip.rs @@ -0,0 +1,325 @@ +//! Provided-runtime sources and retained per-plan code preparation ownership. + +#[cfg(test)] +use std::sync::atomic::AtomicU8; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use tokio::{ + runtime::Handle, + sync::watch, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + CodeRetirement, CodeVolumeIdentity, CodeVolumePort, DirectoryLease, DockerCodeArchive, + DockerError, DockerKey, PinnedCode, check_deadline, connection::stopped, models::Mount, + ownership::codes::volume_key, +}; +use crate::{ + artifact::{PinnedImage, ValidatedArtifact}, + invocation::CleanupTask, +}; + +/// Exact affiliated OS-image and code-owner inputs; production startup qualifies them. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct DockerZipRuntime { + pub(super) base: PinnedImage, + pub(super) code: CodeVolumePort, +} +impl DockerZipRuntime { + /// Bind a live source and producer to the exact code-profile acquisition. + /// + /// This performs no IO and confers no production readiness. + /// + /// # Errors + /// Rejects foreign acquisitions, older profiles or unavailable ownership. + /// + /// ```no_run + /// # fn source(directory:rustack_lambda_core::docker::DirectoryLease,base:rustack_lambda_core::artifact::PinnedImage,code:rustack_lambda_core::docker::CodeVolumePort)->Result<(),rustack_lambda_core::docker::DockerError>{ + /// let source=rustack_lambda_core::docker::DockerZipRuntime::new(&directory,base,code)?; + /// # Ok(()) } + /// ``` + pub fn new( + directory: &DirectoryLease, + base: PinnedImage, + code: CodeVolumePort, + ) -> Result { + let source = Self { base, code }; + if !directory.code_journaled() || !source.same_scope(directory) { + return Err(DockerError::Configuration); + } + source.available()?; + Ok(source) + } + pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { + self.base.belongs_to(directory) && self.code.same_scope(directory) + } + pub(super) fn live(&self) -> Result<(), DockerError> { + if !self.base.liveness().is_live() || !self.code.live() { + return Err(DockerError::Stopped); + } + Ok(()) + } + pub(super) fn available(&self) -> Result<(), DockerError> { + self.live()?; + if !self.code.status().accepting { + return Err(DockerError::Quarantined); + } + Ok(()) + } +} + +#[derive(Debug, Clone, thiserror::Error)] +pub(super) enum CodeStageError { + #[error("code preparation admission saturated")] + AdmissionBusy, + #[error(transparent)] + Docker(#[from] DockerError), +} + +#[derive(Debug, Default)] +struct Preparation { + ready: Option, + receipt: Option, + outcome: Option>, +} + +#[derive(Debug, TypedBuilder)] +pub(super) struct ZipPlan { + pub(super) source: DockerZipRuntime, + identity: watch::Sender>, + pub(super) volume: DockerKey, + #[builder(default)] + stage: CleanupTask, + state: watch::Sender, + #[builder(default)] + claimed: AtomicBool, + #[builder(default)] + started: AtomicBool, + #[builder(default = AtomicBool::new(true))] + settled: AtomicBool, + #[cfg(test)] + #[builder(default)] + fault: AtomicU8, +} +impl ZipPlan { + #[cfg(test)] + fn fault(&self, point: u8) { + if self + .fault + .compare_exchange(point, 0, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + panic!("deliberate ZIP stage fault"); + } + } + pub(super) fn new( + source: DockerZipRuntime, + identity: CodeVolumeIdentity, + ) -> Result, DockerError> { + let volume = volume_key(identity.environment_id())?; + Ok(Arc::new( + Self::builder() + .source(source) + .identity(watch::channel(Some(identity)).0) + .volume(volume) + .state(watch::channel(Preparation::default()).0) + .build(), + )) + } + pub(super) fn claim(self: &Arc) -> Result { + self.claimed + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| DockerError::Configuration)?; + Ok(AllocationClaim { + plan: self.clone(), + committed: false, + }) + } + pub(super) fn live(&self) -> Result<(), DockerError> { + self.source.live()?; + if self.started.load(Ordering::Acquire) && self.clean() { + return Err(DockerError::Stopped); + } + if self + .state + .borrow() + .ready + .as_ref() + .is_some_and(|ready| !ready.is_live()) + { + return Err(DockerError::Stopped); + } + Ok(()) + } + pub(super) fn ready_mount(&self) -> Result { + self.live()?; + let state = self.state.borrow(); + let ready = state.ready.as_ref().ok_or(DockerError::Configuration)?; + if ready.volume_name() != &self.volume + || ready.base_image_id() != &self.source.base.resolved().metadata().image_id + { + return Err(DockerError::Quarantined); + } + ready.mount() + } + pub(super) fn clean(&self) -> bool { + self.settled.load(Ordering::Acquire) + } + pub(super) fn start( + self: &Arc, + input: StageInput, + runtime: &Handle, + ) -> Result<(), DockerError> { + self.started + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| DockerError::Configuration)?; + self.settled.store(false, Ordering::Release); + let retained = self.clone(); + self.stage.start_on(runtime, move || async move { + let outcome = retained.prepare(input).await; + retained + .state + .send_modify(|state| state.outcome = Some(outcome)); + Ok(()) + }); + Ok(()) + } + async fn prepare(&self, mut input: StageInput) -> Result<(), CodeStageError> { + input.check()?; + self.source.available()?; + let mut identity = None; + self.identity.send_modify(|slot| identity = slot.take()); + let identity = identity.ok_or(DockerError::Configuration)?; + let archive = DockerCodeArchive::prepare( + input.artifact.clone(), + self.source.base.resolved().metadata().architecture, + input.deadline, + ) + .await?; + input.check()?; + #[cfg(test)] + self.fault(1); + let preparation = self + .source + .code + .start_with_receipt( + identity, + archive, + self.source.base.clone(), + input.deadline, + |receipt| { + #[cfg(test)] + self.fault(2); + self.state + .send_modify(|state| state.receipt = Some(receipt)); + #[cfg(test)] + self.fault(3); + }, + ) + .map_err(|error| match error { + DockerError::Busy => CodeStageError::AdmissionBusy, + other => CodeStageError::Docker(other), + })?; + let ready = tokio::select! {biased; + ()=stopped(&mut input.global)=>return Err(DockerError::Cancelled.into()), + ()=stopped(&mut input.stop)=>return Err(DockerError::Cancelled.into()), + ready=preparation.wait()=>ready?, + }; + input.check()?; + if ready.volume_name() != &self.volume + || ready.base_image_id() != &self.source.base.resolved().metadata().image_id + { + return Err(DockerError::Quarantined.into()); + } + self.state.send_modify(|state| state.ready = Some(ready)); + Ok(()) + } + pub(super) async fn wait_ready(&self, deadline: Instant) -> Result<(), CodeStageError> { + timeout_at(deadline, self.stage.wait()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + check_deadline(deadline)?; + self.state + .borrow() + .outcome + .clone() + .ok_or(DockerError::Worker)??; + self.ready_mount()?; + Ok(()) + } + /// Returns a sticky supervisor diagnostic separately from positive retirement. + pub(super) async fn retire(&self, deadline: Instant) -> Result { + let joined = timeout_at(deadline, self.stage.wait()) + .await + .map_err(|_| DockerError::Deadline)?; + let failed = joined.is_err() + || matches!( + self.state.borrow().outcome, + Some(Err(CodeStageError::Docker(DockerError::Worker))) + ); + self.state.send_modify(|state| { + state.ready.take(); + }); + let receipt = self.state.borrow().receipt.clone(); + if let Some(mut receipt) = receipt { + receipt.wait(deadline).await?; + } + // Every accepted preparation publishes above before-enqueue; no receipt + // after this original stage joins therefore proves no acceptance. + check_deadline(deadline)?; + self.settled.store(true, Ordering::Release); + Ok(failed) + } +} + +#[derive(Debug)] +pub(super) struct AllocationClaim { + plan: Arc, + committed: bool, +} +impl AllocationClaim { + pub(super) fn belongs_to(&self, plan: &Arc) -> bool { + Arc::ptr_eq(&self.plan, plan) + } + pub(super) fn commit(&mut self) { + self.committed = true; + } +} +impl Drop for AllocationClaim { + fn drop(&mut self) { + if !self.committed { + self.plan.claimed.store(false, Ordering::Release); + } + } +} + +#[derive(Debug, TypedBuilder)] +pub(super) struct StageInput { + artifact: ValidatedArtifact, + deadline: Instant, + global: watch::Receiver, + stop: watch::Receiver, +} +impl StageInput { + fn check(&self) -> Result<(), DockerError> { + check_deadline(self.deadline)?; + if *self.global.borrow() + || *self.stop.borrow() + || self.global.has_changed().is_err() + || self.stop.has_changed().is_err() + { + return Err(DockerError::Cancelled); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/zip/tests.rs b/crates/rustack-lambda-core/src/docker/zip/tests.rs new file mode 100644 index 00000000..c302626a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/zip/tests.rs @@ -0,0 +1,483 @@ +use std::{sync::Arc, time::Duration}; + +use rstest::rstest; +use rustack_lambda_model::types::ImageConfig; +use serde_json::to_vec; +use tempfile::TempDir; +use tokio::{spawn, sync::mpsc, task::yield_now, time::timeout}; +use uuid::Uuid; + +use super::*; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + docker::{ + CodeVolumeConfig, CodeVolumeIdentity, CodeVolumeManager, DockerExecutorError, DockerPort, + EnvironmentError, EnvironmentManager, EnvironmentPlan, EnvironmentPort, + EnvironmentSettings, Operation, + code::tests::{elf, source, zip}, + code_volume::tests::{ + allocated, base, begin_volume, code, deadline, finish_loader, fixture, retire, + }, + plan::tests::{function_body, image_with, policy, revision}, + }, + error::LambdaServiceError, + executor::ExecutorError, +}; + +async fn plan( + directory: &DirectoryLease, + runtime: DockerZipRuntime, +) -> (TempDir, Arc) { + let (root, code) = source(zip(&[("bootstrap", &elf(Architecture::Arm64), 0o755)])).await; + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Zip(code); + revision.runtime = Some("provided.al2023".into()); + revision.handler = Some("handler.entry".into()); + let plan = EnvironmentPlan::new_zip( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision), + policy(), + runtime, + ) + .unwrap(); + (root, Arc::new(plan)) +} +fn environment(docker: DockerPort, directory: DirectoryLease) -> EnvironmentManager { + EnvironmentManager::start( + docker, + directory, + EnvironmentSettings::builder() + .environments(2) + .io(Duration::from_secs(1)) + .start(Duration::from_secs(10)) + .build() + .try_into() + .unwrap(), + ) + .unwrap() +} +async fn empty(port: &EnvironmentPort) { + timeout(Duration::from_secs(5), async { + while port.status().allocated != 0 { + yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_should_materialize_fixed_provided_body_and_reject_function_creation_before_code_ready() + { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::default(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + let body = function_body(&plan); + assert_eq!(body.entrypoint, Some(vec!["/var/task/bootstrap".into()])); + assert_eq!(body.cmd, Some(vec![])); + assert_eq!(body.user.as_deref(), Some("1000:1000")); + assert_eq!(body.working_dir.as_deref(), Some("/var/task")); + let host = body.host_config.as_ref().unwrap(); + let mounts = host.mounts.as_ref().unwrap(); + assert_eq!(mounts.len(), 1); + let mount = mounts.first().unwrap(); + assert_eq!(mount.target.as_deref(), Some("/var/task")); + assert_eq!(mount.read_only, Some(true)); + assert_eq!(mount.volume_options.as_ref().unwrap().no_copy, Some(true)); + assert!( + body.env + .as_ref() + .unwrap() + .contains(&"_HANDLER=handler.entry".into()) + ); + assert!(matches!( + plan.create_function(&docker, deadline()).await, + Err(DockerError::Configuration) + )); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + code.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_environment_memory_until_cancelled_preparation_physically_retires() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::default(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + let mut owner = environment(docker, directory.clone()); + let port = owner.port(); + let mut handle = port.spawn_shared(plan.clone(), deadline()).unwrap(); + let (request, volume) = begin_volume(&mut daemon, &directory).await; + handle.cancel(); + assert!(matches!( + handle + .shutdown(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + assert_eq!(port.status().allocated, 1); + assert!(port.status().memory_bytes > 0); + assert!(!plan.code_clean()); + assert_eq!(code.port().status().volumes, 1); + request.reply(201, &to_vec(&volume).unwrap()).await; + retire(&mut daemon, &volume).await; + handle.shutdown(deadline()).await.unwrap(); + empty(&port).await; + assert_eq!(port.status().memory_bytes, 0); + assert!(plan.code_clean()); + assert!(matches!( + handle.status().error, + Some(EnvironmentError::Docker(DockerError::Cancelled)) + )); + owner.shutdown(deadline()).await.unwrap(); + code.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case(1)] +#[case(2)] +#[case(3)] +#[tokio::test] +async fn test_should_reap_stage_panic_before_enqueue_without_inventing_acceptance( + #[case] point: u8, +) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::default(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + plan.zip + .as_ref() + .unwrap() + .fault + .store(point, Ordering::Release); + let mut owner = environment(docker, directory.clone()); + let port = owner.port(); + let mut handle = port.spawn_shared(plan.clone(), deadline()).unwrap(); + timeout(Duration::from_secs(5), async { + while !port.status().worker_failed { + yield_now().await; + } + }) + .await + .unwrap(); + assert!(matches!( + handle.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + empty(&port).await; + assert_eq!(port.status().memory_bytes, 0); + assert!(plan.code_clean()); + assert_eq!(code.port().status().volumes, 0); + assert_eq!(directory.code_records(deadline()).await.unwrap(), []); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + code.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_join_original_upload_and_receipt_after_environment_root_loss() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::default(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + let mut owner = environment(docker, directory.clone()); + let port = owner.port(); + let mut handle = port.spawn_shared(plan.clone(), deadline()).unwrap(); + let (volume, inspected) = allocated(&mut daemon, &directory).await; + let upload = daemon.next().await; + assert!(upload.head.starts_with("PUT ")); + owner.abort_test_worker(); + assert!(matches!( + handle + .shutdown(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + assert_eq!(port.status().allocated, 1); + assert!(port.status().memory_bytes > 0); + assert!(!plan.code_clean()); + upload.reply(200, b"").await; + finish_loader(&mut daemon, &inspected).await; + retire(&mut daemon, &volume).await; + assert!(matches!( + handle.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + empty(&port).await; + assert_eq!(port.status().memory_bytes, 0); + assert!(plan.code_clean()); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + code.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_exclude_cross_manager_plan_replay_and_reset_only_unaccepted_claims() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::default(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + let mut first = environment(docker.clone(), directory.clone()); + let mut second = environment(docker, directory.clone()); + let one = first.port(); + let two = second.port(); + let mut queued = one.spawn_shared(plan.clone(), deadline()).unwrap(); + assert!(two.spawn_shared(plan.clone(), deadline()).is_err()); + queued.cancel(); + queued.shutdown(deadline()).await.unwrap(); + empty(&one).await; + assert!(!plan.zip.as_ref().unwrap().claimed.load(Ordering::Acquire)); + let mut committed = two.spawn_shared(plan.clone(), deadline()).unwrap(); + let (request, volume) = begin_volume(&mut daemon, &directory).await; + assert!(one.spawn_shared(plan.clone(), deadline()).is_err()); + committed.cancel(); + request.reply(201, &to_vec(&volume).unwrap()).await; + retire(&mut daemon, &volume).await; + committed.shutdown(deadline()).await.unwrap(); + empty(&two).await; + assert!(one.spawn_shared(plan, deadline()).is_err()); + assert_eq!(one.status().allocated, 0); + assert_eq!(two.status().allocated, 0); + first.shutdown(deadline()).await.unwrap(); + second.shutdown(deadline()).await.unwrap(); + code.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_warm_identity_independent_of_fresh_volume_and_isolate_handler_changes() { + let (_root, directory, daemon, mut wire, docker) = fixture().await; + let mut owner = + CodeVolumeManager::start(docker, directory.clone(), CodeVolumeConfig::default()).unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, owner.port()).unwrap(); + let (_source, original) = plan(&directory, runtime.clone()).await; + let relay = image_with("1000", vec![]); + let second = EnvironmentPlan::new_zip( + directory.clone(), + relay.clone(), + Arc::new(original.revision().clone()), + policy(), + runtime.clone(), + ) + .unwrap(); + assert_eq!(original.warm_key(), second.warm_key()); + assert_ne!(original.function_key(), second.function_key()); + let mut revised = original.revision().clone(); + revised.handler = Some("other.entry".into()); + let changed = EnvironmentPlan::new_zip( + directory.clone(), + relay, + Arc::new(revised), + policy(), + runtime, + ) + .unwrap(); + assert_ne!(original.warm_key(), changed.warm_key()); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest] +#[case::runtime(0)] +#[case::handler(1)] +#[case::architecture(2)] +#[case::image_config(3)] +#[tokio::test] +async fn test_should_reject_incompatible_zip_plan_before_any_effect(#[case] case: u8) { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut owner = + CodeVolumeManager::start(docker, directory.clone(), CodeVolumeConfig::default()).unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, owner.port()).unwrap(); + let (_source, original) = plan(&directory, runtime.clone()).await; + let mut revision = original.revision().clone(); + match case { + 0 => revision.runtime = Some("python3.13".into()), + 1 => revision.handler = Some("bad\nhandler".into()), + 2 => revision.architecture = Architecture::X86_64, + _ => revision.image_config = Some(ImageConfig::default()), + } + assert!( + EnvironmentPlan::new_zip( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision), + policy(), + runtime + ) + .is_err() + ); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_throttle_only_no_enqueue_code_admission_and_release_parent_memory() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let mut code_owner = CodeVolumeManager::start( + docker.clone(), + directory.clone(), + CodeVolumeConfig::new(2, 1, Duration::from_secs(1), Duration::from_secs(2)).unwrap(), + ) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image.clone(), code_owner.port()).unwrap(); + let (_first_source, archive) = code().await; + let first = code_owner + .port() + .start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + archive, + image, + deadline(), + ) + .unwrap(); + let mut first_retirement = first.retirement(); + let (held, volume) = begin_volume(&mut daemon, &directory).await; + let (_source, plan) = plan(&directory, runtime).await; + let mut owner = environment(docker, directory.clone()); + let port = owner.port(); + let mut handle = port.spawn_shared(plan.clone(), deadline()).unwrap(); + let error = handle.wait_ready(deadline()).await.unwrap_err(); + assert!(matches!(error, EnvironmentError::Busy)); + assert!(matches!( + LambdaServiceError::from(ExecutorError::Docker(DockerExecutorError::Environment( + error + ))), + LambdaServiceError::TooManyRequests + )); + handle.shutdown(deadline()).await.unwrap(); + empty(&port).await; + assert_eq!(port.status().memory_bytes, 0); + assert!(plan.code_clean()); + assert_eq!(code_owner.port().status().volumes, 1); + assert_eq!(code_owner.port().status().preparing, 1); + assert!( + timeout(Duration::from_millis(20), daemon.next()) + .await + .is_err() + ); + drop(first); + held.reply(201, &to_vec(&volume).unwrap()).await; + retire(&mut daemon, &volume).await; + first_retirement.wait(deadline()).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + code_owner.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_accepted_nested_wire_busy_as_infrastructure_error_until_retirement() { + let (_root, directory, mut daemon, mut wire, docker) = fixture().await; + let (sender, mut receiver) = mpsc::channel(32); + let actual = docker.raw.business.clone(); + let mut code_docker = docker.clone(); + code_docker.raw.business = sender; + let forward = spawn(async move { + while let Some(message) = receiver.recv().await { + if matches!(message.request.operation, Operation::UploadCode) { + let _ = message.reply.send(Err(DockerError::Busy)); + } else { + actual.send(message).await.unwrap(); + } + } + }); + let mut code_owner = + CodeVolumeManager::start(code_docker, directory.clone(), CodeVolumeConfig::default()) + .unwrap(); + let (image, _stop, _released) = base(&directory); + let runtime = DockerZipRuntime::new(&directory, image, code_owner.port()).unwrap(); + let (_source, plan) = plan(&directory, runtime).await; + let mut owner = environment(docker, directory.clone()); + let port = owner.port(); + let mut handle = port.spawn_shared(plan.clone(), deadline()).unwrap(); + let (volume, inspected) = allocated(&mut daemon, &directory).await; + let error = handle.wait_ready(deadline()).await.unwrap_err(); + assert!(matches!(error, EnvironmentError::Docker(DockerError::Busy))); + assert!(matches!( + LambdaServiceError::from(ExecutorError::Docker(DockerExecutorError::Environment( + error + ))), + LambdaServiceError::Internal { .. } + )); + assert_eq!(port.status().allocated, 1); + assert!(port.status().memory_bytes > 0); + assert!(!plan.code_clean()); + finish_loader(&mut daemon, &inspected).await; + retire(&mut daemon, &volume).await; + handle.shutdown(deadline()).await.unwrap(); + empty(&port).await; + assert_eq!(port.status().memory_bytes, 0); + assert!(plan.code_clean()); + owner.shutdown(deadline()).await.unwrap(); + code_owner.shutdown(deadline()).await.unwrap(); + timeout(Duration::from_secs(3), forward) + .await + .unwrap() + .unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/error.rs b/crates/rustack-lambda-core/src/error.rs index 3a5637b7..5c1cb0f8 100644 --- a/crates/rustack-lambda-core/src/error.rs +++ b/crates/rustack-lambda-core/src/error.rs @@ -3,7 +3,69 @@ //! These errors represent business logic failures and are converted to //! [`LambdaError`] for the HTTP response layer. +#[cfg(unix)] +use std::{error::Error, fmt}; + use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; +#[cfg(unix)] +use tokio::task::JoinError; + +#[cfg(unix)] +use crate::docker::DockerError; + +/// Typed validation worker failure; diagnostics never expose its panic payload. +#[cfg(unix)] +#[non_exhaustive] +pub struct ImageRestoreWorker { + source: JoinError, +} +#[cfg(unix)] +impl From for ImageRestoreWorker { + fn from(source: JoinError) -> Self { + Self { source } + } +} +#[cfg(unix)] +impl ImageRestoreWorker { + /// Whether the original retained worker panicked. + /// + /// ```no_run + /// # fn inspect(error: &rustack_lambda_core::error::ImageRestoreWorker) { let panicked = error.is_panic(); } + /// ``` + #[must_use] + pub fn is_panic(&self) -> bool { + self.source.is_panic() + } + /// Whether the original retained worker was canceled. + /// + /// ```no_run + /// # fn inspect(error: &rustack_lambda_core::error::ImageRestoreWorker) { let cancelled = error.is_cancelled(); } + /// ``` + #[must_use] + pub fn is_cancelled(&self) -> bool { + self.source.is_cancelled() + } +} +#[cfg(unix)] +impl fmt::Debug for ImageRestoreWorker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +#[cfg(unix)] +impl fmt::Display for ImageRestoreWorker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(if self.is_panic() { + "image archive validation worker panicked" + } else { + "image archive validation worker cancelled" + }) + } +} +// Retain JoinError for typed classification; chaining its Display would leak +// arbitrary panic payload through startup's anyhow error rendering. +#[cfg(unix)] +impl Error for ImageRestoreWorker {} /// Internal service error type for Lambda operations. #[derive(Debug, thiserror::Error)] @@ -50,6 +112,10 @@ pub enum LambdaServiceError { message: String, }, + /// An explicit optimistic-update precondition did not match. + #[error("The supplied precondition does not match the current function revision")] + PreconditionFailed, + /// Invalid parameter value. #[error("Invalid parameter: {message}")] InvalidParameter { @@ -82,6 +148,24 @@ pub enum LambdaServiceError { #[error("Docker not available")] DockerNotAvailable, + /// Immutable image archive reacquisition failed without publishing resources. + #[cfg(unix)] + #[error("Lambda immutable image restore failed")] + ImageRestore { + /// Redacted typed daemon/ownership/deadline provenance. + #[source] + source: Box, + }, + + /// Retained image metadata validation worker failed. + #[cfg(unix)] + #[error("Lambda image archive validation worker failed")] + ImageRestoreWorker { + /// Typed worker provenance with redacted panic text. + #[source] + source: ImageRestoreWorker, + }, + /// Event source mapping does not exist. #[error("Event source mapping not found: {uuid}")] EventSourceMappingNotFound { @@ -152,6 +236,10 @@ impl From for LambdaError { LambdaServiceError::ResourceConflict { ref message } => { LambdaError::resource_conflict(message) } + LambdaServiceError::PreconditionFailed => LambdaError::new( + LambdaErrorCode::PreconditionFailedException, + "The supplied precondition does not match the current function revision", + ), LambdaServiceError::InvalidParameter { ref message } => { LambdaError::invalid_parameter(message) } @@ -164,6 +252,11 @@ impl From for LambdaError { LambdaServiceError::ResourceNotReady { ref message } => { LambdaError::new(LambdaErrorCode::ResourceNotReadyException, message.clone()) } + #[cfg(unix)] + LambdaServiceError::ImageRestore { .. } + | LambdaServiceError::ImageRestoreWorker { .. } => { + LambdaError::service_error("Immutable image archive restore failed") + } LambdaServiceError::DockerNotAvailable => LambdaError::service_error( "Docker execution is not available. Set LAMBDA_DOCKER_ENABLED=true to enable.", ), @@ -236,3 +329,18 @@ mod tests { ); } } + +#[cfg(test)] +mod precondition_tests { + use super::*; + + #[test] + fn test_should_encode_stale_revision_as_http_412_with_the_aws_error_code() { + let error: LambdaError = LambdaServiceError::PreconditionFailed.into(); + assert_eq!(error.code.as_str(), "PreconditionFailedException"); + assert_eq!( + error.code.status_code(), + http::StatusCode::PRECONDITION_FAILED + ); + } +} diff --git a/crates/rustack-lambda-core/src/executor/auto.rs b/crates/rustack-lambda-core/src/executor/auto.rs index 2ace648b..5f8e69ff 100644 --- a/crates/rustack-lambda-core/src/executor/auto.rs +++ b/crates/rustack-lambda-core/src/executor/auto.rs @@ -10,9 +10,10 @@ use std::time::Duration; use async_trait::async_trait; use super::{ - Executor, ExecutorError, InvokeRequest, InvokeResponse, NativeExecutor, PackageType, - SquibExecutor, SquibExecutorConfig, + Executor, ExecutorError, InvokeRequest, InvokeResponse, NativeExecutor, SquibExecutor, + SquibExecutorConfig, }; +use crate::{artifact::DeploymentArtifact, invocation::InvocationContext}; /// Executor that chooses the concrete backend for each invocation. #[derive(Debug)] @@ -38,66 +39,60 @@ impl AutoExecutor { } fn should_use_squib(req: &InvokeRequest) -> bool { - cfg!(target_os = "macos") && req.package_type == PackageType::Zip + cfg!(target_os = "macos") && matches!(req.revision.artifact, DeploymentArtifact::Zip(_)) } } #[async_trait] impl Executor for AutoExecutor { - async fn invoke(&self, req: InvokeRequest) -> Result { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + if Self::should_use_squib(request) { + self.squib.available(request) + } else { + self.native.available(request) + } + } + + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { if Self::should_use_squib(&req) { - self.squib.invoke(req).await + self.squib.invoke(req, context).await } else { - self.native.invoke(req).await + self.native.invoke(req, context).await } } - async fn shutdown(&self) { - self.squib.shutdown().await; - self.native.shutdown().await; + async fn shutdown(&self) -> Result<(), ExecutorError> { + let squib = self.squib.shutdown().await; + let native = self.native.shutdown().await; + squib.and(native) } } #[cfg(test)] mod tests { - use std::{collections::HashMap, path::PathBuf, time::Duration}; - - use bytes::Bytes; + use std::sync::Arc; use super::*; - - fn request(package_type: PackageType) -> InvokeRequest { - InvokeRequest { - function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: Some("provided.al2023".to_owned()), - handler: Some("bootstrap".to_owned()), - architectures: vec!["arm64".to_owned()], - package_type, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04")), - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_secs(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, - } - } + use crate::artifact::{Architecture, ResolvedImage}; #[test] fn test_should_default_macos_zip_to_squib() { assert_eq!( - AutoExecutor::should_use_squib(&request(PackageType::Zip)), + AutoExecutor::should_use_squib(&InvokeRequest::fixture()), cfg!(target_os = "macos") ); } #[test] fn test_should_not_use_squib_for_image_package() { - assert!(!AutoExecutor::should_use_squib(&request( - PackageType::Image - ))); + let mut request = InvokeRequest::fixture(); + Arc::make_mut(&mut request.revision).artifact = DeploymentArtifact::Image( + crate::artifact::PinnedImage::fixture(ResolvedImage::fixture(Architecture::Arm64)), + ); + assert!(!AutoExecutor::should_use_squib(&request)); } } diff --git a/crates/rustack-lambda-core/src/executor/error.rs b/crates/rustack-lambda-core/src/executor/error.rs index 78719d73..31a2d80d 100644 --- a/crates/rustack-lambda-core/src/executor/error.rs +++ b/crates/rustack-lambda-core/src/executor/error.rs @@ -1,10 +1,25 @@ //! Errors surfaced by an [`Executor`](super::Executor). +#[cfg(unix)] +use crate::docker::{DockerExecutorError, EnvironmentError}; use crate::error::LambdaServiceError; /// Failure modes for an execution backend. -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] pub enum ExecutorError { + /// Typed image-runtime, request or readiness failure with redacted sources. + #[cfg(unix)] + #[error("{0}")] + Docker(#[from] DockerExecutorError), + /// Caller or owner requested cancellation; settlement is tracked separately. + #[error("invocation cancelled")] + Cancelled, + /// Absolute invocation budget expired. + #[error("invocation deadline exceeded")] + DeadlineExceeded, + /// Cleanup could not be confirmed; ownership remains supervised. + #[error("cleanup not confirmed: {0}")] + CleanupFailed(String), /// No backend is wired up — set `LAMBDA_EXECUTOR=native` (or `docker`). #[error("execution is disabled. Set LAMBDA_EXECUTOR=native or LAMBDA_EXECUTOR=docker")] Disabled, @@ -39,6 +54,19 @@ pub enum ExecutorError { impl From for LambdaServiceError { fn from(e: ExecutorError) -> Self { match e { + #[cfg(unix)] + ExecutorError::Docker(error) => match &error { + DockerExecutorError::Unavailable => Self::DockerNotAvailable, + DockerExecutorError::Environment( + EnvironmentError::Busy | EnvironmentError::MemoryLimit, + ) => Self::TooManyRequests, + _ => Self::Internal { + message: error.to_string(), + }, + }, + ExecutorError::Cancelled | ExecutorError::DeadlineExceeded => Self::ResourceNotReady { + message: e.to_string(), + }, ExecutorError::Disabled => Self::ResourceNotReady { message: "Lambda execution is disabled; explicitly select a trusted execution \ backend" @@ -48,7 +76,8 @@ impl From for LambdaServiceError { | ExecutorError::InvalidCode(m) | ExecutorError::InitFailed(m) | ExecutorError::RuntimeExited(m) - | ExecutorError::Io(m) => Self::Internal { message: m }, + | ExecutorError::Io(m) + | ExecutorError::CleanupFailed(m) => Self::Internal { message: m }, ExecutorError::Timeout(d) => Self::Internal { message: format!("function timed out after {d:?}"), }, diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 573e287f..3493c664 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -1,26 +1,39 @@ //! A single warm execution instance and its pool. //! -//! Used by both the native and Docker backends. Each instance owns its own +//! Used by host bootstrap backends. Each instance owns its own //! [`runtime_api::RuntimeApiHandle`] and a way to start / kill the //! corresponding bootstrap (process or container). Pool entries are scoped //! per `(function_name, qualifier)` because code can diverge between //! versions. use std::{ + mem::take, + net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; +use bytes::Bytes; use dashmap::DashMap; -use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot}; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, HandlerTimeout, Payload}, +}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore, oneshot}, + task::{JoinHandle, JoinSet}, + time::{Instant as TokioInstant, timeout_at}, +}; use tracing::debug; +use typed_builder::TypedBuilder; use super::{ error::ExecutorError, - runtime_api::{self, Job, RuntimeApiHandle, RuntimeResult}, + runtime_api::{self, Invocation, RuntimeApiHandle, RuntimeResult, SessionError}, types::{InvokeRequest, InvokeResponse}, }; +use crate::{artifact::DeploymentArtifact, invocation::InvocationContext}; /// Identifier for the pool slot — a function/version pair. pub(crate) type PoolKey = (String, String); @@ -36,7 +49,8 @@ pub(crate) trait InstanceBackend: Send + Sync + std::fmt::Debug { } /// Spawn a bootstrap pointing at `runtime_api_addr` for the given function. - /// Returns a handle the pool will keep alive until the instance is reaped. + /// Register external ownership before an await after creation. Cancellation + /// before returning a handle must leave no unowned process. async fn spawn( &self, req: &InvokeRequest, @@ -46,8 +60,11 @@ pub(crate) trait InstanceBackend: Send + Sync + std::fmt::Debug { /// Opaque handle to a backend-specific running thing (process or container). /// Drop must clean it up; the pool also calls `kill` on graceful shutdown. +#[async_trait] pub(crate) trait BackendHandleObj: Send + Sync + std::fmt::Debug { fn kill(&mut self); + fn running(&self) -> bool; + async fn wait(&mut self) -> Result<(), ExecutorError>; } /// Wrapper for object-safety + Drop. @@ -58,8 +75,15 @@ impl BackendHandle { Self(Box::new(handle)) } - pub(crate) fn kill(mut self) { + fn running(&self) -> bool { + self.0.running() + } + async fn wait(&mut self) -> Result<(), ExecutorError> { + self.0.wait().await + } + async fn terminate(mut self) -> Result<(), ExecutorError> { self.0.kill(); + self.0.wait().await } } @@ -82,7 +106,52 @@ struct Instance { backend: BackendHandle, last_used: Instant, idle_permit: Option, - init_error: Option>, + init_error: Option>, +} + +impl Instance { + async fn run( + &mut self, + invocation: Invocation, + context: &InvocationContext, + timeout: Duration, + ) -> Result { + let response = async { + let ticket = self + .api + .session() + .submit(invocation, context.deadline()) + .await + .map_err(|error| invocation_error(error, timeout))?; + match self.init_error.take() { + Some(mut init) => tokio::select! { + biased; + result = ticket.wait() => result.map_err(|error| invocation_error(error, timeout)), + error = &mut init => error.map(RuntimeResult::InitError).map_err(|_| ExecutorError::RuntimeExited("Runtime initialization channel closed".into())), + }, + None => ticket + .wait() + .await + .map_err(|error| invocation_error(error, timeout)), + } + }; + tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = response => result, + exit = self.backend.wait() => exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before completing invocation".into()))), + } + } + + async fn retire(mut self) -> Result<(), ExecutorError> { + let api = self + .api + .shutdown() + .await + .map_err(|_| ExecutorError::CleanupFailed("Runtime API shutdown failed".into())); + let backend = self.backend.terminate().await; + backend.and(api) + } } /// Pool of warm instances per `(function, qualifier)` key. @@ -90,7 +159,7 @@ struct Instance { /// `acquire` fast-paths a warm instance and otherwise asks the backend to spawn /// a new one. `release` returns the instance to the pool subject to /// `max_warm`. `reap_idle` evicts instances older than `idle_timeout`. -#[derive(Debug)] +#[derive(Debug, TypedBuilder)] pub(crate) struct InstancePool { backend: Arc, max_warm: usize, @@ -107,188 +176,352 @@ impl InstancePool { idle_timeout: Duration, init_timeout: Duration, ) -> Self { - Self { - backend, - max_warm, - idle_timeout, - init_timeout, - pools: DashMap::new(), - idle_capacity: Arc::new(Semaphore::new(32)), - } + Self::builder() + .backend(backend) + .max_warm(max_warm) + .idle_timeout(idle_timeout) + .init_timeout(init_timeout) + .pools(DashMap::new()) + .idle_capacity(Arc::new(Semaphore::new(32))) + .build() } pub(crate) fn key(req: &InvokeRequest) -> PoolKey { - let mut environment: Vec<_> = req.environment.iter().collect(); + let mut environment: Vec<_> = req.revision.environment.iter().collect(); environment.sort(); + let code_root = match &req.revision.artifact { + DeploymentArtifact::Zip(artifact) => Some(artifact.code_root()), + DeploymentArtifact::Image(_) => None, + }; let identity = format!( "{:?}", ( - &req.qualifier, - &req.code_root, - &req.image_uri, - &req.runtime, - &req.handler, - &req.architectures, + ( + req.revision.resource_id, + &req.revision.revision_id, + &req.revision.qualifier, + req.revision.backend, + ), + (&req.revision.artifact, code_root), + ( + &req.revision.runtime, + &req.revision.handler, + req.revision.architecture + ), environment, - req.timeout, - req.memory_mb + ( + req.revision.timeout, + req.revision.memory_mb, + req.revision.ephemeral_storage_mb + ), + &req.revision.image_config, ) ); ( - req.function_name.clone(), + req.revision.function_name.clone(), crate::storage::compute_sha256(identity.as_bytes()), ) } /// Run a single invocation against an acquired (or freshly spawned) instance. - pub(crate) async fn invoke(&self, req: InvokeRequest) -> Result { + pub(crate) async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + context.check()?; + let attempt: AttemptId = context + .id() + .to_string() + .parse() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation identity".into()))?; + let arn = req + .function_arn + .parse() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation ARN".into()))?; + let trace = local_trace_id() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation clock metadata".into()))?; + let payload = Payload::try_from(req.payload.clone()) + .map_err(|_| ExecutorError::InvalidCode("Invocation payload too large".into()))?; + let handler_timeout = HandlerTimeout::try_from(req.revision.timeout) + .map_err(|_| ExecutorError::InvalidCode("Invalid handler timeout".into()))?; let key = Self::key(&req); - let mut instance = match self.try_acquire(&key) { - Some(inst) => inst, - None => self.spawn_new(&req).await?, - }; - - let request_id = uuid::Uuid::new_v4().to_string(); - let deadline = Instant::now() + req.timeout; - let (resp_tx, resp_rx) = oneshot::channel(); - let job = Job { - request_id: request_id.clone(), - function_arn: req.function_arn.clone(), - deadline, - payload: req.payload.clone(), - response_tx: resp_tx, - }; - instance - .api - .submit(job) - .await - .map_err(|e| ExecutorError::Io(e.to_string()))?; - - let response = async { - match instance.init_error.take() { - Some(mut init) => tokio::select! { - response = resp_rx => response, - error = &mut init => error.map(RuntimeResult::InitError), - }, - None => resp_rx.await, - } - }; - let result = match tokio::time::timeout(req.timeout, response).await { - Ok(Ok(r)) => r, - Ok(Err(_)) => { - // Bootstrap died before responding. - instance.backend.kill(); - return Err(ExecutorError::RuntimeExited( - "bootstrap exited mid-invocation".to_owned(), - )); + let (instance, retired) = self.try_acquire(&key); + retire_all(retired).await?; + let instance = match instance { + Some(instance) if instance.backend.running() && !instance.api.session().is_closed() => { + Some(instance) } - Err(_) => { - instance.backend.kill(); - return Err(ExecutorError::Timeout(req.timeout)); + Some(instance) => { + instance.retire().await?; + None } + None => None, }; - - if !matches!(result, RuntimeResult::InitError(_)) { + let mut instance = match instance { + Some(instance) => instance, + None => self.spawn_new(&req, &context).await?, + }; + let Ok(invocation) = Invocation::new(attempt, context.deadline(), arn, trace, payload) + else { + instance.retire().await?; + context.check()?; + return Err(ExecutorError::Timeout(req.revision.timeout)); + }; + let result = instance + .run( + invocation.with_handler_timeout(handler_timeout), + &context, + req.revision.timeout, + ) + .await; + let result = context.check().and(result); + let reusable = matches!(&result, Ok(RuntimeResult::Success(_))) + && instance.backend.running() + && !instance.api.session().is_closed(); + if reusable { instance.last_used = Instant::now(); - self.release(key, instance); + if let Some(instance) = self.release(key, instance) { + instance.retire().await?; + } + } else { + instance.retire().await?; } - - match result { - RuntimeResult::Success(payload) => Ok(InvokeResponse { + match result? { + RuntimeResult::Success(payload) => Ok(InvokeResponse::success( + Bytes::from(payload), + req.revision.qualifier.clone(), + )), + RuntimeResult::FunctionError(payload) => Ok(InvokeResponse { status: 200, - payload, - function_error: None, + payload: Bytes::from(payload), + function_error: Some("Unhandled".into()), log_tail: None, - executed_version: req.qualifier, + executed_version: req.revision.qualifier.clone(), }), - RuntimeResult::Error(payload) => Ok(InvokeResponse { - status: 200, - payload, - function_error: Some("Unhandled".to_owned()), - log_tail: None, - executed_version: req.qualifier, - }), - RuntimeResult::InitError(payload) => { - let msg = String::from_utf8_lossy(&payload).into_owned(); - Err(ExecutorError::InitFailed(msg)) - } + RuntimeResult::InitError(_) => Err(ExecutorError::InitFailed( + "Bootstrap reported initialization failure".into(), + )), + _ => Err(ExecutorError::RuntimeExited( + "Unsupported runtime result".into(), + )), } } - fn try_acquire(&self, key: &PoolKey) -> Option { - let mut bucket = self.pools.get_mut(&key.0)?; - let Some(index) = bucket.iter().position(|(revision, _)| revision == &key.1) else { - bucket.clear(); - return None; + fn try_acquire(&self, key: &PoolKey) -> (Option, Vec) { + let Some(mut bucket) = self.pools.get_mut(&key.0) else { + return (None, Vec::new()); }; - let (_, mut instance) = bucket.swap_remove(index); - instance.idle_permit.take(); - Some(instance) + let result = if let Some(index) = bucket.iter().position(|(revision, _)| revision == &key.1) + { + let (_, mut instance) = bucket.swap_remove(index); + instance.idle_permit.take(); + (Some(instance), Vec::new()) + } else { + ( + None, + take(&mut *bucket) + .into_iter() + .map(|(_, instance)| instance) + .collect(), + ) + }; + drop(bucket); + self.pools.remove_if(&key.0, |_, bucket| bucket.is_empty()); + result } - fn release(&self, key: PoolKey, mut instance: Instance) { + fn release(&self, key: PoolKey, mut instance: Instance) -> Option { let Ok(permit) = Arc::clone(&self.idle_capacity).try_acquire_owned() else { - return; + return Some(instance); }; - let mut bucket = self.pools.entry(key.0).or_default(); - if bucket.len() < self.max_warm.min(1) { + let mut bucket = self.pools.entry(key.0.clone()).or_default(); + if !self.idle_capacity.is_closed() && bucket.len() < self.max_warm.min(1) { instance.idle_permit = Some(permit); bucket.push((key.1, instance)); + None + } else { + drop(bucket); + self.pools.remove_if(&key.0, |_, bucket| bucket.is_empty()); + Some(instance) } } - async fn spawn_new(&self, req: &InvokeRequest) -> Result { + async fn spawn_new( + &self, + req: &InvokeRequest, + context: &InvocationContext, + ) -> Result { + context.check()?; if !self.backend.has_capacity() { + let mut evicted = None; for mut bucket in self.pools.iter_mut() { if let Some((_, instance)) = bucket.pop() { - drop(instance); + evicted = Some(instance); break; } } + if let Some(instance) = evicted { + instance.retire().await?; + } } - let api = runtime_api::start() - .await - .map_err(|e| ExecutorError::Io(format!("bind runtime api: {e}")))?; + let deadline = TokioInstant::now() + .checked_add(self.init_timeout) + .ok_or_else(|| ExecutorError::InvalidCode("Invalid initialization budget".into()))? + .min(context.deadline()); + let mut api = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = timeout_at(deadline, runtime_api::start_at(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), self.idle_timeout)) => { + result.map_err(|_| initialization_error(SessionError::Deadline, self.init_timeout, context))?.map_err(|_| ExecutorError::InitFailed("Runtime API startup failed".into()))? + } + }; let addr = api.addr(); - let init_error = api.take_init_error_rx().await; - - // Race: backend spawn + first /next poll. We don't observe /next here - // directly — we rely on either submit landing on a polling bootstrap - // OR an `/init/error` arriving. To keep liveness, spawn the backend - // within the init window and watch the init-error channel for a - // fast-fail signal. - let backend = tokio::time::timeout(self.init_timeout, self.backend.spawn(req, addr)) - .await - .map_err(|_| ExecutorError::Timeout(self.init_timeout))??; - let inst = Instance { + let init_error = api.take_initialization_error(); + let backend = tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = timeout_at(deadline, self.backend.spawn(req, addr)) => result.map_err(|_| initialization_error(SessionError::Deadline, self.init_timeout, context)).and_then(|result| result), + }; + let backend = match backend { + Ok(backend) => backend, + Err(error) => { + api.shutdown().await.map_err(|_| { + ExecutorError::CleanupFailed("Runtime API startup cleanup failed".into()) + })?; + return Err(error); + } + }; + let mut instance = Instance { api, backend, last_used: Instant::now(), idle_permit: None, init_error, }; - debug!(function = %req.function_name, addr = %addr, "spawned new lambda instance"); - Ok(inst) + let readiness = { + let ready = async { + match instance.init_error.as_mut() { + Some(init) => tokio::select! { + biased; + error = init => match error { + Ok(_) => Err(ExecutorError::InitFailed("Bootstrap reported initialization failure".into())), + Err(_) => Err(ExecutorError::RuntimeExited("Runtime initialization channel closed".into())), + }, + ready = instance.api.session().bootstrap_ready(deadline) => ready.map_err(|error| initialization_error(error, self.init_timeout, context)), + }, + None => instance + .api + .session() + .bootstrap_ready(deadline) + .await + .map_err(|error| initialization_error(error, self.init_timeout, context)), + } + }; + tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = ready => result, + exit = instance.backend.wait() => exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before first Runtime API poll".into()))), + } + }; + if let Err(error) = readiness { + instance.retire().await?; + return Err(error); + } + debug!(function = %req.revision.function_name, "Lambda bootstrap completed first Runtime API poll"); + Ok(instance) } - /// Reap instances idle for longer than `idle_timeout`. Returns count. - pub(crate) fn reap_idle(&self) -> usize { + /// Reap expired idle instances and join their API/process cleanup. + pub(crate) async fn reap_idle(&self) -> Result { let now = Instant::now(); - let idle = self.idle_timeout; - let mut killed = 0usize; + let mut retired = Vec::new(); for mut bucket in self.pools.iter_mut() { - let before = bucket.len(); - bucket.retain(|(_, instance)| now.duration_since(instance.last_used) <= idle); - killed += before.saturating_sub(bucket.len()); + let mut index = 0; + while index < bucket.len() { + let expired = bucket.get(index).is_some_and(|(_, instance)| { + now.duration_since(instance.last_used) > self.idle_timeout + }); + if expired { + let (_, instance) = bucket.swap_remove(index); + retired.push(instance); + } else { + index += 1; + } + } } self.pools.retain(|_, bucket| !bucket.is_empty()); - killed + let count = retired.len(); + retire_all(retired).await?; + Ok(count) } - /// Drain and kill every instance in every pool. - pub(crate) fn shutdown(&self) { + /// Close warm retention before starting owned asynchronous cleanup. + pub(crate) fn stop_retention(&self) { + self.idle_capacity.close(); + } + + /// Stop warm retention and join every idle API/process owner. + pub(crate) async fn shutdown(&self) -> Result<(), ExecutorError> { + self.idle_capacity.close(); + let mut retired = Vec::new(); + for mut bucket in self.pools.iter_mut() { + retired.extend(take(&mut *bucket).into_iter().map(|(_, instance)| instance)); + } self.pools.clear(); + retire_all(retired).await + } +} + +fn initialization_error( + error: SessionError, + timeout: Duration, + context: &InvocationContext, +) -> ExecutorError { + if let Err(error) = context.check() { + return error; + } + match error { + SessionError::Deadline => ExecutorError::Timeout(timeout), + _ => invocation_error(error, timeout), + } +} + +fn invocation_error(error: SessionError, timeout: Duration) -> ExecutorError { + match error { + SessionError::HandlerTimeout => ExecutorError::Timeout(timeout), + SessionError::Deadline => ExecutorError::DeadlineExceeded, + SessionError::Cancelled => ExecutorError::Cancelled, + SessionError::SupervisorFailed => { + ExecutorError::CleanupFailed("Runtime session supervisor failed".into()) + } + _ => ExecutorError::RuntimeExited("Runtime session is unavailable".into()), + } +} + +async fn retire_all(instances: Vec) -> Result<(), ExecutorError> { + // Idle retention is bounded to32; retire independently so one stalled + // HTTP connection cannot multiply the shutdown budget by the pool size. + let mut tasks = JoinSet::new(); + for instance in instances { + tasks.spawn(instance.retire()); + } + let mut failure = None; + while let Some(result) = tasks.join_next().await { + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => { + failure.get_or_insert(error); + } + Err(_) => { + failure.get_or_insert(ExecutorError::CleanupFailed( + "Idle instance cleanup task failed".into(), + )); + } + } } + failure.map_or(Ok(()), Err) } /// Spawn a periodic idle-reaper background task. Stops when `cancel` flips. @@ -296,20 +529,87 @@ pub(crate) fn spawn_reaper( pool: Arc, interval: Duration, mut cancel: tokio::sync::watch::Receiver, -) -> tokio::task::JoinHandle<()> { +) -> JoinHandle> { tokio::spawn(async move { let mut tick = tokio::time::interval(interval); tick.tick().await; // skip the immediate tick loop { tokio::select! { - _ = cancel.changed() => break, + biased; + () = async { let _ = cancel.wait_for(|stopped| *stopped).await; } => break, _ = tick.tick() => { - let n = pool.reap_idle(); - if n > 0 { - debug!(reaped = n, "lambda idle reaper"); + match pool.reap_idle().await { + Ok(n) if n > 0 => debug!(reaped = n, "lambda idle reaper"), + Ok(_) => {}, + Err(error) => { pool.idle_capacity.close(); return Err(error); } } } } } + Ok(()) }) } + +#[cfg(test)] +mod tests { + use rustack_lambda_model::types::ImageConfig; + + use super::*; + + #[test] + fn test_should_key_warm_environment_by_resource_and_captured_configuration() { + let request = InvokeRequest::fixture(); + let original = InstancePool::key(&request); + let mut alias = request.clone(); + alias.function_arn.push_str(":prod"); + assert_eq!(InstancePool::key(&alias), original); + + let mut recreated = request.clone(); + Arc::make_mut(&mut recreated.revision).resource_id = + crate::identity::FunctionResourceId::default(); + assert_ne!(InstancePool::key(&recreated), original); + let mut updated = request.clone(); + Arc::make_mut(&mut updated.revision).revision_id = "new-revision".into(); + assert_ne!(InstancePool::key(&updated), original); + let mut rehydrated = request.clone(); + Arc::make_mut(&mut rehydrated.revision).artifact = + DeploymentArtifact::Zip(crate::artifact::ValidatedArtifact::fixture()); + assert_ne!(InstancePool::key(&rehydrated), original); + let mut storage = request.clone(); + Arc::make_mut(&mut storage.revision).ephemeral_storage_mb = 1024; + assert_ne!(InstancePool::key(&storage), original); + let mut image = request.clone(); + Arc::make_mut(&mut image.revision).image_config = Some(ImageConfig { + command: Some(vec![]), + ..ImageConfig::default() + }); + assert_ne!(InstancePool::key(&image), original); + let explicitly_empty = InstancePool::key(&image); + Arc::make_mut(&mut image.revision).image_config = Some(ImageConfig::default()); + assert_ne!(InstancePool::key(&image), explicitly_empty); + } + + #[test] + fn test_should_fingerprint_environment_independently_of_hashmap_iteration_order() { + let mut first = InvokeRequest::fixture(); + Arc::make_mut(&mut first.revision) + .environment + .insert("FIRST".into(), "one".into()); + Arc::make_mut(&mut first.revision) + .environment + .insert("SECOND".into(), "two".into()); + let mut second = first.clone(); + Arc::make_mut(&mut second.revision).environment.clear(); + Arc::make_mut(&mut second.revision) + .environment + .insert("SECOND".into(), "two".into()); + Arc::make_mut(&mut second.revision) + .environment + .insert("FIRST".into(), "one".into()); + assert_eq!(InstancePool::key(&first), InstancePool::key(&second)); + Arc::make_mut(&mut second.revision) + .environment + .insert("SECOND".into(), "changed".into()); + assert_ne!(InstancePool::key(&first), InstancePool::key(&second)); + } +} diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index aec3e863..9452f1eb 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -2,15 +2,16 @@ //! //! Routes execution through an object-safe backend with explicit availability. //! -//! - [`NoopExecutor`] rejects execution in Disabled/unsupported Docker modes. +//! - [`NoopExecutor`] rejects execution in Disabled/unqualified Docker modes. //! - [`AutoExecutor`] explicitly opts into Squib on macOS Zip functions, native otherwise. //! - [`NativeExecutor`] runs trusted host-matching bootstraps without isolation. //! - [`SquibExecutor`] runs arm64 Zip functions through a microVM guest agent. //! -//! Docker execution is not supported; it never falls back to native or successful echo. +//! On Unix, `DockerExecutor` composes the retained image runtime using a trusted +//! readiness port. Ordinary CLI Docker selection requires the startup coordinator. //! -//! All backends share a single in-process Lambda Runtime API server (Phase 2) -//! so the bootstrap-side protocol is identical to AWS. +//! Native instances expose a host Runtime API socket; Squib delegates the +//! bootstrap-side protocol to its guest agent. //! //! `async-trait` is required because `RustackLambda` stores the executor as //! `Arc` for backend swapping at startup; the trait must be @@ -31,19 +32,111 @@ pub use error::ExecutorError; pub use native::NativeExecutor; pub use noop::NoopExecutor; pub use squib::{SquibExecutor, SquibExecutorConfig}; -pub use types::{ExecutorBackend, InvokeRequest, InvokeResponse, PackageType}; +pub use types::{ExecutionRevision, ExecutorBackend, InvokeRequest, InvokeResponse}; + +#[cfg(unix)] +pub use crate::docker::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, +}; +use crate::{artifact::Architecture, invocation::InvocationContext}; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. #[async_trait] pub trait Executor: std::fmt::Debug + Send + Sync + 'static { + /// Verify image execution readiness before admitting asynchronous deployment. + /// + /// Backends must explicitly opt in after their Docker runtime probes succeed. + /// + /// # Errors + /// The default rejects image deployment for unsupported or disabled backends. + /// + /// ``` + /// # fn check(executor: &dyn rustack_lambda_core::executor::Executor) { + /// let ready = executor.image_available().is_ok(); + /// # } + /// ``` + fn image_available(&self) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported( + "Image execution is unavailable".into(), + )) + } + + /// Configured Docker ZIP platform for bounded archive profile selection. + /// + /// This does not grant readiness; callers must separately use + /// [`Self::zip_available`] before preparation and publication. + /// + /// ``` + /// # fn profile(executor: &dyn rustack_lambda_core::executor::Executor) { + /// let configured = executor.docker_zip_architecture(); + /// # } + /// ``` + fn docker_zip_architecture(&self) -> Option { + None + } + + /// Check optional provided-runtime ZIP support for a captured architecture. + /// + /// # Errors + /// The default rejects capability which this backend has not installed. + /// + /// ``` + /// # fn check(executor:&dyn rustack_lambda_core::executor::Executor) { + /// let available=executor.zip_available(rustack_lambda_core::artifact::Architecture::Arm64).is_ok(); + /// # } + /// ``` + fn zip_available(&self, _architecture: Architecture) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported( + "ZIP execution is unavailable".into(), + )) + } + /// Check whether execution is enabled before accepting asynchronous work. - fn available(&self) -> Result<(), ExecutorError> { + /// + /// # Errors + /// Returns an unavailable-backend or incompatible-artifact/configuration error. + /// + /// ``` + /// # use rustack_lambda_core::executor::{Executor, InvokeRequest}; + /// # fn probe(executor: &dyn Executor, request: &InvokeRequest) { + /// let available = executor.available(request).is_ok(); + /// # } + /// ``` + fn available(&self, _request: &InvokeRequest) -> Result<(), ExecutorError> { Ok(()) } /// Run the function and return its response. - async fn invoke(&self, req: InvokeRequest) -> Result; + /// + /// The work supervisor retains this future through cancellation and cleanup. + /// + /// # Errors + /// Returns invocation, initialization, transport or cleanup errors. A clean + /// outcome requires backend resources to be reusable or confirmed retired. + /// + /// ```no_run + /// # use rustack_lambda_core::{executor::{Executor, InvokeRequest}, invocation::InvocationContext}; + /// # async fn invoke(executor: &dyn Executor, request: InvokeRequest, context: InvocationContext) { + /// let result = executor.invoke(request, context).await; + /// # } + /// ``` + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result; /// Stop all warm instances and release resources. - async fn shutdown(&self); + /// + /// # Errors + /// Returns `CleanupFailed` if owned resource cleanup cannot be confirmed. + /// Dropping a shutdown waiter must retain the underlying cleanup operation. + /// + /// ```no_run + /// # async fn stop(executor: &dyn rustack_lambda_core::executor::Executor) { + /// let clean = executor.shutdown().await.is_ok(); + /// # } + /// ``` + async fn shutdown(&self) -> Result<(), ExecutorError>; } diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 38310114..22694953 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -5,39 +5,83 @@ //! match the host. The auto executor keeps macOS Zip functions on Squib and //! leaves unsupported image packages for the future Docker backend. -use std::{net::SocketAddr, path::Path, process::Stdio, sync::Arc, time::Duration}; +use std::{ + future::poll_fn, + net::SocketAddr, + path::Path, + process::Stdio, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; use async_trait::async_trait; use parking_lot::Mutex as PMutex; use tokio::{ io::AsyncReadExt, process::Command, + runtime::Handle, sync::{Semaphore, oneshot, watch}, task::JoinSet, }; use tracing::{debug, warn}; use super::{ - Executor, ExecutorError, InvokeRequest, InvokeResponse, PackageType, + Executor, ExecutorError, InvokeRequest, InvokeResponse, instance::{BackendHandle, BackendHandleObj, InstanceBackend, InstancePool, spawn_reaper}, }; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + invocation::{CleanupTask, InvocationContext, Settlement}, +}; /// Native process executor. #[derive(Debug)] pub struct NativeExecutor { pool: Arc, cancel_tx: watch::Sender, - reaper: PMutex>>, - processes: Arc>>, + reaper: PMutex>>>, + processes: Arc, + cleanup: CleanupTask, +} + +#[derive(Debug)] +struct NativeProcesses { + tasks: PMutex>, + cleanup: CleanupTask, + accepting: AtomicBool, + capacity: Arc, + runtime: Handle, +} + +impl NativeProcesses { + fn start_cleanup(self: &Arc) { + let processes = Arc::clone(self); + self.cleanup.start_on(&self.runtime, move || async move { + let mut failed = false; + while let Some(result) = + poll_fn(|context| processes.tasks.lock().poll_join_next(context)).await + { + if result.is_err() { + failed = true; + } + } + if failed { + Err(ExecutorError::CleanupFailed( + "Native process supervisor failed".into(), + )) + } else { + Ok(()) + } + }); + } } impl Drop for NativeExecutor { fn drop(&mut self) { - self.pool.shutdown(); - if let Some(reaper) = self.reaper.lock().take() { - reaper.abort(); - } - self.processes.lock().abort_all(); + self.start_cleanup(); } } @@ -45,10 +89,17 @@ impl NativeExecutor { /// Build a new native executor. #[must_use] pub fn new(max_warm: usize, idle_timeout: Duration, init_timeout: Duration) -> Self { - let processes = Arc::new(PMutex::new(JoinSet::new())); + let processes = Arc::new(NativeProcesses { + tasks: PMutex::new(JoinSet::new()), + cleanup: CleanupTask::default(), + accepting: AtomicBool::new(true), + capacity: Arc::new(Semaphore::new(32)), + runtime: Handle::current(), + }); + let (cancel_tx, cancel_rx) = watch::channel(false); let backend = Arc::new(NativeBackend { processes: Arc::clone(&processes), - capacity: Arc::new(Semaphore::new(32)), + shutdown: cancel_rx.clone(), }); let pool = Arc::new(InstancePool::new( backend, @@ -56,32 +107,80 @@ impl NativeExecutor { idle_timeout, init_timeout, )); - let (cancel_tx, cancel_rx) = watch::channel(false); let reaper = spawn_reaper(Arc::clone(&pool), Duration::from_secs(30), cancel_rx); Self { pool, cancel_tx, reaper: PMutex::new(Some(reaper)), processes, + cleanup: CleanupTask::default(), } } + + fn start_cleanup(&self) { + self.processes.accepting.store(false, Ordering::Release); + self.cancel_tx.send_replace(true); + self.pool.stop_retention(); + self.cleanup.start_on(&self.processes.runtime, || { + let pool = Arc::clone(&self.pool); + let processes = Arc::clone(&self.processes); + let reaper = self.reaper.lock().take(); + async move { + processes.start_cleanup(); + let reaper = match reaper { + Some(reaper) => reaper + .await + .map_err(|_| ExecutorError::CleanupFailed("Native reaper failed".into())) + .and_then(|result| result), + None => Ok(()), + }; + let pool = pool.shutdown().await; + let children = processes.cleanup.wait().await; + reaper.and(pool).and(children) + } + }); + } } #[async_trait] impl Executor for NativeExecutor { - async fn invoke(&self, req: InvokeRequest) -> Result { - if req.package_type != PackageType::Zip { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + if !self.processes.accepting.load(Ordering::Acquire) { + return Err(ExecutorError::Cancelled); + } + if !matches!(request.revision.artifact, DeploymentArtifact::Zip(_)) { return Err(ExecutorError::Unsupported( - "native backend only supports Zip packages".to_owned(), + "native backend only supports Zip packages".into(), )); } - let code_root = req - .code_root - .as_ref() - .ok_or_else(|| ExecutorError::InvalidCode("missing code root".to_owned()))?; + if request.revision.architecture.as_str() != host_architecture() { + return Err(ExecutorError::Unsupported( + "native backend requires the host architecture".into(), + )); + } + Ok(()) + } + + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.available(&req)?; + context.check()?; + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { + return Err(ExecutorError::Unsupported( + "native backend only supports Zip packages".into(), + )); + }; + let code_root = artifact.code_root(); let bootstrap = code_root.join("bootstrap"); - ensure_executable(&bootstrap).await?; - if !bootstrap_runs_on_host(&bootstrap, &req.architectures).await? { + let supported = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = async { ensure_executable(&bootstrap).await?; bootstrap_runs_on_host(&bootstrap, req.revision.architecture).await } => result?, + }; + if !supported { return Err(ExecutorError::Unsupported(format!( "bootstrap {} cannot run on host {}/{}; use docker backend", bootstrap.display(), @@ -89,51 +188,41 @@ impl Executor for NativeExecutor { std::env::consts::ARCH, ))); } - self.pool.invoke(req).await + self.pool.invoke(req, context).await } - async fn shutdown(&self) { - let _ = self.cancel_tx.send(true); - let reaper = self.reaper.lock().take(); - if let Some(r) = reaper { - if let Err(error) = r.await { - warn!(%error, "Lambda idle reaper failed"); - } - } - self.pool.shutdown(); - let mut processes = std::mem::take(&mut *self.processes.lock()); - while let Some(result) = processes.join_next().await { - if let Err(error) = result { - warn!(%error, "Lambda process supervisor failed"); - } - } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.start_cleanup(); + self.cleanup.wait().await } } #[derive(Debug)] struct NativeBackend { - processes: Arc>>, - capacity: Arc, + processes: Arc, + shutdown: watch::Receiver, } #[async_trait] impl InstanceBackend for NativeBackend { fn has_capacity(&self) -> bool { - self.capacity.available_permits() != 0 + self.processes.capacity.available_permits() != 0 } async fn spawn( &self, req: &InvokeRequest, runtime_api_addr: SocketAddr, ) -> Result { - let code_root = req - .code_root - .clone() - .ok_or_else(|| ExecutorError::InvalidCode("missing code root".to_owned()))?; + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { + return Err(ExecutorError::Unsupported( + "native backend only supports Zip packages".into(), + )); + }; + let code_root = artifact.code_root(); let bootstrap = code_root.join("bootstrap"); let mut cmd = Command::new(&bootstrap); - cmd.current_dir(&code_root) + cmd.current_dir(code_root) .env_clear() .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -143,6 +232,14 @@ impl InstanceBackend for NativeBackend { cmd.env(k, v); } + let permit = Arc::clone(&self.processes.capacity) + .acquire_owned() + .await + .map_err(|_| ExecutorError::Cancelled)?; + let mut processes = self.processes.tasks.lock(); + if !self.processes.accepting.load(Ordering::Acquire) { + return Err(ExecutorError::Cancelled); + } let mut child = cmd .spawn() .map_err(|e| ExecutorError::Io(format!("spawn bootstrap: {e}")))?; @@ -157,22 +254,47 @@ impl InstanceBackend for NativeBackend { logs.spawn(drain_to_buf(stderr, log_buf, "stderr")); } let (cancel, cancelled) = oneshot::channel(); - let mut processes = self.processes.lock(); + let artifact_lease = artifact.clone(); while let Some(result) = processes.try_join_next() { if let Err(error) = result { warn!(%error, "Lambda process supervisor failed"); } } + let (completion, completed) = watch::channel(Settlement::Pending); + let mut shutdown = self.shutdown.clone(); processes.spawn(async move { - tokio::select! { - _ = cancelled => { if let Err(error) = child.kill().await { warn!(%error, "Lambda process cancellation failed"); } }, - result = child.wait() => { if let Err(error) = result { warn!(%error, "Lambda process wait failed"); } }, + { + let _artifact_lease = artifact_lease; + let _permit = permit; + let result = tokio::select! { + biased; + () = async { let _ = shutdown.wait_for(|stopped| *stopped).await; } => child.kill().await, + _ = cancelled => child.kill().await, + result = child.wait() => result.map(|_| ()), + }; + if result.is_err() { + completion.send_replace(Settlement::Uncertain); + // Preserve child/lease/allocation ownership until reaping is confirmed. + loop { + let _ = child.start_kill(); + match child.wait().await { + Ok(_) => break, + Err(_) => tokio::time::sleep(Duration::from_secs(1)).await, + } + } + } + logs.abort_all(); + while let Some(result) = logs.join_next().await { + if result.is_err_and(|error| !error.is_cancelled()) { + warn!("Native log task failed"); + } + } } - logs.abort_all(); - while let Some(result) = logs.join_next().await { if let Err(error) = result { if !error.is_cancelled() { warn!(%error, "Lambda log task failed"); } } } + completion.send_replace(Settlement::Clean); }); Ok(BackendHandle::new(NativeHandle { cancel: Some(cancel), + completion: completed, })) } } @@ -180,9 +302,28 @@ impl InstanceBackend for NativeBackend { #[derive(Debug)] struct NativeHandle { cancel: Option>, + completion: watch::Receiver, } +#[async_trait] impl BackendHandleObj for NativeHandle { + fn running(&self) -> bool { + *self.completion.borrow() == Settlement::Pending + } + + async fn wait(&mut self) -> Result<(), ExecutorError> { + match self + .completion + .wait_for(|state| *state != Settlement::Pending) + .await + { + Ok(state) if *state == Settlement::Clean => Ok(()), + _ => Err(ExecutorError::CleanupFailed( + "Native child exit is not confirmed".into(), + )), + } + } + fn kill(&mut self) { if let Some(cancel) = self.cancel.take() { if cancel.send(()).is_err() { @@ -201,16 +342,11 @@ impl Drop for NativeHandle { /// Check declared architecture and executable format with fallible asynchronous IO. async fn bootstrap_runs_on_host( path: &Path, - declared_archs: &[String], + architecture: Architecture, ) -> Result { // Architecture check first — declared `architectures` must include the // host arch. - let host_arch = match std::env::consts::ARCH { - "x86_64" => "x86_64", - "aarch64" => "arm64", - other => other, - }; - if !declared_archs.iter().any(|a| a == host_arch) { + if architecture.as_str() != host_architecture() { return Ok(false); } let mut file = tokio::fs::File::open(path) @@ -230,6 +366,13 @@ async fn bootstrap_runs_on_host( Ok((elf && host_is_linux) || (macho && host_is_macos)) } +fn host_architecture() -> &'static str { + match std::env::consts::ARCH { + "aarch64" => "arm64", + other => other, + } +} + /// Require a regular executable bootstrap without modifying immutable artifacts. #[cfg(unix)] async fn ensure_executable(path: &Path) -> Result<(), ExecutorError> { @@ -251,32 +394,40 @@ async fn ensure_executable(_path: &Path) -> Result<(), ExecutorError> { } fn lambda_env_vars(req: &InvokeRequest, runtime_api: SocketAddr) -> Vec<(String, String)> { - let mut env = Vec::with_capacity(16 + req.environment.len()); - let task_root = req - .code_root - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_default(); + let mut env = Vec::with_capacity(16 + req.revision.environment.len()); + let task_root = match &req.revision.artifact { + DeploymentArtifact::Zip(artifact) => artifact.code_root().display().to_string(), + DeploymentArtifact::Image(_) => String::new(), + }; env.push(("AWS_LAMBDA_RUNTIME_API".into(), runtime_api.to_string())); - env.push(("AWS_LAMBDA_FUNCTION_NAME".into(), req.function_name.clone())); - env.push(("AWS_LAMBDA_FUNCTION_VERSION".into(), req.qualifier.clone())); + env.push(( + "AWS_LAMBDA_FUNCTION_NAME".into(), + req.revision.function_name.clone(), + )); + env.push(( + "AWS_LAMBDA_FUNCTION_VERSION".into(), + req.revision.qualifier.clone(), + )); env.push(( "AWS_LAMBDA_FUNCTION_MEMORY_SIZE".into(), - req.memory_mb.to_string(), + req.revision.memory_mb.to_string(), )); env.push(( "AWS_LAMBDA_FUNCTION_TIMEOUT".into(), - req.timeout.as_secs().to_string(), + req.revision.timeout.as_secs().to_string(), )); env.push(( "AWS_LAMBDA_LOG_GROUP_NAME".into(), - format!("/aws/lambda/{}", req.function_name), + format!("/aws/lambda/{}", req.revision.function_name), )); env.push(( "AWS_LAMBDA_LOG_STREAM_NAME".into(), uuid::Uuid::new_v4().to_string(), )); - env.push(("_HANDLER".into(), req.handler.clone().unwrap_or_default())); + env.push(( + "_HANDLER".into(), + req.revision.handler.clone().unwrap_or_default(), + )); env.push(("LAMBDA_TASK_ROOT".into(), task_root.clone())); env.push(("LAMBDA_RUNTIME_DIR".into(), task_root)); env.push(("TZ".into(), ":UTC".into())); @@ -286,7 +437,7 @@ fn lambda_env_vars(req: &InvokeRequest, runtime_api: SocketAddr) -> Vec<(String, env.push(("AWS_SECRET_ACCESS_KEY".into(), "test".into())); env.push(("AWS_SESSION_TOKEN".into(), "test".into())); // User env overrides last, but do not let it shadow the runtime API var. - for (k, v) in &req.environment { + for (k, v) in &req.revision.environment { if k == "AWS_LAMBDA_RUNTIME_API" || k == "_HANDLER" || k == "LAMBDA_TASK_ROOT" { warn!(key = %k, "ignoring user override for reserved Lambda env var"); continue; @@ -331,7 +482,14 @@ where match reader.read(&mut chunk).await { Ok(0) => break, Ok(n) => { - buf.lock().push(&chunk[..n]); + let Some(bytes) = chunk.get(..n) else { + warn!( + stream = label, + "lambda runtime reader returned an invalid byte count" + ); + break; + }; + buf.lock().push(bytes); debug!(stream = label, bytes = n, "lambda runtime output"); } Err(e) => { diff --git a/crates/rustack-lambda-core/src/executor/noop.rs b/crates/rustack-lambda-core/src/executor/noop.rs index 7d052c74..8c0d3465 100644 --- a/crates/rustack-lambda-core/src/executor/noop.rs +++ b/crates/rustack-lambda-core/src/executor/noop.rs @@ -2,11 +2,12 @@ use async_trait::async_trait; use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; +use crate::invocation::InvocationContext; /// Explicit unavailable backend for metadata-only operation. #[derive(Debug, Default, Clone)] pub struct NoopExecutor { - unsupported_docker: bool, + unqualified_docker: bool, } impl NoopExecutor { /// Construct a disabled executor. @@ -14,14 +15,14 @@ impl NoopExecutor { pub fn new() -> Self { Self::default() } - pub(crate) fn docker_unavailable() -> Self { + pub(crate) fn without_docker_runtime() -> Self { Self { - unsupported_docker: true, + unqualified_docker: true, } } fn error(&self) -> ExecutorError { - if self.unsupported_docker { - ExecutorError::Unsupported("Docker execution is not implemented in this build".into()) + if self.unqualified_docker { + ExecutorError::Unsupported("qualified Docker startup has not been installed".into()) } else { ExecutorError::Disabled } @@ -29,11 +30,17 @@ impl NoopExecutor { } #[async_trait] impl Executor for NoopExecutor { - fn available(&self) -> Result<(), ExecutorError> { + fn available(&self, _request: &InvokeRequest) -> Result<(), ExecutorError> { Err(self.error()) } - async fn invoke(&self, _req: InvokeRequest) -> Result { + async fn invoke( + &self, + _req: InvokeRequest, + _context: InvocationContext, + ) -> Result { Err(self.error()) } - async fn shutdown(&self) {} + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } } diff --git a/crates/rustack-lambda-core/src/executor/runtime_api.rs b/crates/rustack-lambda-core/src/executor/runtime_api.rs index c32e7b95..826d19c5 100644 --- a/crates/rustack-lambda-core/src/executor/runtime_api.rs +++ b/crates/rustack-lambda-core/src/executor/runtime_api.rs @@ -1,518 +1,8 @@ -//! In-process Lambda Runtime API server. +//! Shared Lambda Runtime API reexports for existing executor consumers. //! -//! Each warm runtime instance (process or container) gets its own socket bound -//! on `127.0.0.1:0`. The bootstrap inside the instance speaks the standard -//! Lambda Runtime API to it: -//! -//! - `GET /2018-06-01/runtime/invocation/next` — long-poll for the next job. -//! - `POST /2018-06-01/runtime/invocation/{id}/response` — deliver success. -//! - `POST /2018-06-01/runtime/invocation/{id}/error` — deliver function error. -//! - `POST /2018-06-01/runtime/init/error` — fatal init failure. -//! -//! Per AWS, an instance handles one invocation at a time. The receiver is -//! locked once per `/next` call so concurrent polls (which shouldn't happen -//! in practice) serialize cleanly. - -use std::{ - convert::Infallible, - net::SocketAddr, - sync::Arc, - time::{Instant, SystemTime, UNIX_EPOCH}, -}; +//! Native and relay adapters use the same bounded runtime session implementation. -use bytes::Bytes; -use dashmap::DashMap; -use http_body_util::{BodyExt, Full, Limited}; -use hyper::{ - Method, Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, +pub use rustack_lambda_runtime::{ + http::{ApiError, RuntimeApiHandle, start, start_at}, + session::{Invocation, InvocationTicket, RuntimeResult, SessionError}, }; -use hyper_util::rt::TokioIo; -use tokio::{ - net::TcpListener, - sync::{Mutex, mpsc, oneshot, watch}, - task::{JoinHandle, JoinSet}, -}; -use tracing::{debug, warn}; - -/// One unit of work for a runtime instance: payload in, result out. -#[derive(Debug)] -pub struct Job { - /// Synthetic AWS request id (UUID). - pub request_id: String, - /// Function ARN reported via `Lambda-Runtime-Invoked-Function-Arn`. - pub function_arn: String, - /// Wall-clock deadline (used for `Lambda-Runtime-Deadline-Ms`). - pub deadline: Instant, - /// Body delivered as the invocation event. - pub payload: Bytes, - /// Where to send the bootstrap's response (or error). - pub response_tx: oneshot::Sender, -} - -/// Result of a runtime round-trip. -#[derive(Debug, Clone)] -pub enum RuntimeResult { - /// Bootstrap posted to `/response`. - Success(Bytes), - /// Bootstrap posted to `/error`. - Error(Bytes), - /// Bootstrap posted to `/init/error` before invocation. - InitError(Bytes), -} - -/// Handle to a running runtime API socket. Drop or call `shutdown` to stop. -#[derive(Debug)] -pub struct RuntimeApiHandle { - addr: SocketAddr, - job_tx: mpsc::Sender, - shutdown_tx: watch::Sender, - accept_task: Option>, - /// Optional channel that the orchestrator subscribes to in order to be - /// notified of `/init/error` failures. - init_error_rx: Mutex>>, -} - -impl RuntimeApiHandle { - /// Bound socket address. - #[must_use] - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Submit a new invocation job. Fails if the runtime task has stopped. - pub async fn submit(&self, job: Job) -> Result<(), &'static str> { - self.job_tx - .send(job) - .await - .map_err(|_| "runtime api task is gone") - } - - /// Take the init-error receiver. Returns `Some` exactly once. - pub async fn take_init_error_rx(&self) -> Option> { - self.init_error_rx.lock().await.take() - } - - /// Stop the accept loop and wait for it to exit. - pub async fn shutdown(&mut self) { - let _ = self.shutdown_tx.send(true); - if let Some(task) = self.accept_task.take() { - let _ = task.await; - } - } -} - -impl Drop for RuntimeApiHandle { - fn drop(&mut self) { - let _ = self.shutdown_tx.send(true); - if let Some(task) = self.accept_task.take() { - task.abort(); - } - } -} - -/// State shared across connection handlers. -#[derive(Debug)] -struct State { - /// Single-consumer queue of pending jobs. - job_rx: Mutex>, - /// `request_id` -> `oneshot::Sender` for the in-flight job. - pending: DashMap>, - /// One-shot to signal an init-error to the orchestrator. - init_error_tx: Mutex>>, -} - -/// Bind a fresh runtime API socket and return a handle. -/// -/// The bootstrap process should be started **after** this returns, with -/// `AWS_LAMBDA_RUNTIME_API` set to `handle.addr()`. -pub async fn start() -> std::io::Result { - let listener = TcpListener::bind(("127.0.0.1", 0)).await?; - let addr = listener.local_addr()?; - - let (job_tx, job_rx) = mpsc::channel::(8); - let (shutdown_tx, mut shutdown_rx) = watch::channel(false); - let (init_err_tx, init_err_rx) = oneshot::channel::(); - - let state = Arc::new(State { - job_rx: Mutex::new(job_rx), - pending: DashMap::new(), - init_error_tx: Mutex::new(Some(init_err_tx)), - }); - - let accept_task = tokio::spawn(async move { - let mut connections = JoinSet::new(); - loop { - tokio::select! { - result = connections.join_next(), if !connections.is_empty() => { if let Some(Err(error)) = result { warn!(%error, "Runtime API connection task failed"); } } - _ = shutdown_rx.changed() => { - debug!("runtime api: shutdown signalled, exiting accept loop"); - break; - } - accepted = listener.accept() => { - let (stream, _peer) = match accepted { - Ok(c) => c, - Err(e) => { - warn!(error = %e, "runtime api accept failed"); - continue; - } - }; - if connections.len() >= 4 { drop(stream); continue; } - let state = Arc::clone(&state); - connections.spawn(async move { - let io = TokioIo::new(stream); - let svc = service_fn(move |req| { - let state = Arc::clone(&state); - async move { handle(state, req).await } - }); - if let Err(e) = http1::Builder::new() - .serve_connection(io, svc) - .await - { - // EOF / reset is expected when the bootstrap exits. - debug!(error = %e, "runtime api connection closed"); - } - }); - } - } - } - connections.abort_all(); - while let Some(result) = connections.join_next().await { - if let Err(error) = result { - debug!(%error, "Runtime API connection stopped"); - } - } - }); - - Ok(RuntimeApiHandle { - addr, - job_tx, - shutdown_tx, - accept_task: Some(accept_task), - init_error_rx: Mutex::new(Some(init_err_rx)), - }) -} - -/// Service entry point. -async fn handle( - state: Arc, - req: Request, -) -> Result>, Infallible> { - let method = req.method().clone(); - let path = req.uri().path().to_owned(); - debug!(%method, %path, "runtime api request"); - - let result = match (method, path.as_str()) { - (Method::GET, "/2018-06-01/runtime/invocation/next") => handle_next(&state).await, - (Method::POST, p) - if p.starts_with("/2018-06-01/runtime/invocation/") && p.ends_with("/response") => - { - handle_complete(&state, &path, req, RuntimeResult::Success).await - } - (Method::POST, p) - if p.starts_with("/2018-06-01/runtime/invocation/") && p.ends_with("/error") => - { - handle_complete(&state, &path, req, RuntimeResult::Error).await - } - (Method::POST, "/2018-06-01/runtime/init/error") => handle_init_error(&state, req).await, - _ => Ok(simple(StatusCode::NOT_FOUND, b"not found".as_ref())), - }; - - Ok(result.unwrap_or_else(|status| simple(status, b"".as_ref()))) -} - -/// Long-poll for the next job. Returns once a job is queued or shutdown -/// drops the channel. -async fn handle_next(state: &Arc) -> Result>, StatusCode> { - let mut rx = state.job_rx.lock().await; - let Some(job) = rx.recv().await else { - return Err(StatusCode::SERVICE_UNAVAILABLE); - }; - drop(rx); // release before doing any work - - let Job { - request_id, - function_arn, - deadline, - payload, - response_tx, - } = job; - - state.pending.insert(request_id.clone(), response_tx); - - let deadline_ms = deadline_unix_ms(deadline); - // X-Ray-shaped trace id for runtime SDKs that expect the header to be - // present. Format isn't validated downstream — a UUID-derived value is - // fine for local emulation. - let trace_id = format!( - "Root=1-{:08x}-{};Parent={};Sampled=1", - unix_secs_lower32(), - uuid::Uuid::new_v4().simple(), - &uuid::Uuid::new_v4().simple().to_string()[..16], - ); - - let resp = Response::builder() - .status(StatusCode::OK) - .header("Lambda-Runtime-Aws-Request-Id", &request_id) - .header("Lambda-Runtime-Deadline-Ms", deadline_ms.to_string()) - .header("Lambda-Runtime-Invoked-Function-Arn", &function_arn) - .header("Lambda-Runtime-Trace-Id", trace_id) - .header("Content-Type", "application/json") - .body(Full::new(payload)) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(resp) -} - -/// Handle `/response` or `/error` — fire the matching oneshot, return 202. -async fn handle_complete( - state: &Arc, - path: &str, - req: Request, - map_result: F, -) -> Result>, StatusCode> -where - F: FnOnce(Bytes) -> RuntimeResult, -{ - let request_id = parse_request_id(path).ok_or(StatusCode::BAD_REQUEST)?; - let body = collect(req.into_body()).await?; - if let Some((_, sender)) = state.pending.remove(&request_id) { - let _ = sender.send(map_result(body)); - Ok(simple(StatusCode::ACCEPTED, b"".as_ref())) - } else { - Ok(simple( - StatusCode::BAD_REQUEST, - b"unknown request id".as_ref(), - )) - } -} - -async fn handle_init_error( - state: &Arc, - req: Request, -) -> Result>, StatusCode> { - let body = collect(req.into_body()).await?; - if let Some(tx) = state.init_error_tx.lock().await.take() { - let _ = tx.send(body); - } - Ok(simple(StatusCode::ACCEPTED, b"".as_ref())) -} - -fn parse_request_id(path: &str) -> Option { - let after = path.strip_prefix("/2018-06-01/runtime/invocation/")?; - let id = after.split('/').next()?; - if id.is_empty() { - None - } else { - Some(id.to_owned()) - } -} - -fn deadline_unix_ms(deadline: Instant) -> u64 { - let now = Instant::now(); - let remaining = deadline.saturating_duration_since(now); - let now_ms = u64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(), - ) - .unwrap_or(u64::MAX); - now_ms.saturating_add(u64::try_from(remaining.as_millis()).unwrap_or(0)) -} - -#[allow(clippy::cast_possible_truncation)] -fn unix_secs_lower32() -> u32 { - let secs = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - // Wrapping is fine — the trace id is opaque to consumers. - (secs & 0xFFFF_FFFF) as u32 -} - -async fn collect(body: Incoming) -> Result { - tokio::time::timeout( - std::time::Duration::from_secs(5), - Limited::new(body, 6 * 1024 * 1024).collect(), - ) - .await - .map_err(|_| StatusCode::REQUEST_TIMEOUT)? - .map(http_body_util::Collected::to_bytes) - .map_err(|error| { - if error.is::() { - StatusCode::PAYLOAD_TOO_LARGE - } else { - StatusCode::BAD_REQUEST - } - }) -} - -fn simple(status: StatusCode, body: &[u8]) -> Response> { - Response::builder() - .status(status) - .body(Full::new(Bytes::copy_from_slice(body))) - .unwrap_or_else(|_| Response::new(Full::new(Bytes::from_static(b"")))) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpStream, - }; - - use super::*; - - /// Send a raw HTTP/1.1 request, return (status_line, headers_block, body_bytes). - async fn http_request( - addr: SocketAddr, - method: &str, - path: &str, - body: &[u8], - ) -> (String, String, Vec) { - let mut stream = TcpStream::connect(addr).await.unwrap(); - let req = format!( - "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nContent-Length: {len}\r\nConnection: \ - close\r\n\r\n", - len = body.len(), - ); - stream.write_all(req.as_bytes()).await.unwrap(); - if !body.is_empty() { - stream.write_all(body).await.unwrap(); - } - let mut buf = Vec::new(); - stream.read_to_end(&mut buf).await.unwrap(); - - // Split status / headers / body manually. - let split = buf - .windows(4) - .position(|w| w == b"\r\n\r\n") - .expect("header terminator"); - let head_text = String::from_utf8_lossy(&buf[..split]).to_string(); - let body_bytes = buf[split + 4..].to_vec(); - let mut lines = head_text.split("\r\n"); - let status = lines.next().unwrap_or("").to_owned(); - let headers = lines.collect::>().join("\r\n"); - (status, headers, body_bytes) - } - - fn header_value(headers: &str, name: &str) -> Option { - let needle = name.to_ascii_lowercase(); - headers.split("\r\n").find_map(|line| { - let (k, v) = line.split_once(':')?; - if k.trim().to_ascii_lowercase() == needle { - Some(v.trim().to_owned()) - } else { - None - } - }) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_round_trip_invocation_via_runtime_api() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - - let (resp_tx, resp_rx) = oneshot::channel(); - let job = Job { - request_id: "req-1".to_owned(), - function_arn: "arn:fake".to_owned(), - deadline: Instant::now() + Duration::from_secs(3), - payload: Bytes::from_static(b"{\"hi\":1}"), - response_tx: resp_tx, - }; - handle.submit(job).await.unwrap(); - - let bootstrap = tokio::spawn(async move { - let (status, headers, body) = - http_request(addr, "GET", "/2018-06-01/runtime/invocation/next", b"").await; - assert!(status.starts_with("HTTP/1.1 200"), "status: {status}"); - let request_id = - header_value(&headers, "Lambda-Runtime-Aws-Request-Id").expect("request id header"); - assert_eq!(request_id, "req-1"); - assert_eq!(&body[..], b"{\"hi\":1}"); - - let path = format!("/2018-06-01/runtime/invocation/{request_id}/response"); - let (status, _h, _b) = http_request(addr, "POST", &path, b"{\"echo\":1}").await; - assert!(status.starts_with("HTTP/1.1 202"), "post status: {status}"); - }); - - let result = tokio::time::timeout(Duration::from_secs(3), resp_rx) - .await - .expect("oneshot timed out") - .unwrap(); - match result { - RuntimeResult::Success(b) => assert_eq!(&b[..], b"{\"echo\":1}"), - other => panic!("expected Success, got {other:?}"), - } - bootstrap.await.unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_propagate_function_error() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - - let (resp_tx, resp_rx) = oneshot::channel(); - handle - .submit(Job { - request_id: "req-2".to_owned(), - function_arn: "arn:fake".to_owned(), - deadline: Instant::now() + Duration::from_secs(3), - payload: Bytes::from_static(b"{}"), - response_tx: resp_tx, - }) - .await - .unwrap(); - - tokio::spawn(async move { - let (_s, headers, _b) = - http_request(addr, "GET", "/2018-06-01/runtime/invocation/next", b"").await; - let id = header_value(&headers, "Lambda-Runtime-Aws-Request-Id").unwrap(); - let path = format!("/2018-06-01/runtime/invocation/{id}/error"); - http_request( - addr, - "POST", - &path, - br#"{"errorMessage":"boom","errorType":"TestError"}"#, - ) - .await; - }); - - let result = tokio::time::timeout(Duration::from_secs(3), resp_rx) - .await - .unwrap() - .unwrap(); - match result { - RuntimeResult::Error(b) => { - let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); - assert_eq!(v["errorType"], "TestError"); - } - other => panic!("expected Error, got {other:?}"), - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_capture_init_error() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - let init_rx = handle.take_init_error_rx().await.unwrap(); - - tokio::spawn(async move { - http_request( - addr, - "POST", - "/2018-06-01/runtime/init/error", - br#"{"errorMessage":"init failed"}"#, - ) - .await; - }); - let body = tokio::time::timeout(Duration::from_secs(3), init_rx) - .await - .unwrap() - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(v["errorMessage"], "init failed"); - } -} diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index fd656489..790a9682 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -6,8 +6,10 @@ use std::{ collections::HashMap, - env, + env, fmt, + future::Future, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -15,11 +17,18 @@ use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use tokio::{sync::Mutex, time::Instant}; -use tracing::warn; +use tokio::{ + runtime::Handle, + sync::{Mutex, watch}, + time::Instant, +}; use uuid::Uuid; -use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse, PackageType}; +use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + invocation::{CleanupTask, InvocationContext}, +}; const DEFAULT_INSTANCE_ID: &str = "rustack_lambda"; const DEFAULT_STAGE_PORT: u32 = 5003; @@ -33,9 +42,7 @@ const DEFAULT_RESPONSE_LIMIT_BYTES: usize = const DEFAULT_RUN_BUDGET: Duration = Duration::from_hours(24); const DEFAULT_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); const CONNECT_RESPONSE_LIMIT_BYTES: usize = 128; -const CONTROL_RESPONSE_LIMIT_BYTES: usize = 1024; const PROTOCOL_VERSION: &str = "rustack.squib.lambda.v1"; -const SHUTDOWN_REQUEST_KIND: &str = "shutdown"; /// Configuration for [`SquibExecutor`]. #[derive(Debug, Clone)] @@ -60,6 +67,10 @@ pub struct SquibExecutorConfig { impl SquibExecutorConfig { /// Read Squib executor configuration from environment variables. + /// + /// # Errors + /// + /// Rejects invalid paths, instance IDs, ports, timeouts, response limits or run budgets. pub fn from_env() -> Result { Self::from_env_reader(|key| match rustack_core::settings::var(key) { Ok(value) => Some(value), @@ -165,12 +176,61 @@ impl Default for SquibExecutorConfig { #[derive(Debug)] pub struct SquibExecutor { config: SquibExecutorConfig, - runtime: Mutex>, + runtime: Mutex>>, poisoned: std::sync::atomic::AtomicBool, + stop: watch::Sender, + cleanup: Arc, +} + +/// A concrete VM owner; even an idle owner's Drop starts retained shutdown. +struct OwnedVm { + runtime: Option, + cleanup: Arc, + spawner: Handle, +} + +trait VmShutdown: Send + 'static { + fn shutdown(self) -> impl Future> + Send; +} + +impl VmShutdown for squib::Squib { + async fn shutdown(mut self) -> Result<(), ExecutorError> { + squib::Squib::shutdown(&mut self) + .await + .map_err(|_| ExecutorError::CleanupFailed("Squib VM shutdown failed".into())) + } +} + +impl OwnedVm { + fn new(runtime: R, cleanup: Arc) -> Self { + Self { + runtime: Some(runtime), + cleanup, + spawner: Handle::current(), + } + } +} + +impl fmt::Debug for OwnedVm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedVm").finish_non_exhaustive() + } +} + +impl Drop for OwnedVm { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + // This owner creates the only VM for this cleanup task. Use the + // captured runtime even if Drop runs outside an entered Tokio context. + self.cleanup + .start_on(&self.spawner, move || runtime.shutdown()); + } + } } +#[derive(Debug)] struct RuntimeLease<'a> { - runtime: Option, + runtime: Option>, poisoned: &'a std::sync::atomic::AtomicBool, } impl Drop for RuntimeLease<'_> { @@ -178,24 +238,36 @@ impl Drop for RuntimeLease<'_> { if self.runtime.is_some() { self.poisoned .store(true, std::sync::atomic::Ordering::Release); - // Squib's RAII handle requests VM shutdown. Never admit another guest after - // cancellation. + // OwnedVm transfers shutdown to the retained cleanup task on Drop. } } } +impl Drop for SquibExecutor { + fn drop(&mut self) { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + self.stop.send_replace(true); + // An idle VM must follow the same owned shutdown path as a busy one. + drop(self.runtime.get_mut().take()); + } +} + impl SquibExecutor { /// Build a Squib executor. #[must_use] pub fn new(config: SquibExecutorConfig) -> Self { + let (stop, _) = watch::channel(false); Self { config, runtime: Mutex::new(None), poisoned: std::sync::atomic::AtomicBool::new(false), + stop, + cleanup: Arc::new(CleanupTask::default()), } } - async fn start_runtime(&self) -> Result { + async fn start_runtime(&self) -> Result, ExecutorError> { let config_file = self.config.required_config_file()?; ensure_config_file_exists(config_file).await?; @@ -206,21 +278,26 @@ impl SquibExecutor { .without_api_socket() .run_budget(self.config.run_budget); - let squib = builder - .spawn() - .await - .map_err(|err| ExecutorError::InitFailed(format!("start Squib runtime: {err}")))?; - Ok(squib) + let squib = builder.spawn().await.map_err(|_| { + ExecutorError::CleanupFailed( + "Squib startup did not establish confirmed runtime ownership".into(), + ) + })?; + Ok(OwnedVm::new(squib, Arc::clone(&self.cleanup))) } - async fn invoke_guest(&self, req: InvokeRequest) -> Result { + async fn invoke_guest( + &self, + req: &InvokeRequest, + context: &InvocationContext, + ) -> Result { #[cfg(unix)] { let mut stream = connect_stage_socket_with_retry(&self.config).await?; - let request = SquibInvokeRequestWire::from_request(&req)?; + let request = SquibInvokeRequestWire::from_request(req, context.id())?; let request_bytes = serde_json::to_vec(&request) .map_err(|err| ExecutorError::Io(format!("encode Squib request: {err}")))?; - tokio::time::timeout(req.timeout, async { + tokio::time::timeout(req.revision.timeout, async { use tokio::io::AsyncWriteExt as _; stream @@ -236,10 +313,10 @@ impl SquibExecutor { "Squib response", ) .await?; - decode_response(&line, req.qualifier.as_str()) + decode_response(&line, req.revision.qualifier.as_str()) }) .await - .map_err(|_| ExecutorError::Timeout(req.timeout))? + .map_err(|_| ExecutorError::Timeout(req.revision.timeout))? } #[cfg(not(unix))] @@ -250,11 +327,29 @@ impl SquibExecutor { )) } } + async fn retire_vm(&self, vm: OwnedVm) -> Result<(), ExecutorError> { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + drop(vm); + self.wait_for_cleanup().await + } + + async fn wait_for_cleanup(&self) -> Result<(), ExecutorError> { + // Only the waiter has a deadline. CleanupTask retains the VM shutdown + // future even when this budget expires or the caller drops the waiter. + tokio::time::timeout(self.config.shutdown_timeout, self.cleanup.wait()) + .await + .map_err(|_| { + ExecutorError::CleanupFailed("Squib VM shutdown is not confirmed".into()) + })? + } } #[async_trait] impl Executor for SquibExecutor { - fn available(&self) -> Result<(), ExecutorError> { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + validate_request(request)?; + self.config.required_vsock_path()?; if self.poisoned.load(std::sync::atomic::Ordering::Acquire) { return Err(ExecutorError::Unsupported( "Squib guest execution was cancelled; restart the runtime before invoking again" @@ -264,60 +359,70 @@ impl Executor for SquibExecutor { Ok(()) } - async fn invoke(&self, req: InvokeRequest) -> Result { - validate_request(&req)?; - self.config.required_vsock_path()?; - let mut runtime = self.runtime.lock().await; - self.available()?; + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.available(&req)?; + context.check()?; + let mut runtime = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + runtime = self.runtime.lock() => runtime, + }; + self.available(&req)?; + // Squib's startup future owns unjoined VMM handles; never drop it on cancellation. let vm = match runtime.take() { Some(vm) => vm, - None => self.start_runtime().await?, + None => match self.start_runtime().await { + Ok(vm) => vm, + Err(error) => { + if matches!(&error, ExecutorError::CleanupFailed(_)) { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + } + return Err(error); + } + }, }; let mut lease = RuntimeLease { runtime: Some(vm), poisoned: &self.poisoned, }; - let result = self.invoke_guest(req).await; + let mut stop = self.stop.subscribe(); + let mut result = tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + _ = stop.wait_for(|stopped| *stopped) => Err(ExecutorError::Cancelled), + result = self.invoke_guest(&req, &context) => result, + }; + if self.poisoned.load(std::sync::atomic::Ordering::Acquire) || *self.stop.borrow() { + result = Err(ExecutorError::Cancelled); + } + if result.is_ok() { + if let Err(error) = context.check() { + result = Err(error); + } + } if result.is_ok() { *runtime = lease.runtime.take(); + } else if let Some(vm) = lease.runtime.take() { + self.retire_vm(vm).await?; } result } - async fn shutdown(&self) { - let mut runtime = self.runtime.lock().await.take(); - if let Some(squib) = runtime.as_mut() { - #[cfg(unix)] - match tokio::time::timeout( - self.config.shutdown_timeout, - request_guest_shutdown(&self.config), - ) - .await - { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!(error = %err, "failed to request Squib Lambda guest shutdown"); - } - Err(_) => { - warn!( - timeout_ms = self.config.shutdown_timeout.as_millis(), - "timed out requesting Squib Lambda guest shutdown" - ); - } - } - match tokio::time::timeout(self.config.shutdown_timeout, squib.shutdown()).await { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!(error = %err, "failed to shut down Squib Lambda runtime"); - } - Err(_) => { - warn!( - timeout_ms = self.config.shutdown_timeout.as_millis(), - "timed out shutting down Squib Lambda runtime" - ); - } - } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + self.stop.send_replace(true); + let mut runtime = self.runtime.lock().await; + if let Some(vm) = runtime.take() { + self.retire_vm(vm).await?; } + drop(runtime); + self.wait_for_cleanup().await } } @@ -342,36 +447,27 @@ struct SquibInvokeRequestWire { payload_base64: String, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SquibControlRequestWire { - protocol_version: &'static str, - kind: &'static str, -} - impl SquibInvokeRequestWire { - fn from_request(req: &InvokeRequest) -> Result { - let code_zip = req - .code_zip - .as_ref() - .ok_or_else(|| ExecutorError::InvalidCode("missing code zip".to_owned()))?; + fn from_request(req: &InvokeRequest, request_id: Uuid) -> Result { + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { + return Err(ExecutorError::Unsupported( + "Squib backend only supports Zip packages".into(), + )); + }; Ok(Self { protocol_version: PROTOCOL_VERSION, - request_id: Uuid::new_v4().to_string(), + request_id: request_id.to_string(), function_arn: req.function_arn.clone(), - function_name: req.function_name.clone(), - qualifier: req.qualifier.clone(), - runtime: req.runtime.clone(), - handler: req.handler.clone(), - architectures: req.architectures.clone(), - code_root: req - .code_root - .as_ref() - .map(|path| path.display().to_string()), - code_zip_base64: STANDARD.encode(code_zip.as_ref()), - environment: req.environment.clone(), - timeout_ms: duration_millis(req.timeout), - memory_mb: req.memory_mb, + function_name: req.revision.function_name.clone(), + qualifier: req.revision.qualifier.clone(), + runtime: req.revision.runtime.clone(), + handler: req.revision.handler.clone(), + architectures: vec![req.revision.architecture.as_str().into()], + code_root: Some(artifact.code_root().display().to_string()), + code_zip_base64: STANDARD.encode(artifact.zip_bytes()), + environment: req.revision.environment.clone(), + timeout_ms: duration_millis(req.revision.timeout), + memory_mb: req.revision.memory_mb, capture_logs: req.capture_logs, payload_base64: STANDARD.encode(req.payload.as_ref()), }) @@ -393,19 +489,16 @@ struct SquibInvokeResponseWire { } fn validate_request(req: &InvokeRequest) -> Result<(), ExecutorError> { - if req.package_type != PackageType::Zip { + if !matches!(req.revision.artifact, DeploymentArtifact::Zip(_)) { return Err(ExecutorError::Unsupported( "Squib backend only supports Zip packages".to_owned(), )); } - if !req.architectures.iter().any(|arch| arch == "arm64") { + if req.revision.architecture != Architecture::Arm64 { return Err(ExecutorError::Unsupported( "Squib backend requires an arm64 Lambda function".to_owned(), )); } - if req.code_zip.is_none() { - return Err(ExecutorError::InvalidCode("missing code zip".to_owned())); - } Ok(()) } @@ -431,30 +524,6 @@ async fn ensure_config_file_exists(path: &Path) -> Result<(), ExecutorError> { Ok(()) } -#[cfg(unix)] -async fn request_guest_shutdown(config: &SquibExecutorConfig) -> Result<(), ExecutorError> { - use tokio::io::AsyncWriteExt as _; - - let mut stream = connect_stage_socket_with_retry(config).await?; - let request = SquibControlRequestWire { - protocol_version: PROTOCOL_VERSION, - kind: SHUTDOWN_REQUEST_KIND, - }; - let request_bytes = serde_json::to_vec(&request) - .map_err(|err| ExecutorError::Io(format!("encode Squib shutdown request: {err}")))?; - stream - .write_all(&request_bytes) - .await - .map_err(|err| ExecutorError::Io(format!("write Squib shutdown request: {err}")))?; - stream - .write_all(b"\n") - .await - .map_err(|err| ExecutorError::Io(format!("write Squib shutdown newline: {err}")))?; - let _line = - read_bounded_line(&mut stream, CONTROL_RESPONSE_LIMIT_BYTES, "Squib shutdown").await?; - Ok(()) -} - #[cfg(unix)] async fn connect_stage_socket_with_retry( config: &SquibExecutorConfig, @@ -636,10 +705,54 @@ fn default_artifact_dir() -> PathBuf { #[cfg(test)] mod tests { - use std::time::Duration; + use std::{sync::Arc, time::Duration}; use super::*; + struct ProbeVm { + started: Arc, + gate: Arc, + reservation: tokio::sync::OwnedSemaphorePermit, + } + + impl VmShutdown for ProbeVm { + async fn shutdown(self) -> Result<(), ExecutorError> { + self.started.add_permits(1); + self.gate.acquire().await.unwrap().forget(); + drop(self.reservation); + Ok(()) + } + } + + #[tokio::test] + async fn test_should_transfer_idle_vm_drop_to_owned_shutdown_before_releasing_resources() { + let cleanup = Arc::new(CleanupTask::default()); + let capacity = Arc::new(tokio::sync::Semaphore::new(1)); + let started = Arc::new(tokio::sync::Semaphore::new(0)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let owner = OwnedVm::new( + ProbeVm { + started: Arc::clone(&started), + gate: Arc::clone(&gate), + reservation: capacity.clone().acquire_owned().await.unwrap(), + }, + Arc::clone(&cleanup), + ); + // Drop on an ordinary thread with no entered Tokio context. + std::thread::spawn(move || drop(owner)).join().unwrap(); + started.acquire().await.unwrap().forget(); + assert_eq!(capacity.available_permits(), 0); + assert!( + tokio::time::timeout(Duration::ZERO, cleanup.wait()) + .await + .is_err() + ); + assert_eq!(capacity.available_permits(), 0); + gate.add_permits(1); + cleanup.wait().await.unwrap(); + assert_eq!(capacity.available_permits(), 1); + } + #[test] fn test_should_parse_squib_config_from_env_reader() { let config = SquibExecutorConfig::from_env_reader(|key| match key { @@ -700,29 +813,23 @@ mod tests { #[test] fn test_should_encode_request_payload_base64() { - let req = InvokeRequest { - function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: Some("provided.al2023".to_owned()), - handler: Some("bootstrap".to_owned()), - architectures: vec!["arm64".to_owned()], - package_type: PackageType::Zip, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04zip")), - image_uri: None, - environment: HashMap::from([("KEY".to_owned(), "VALUE".to_owned())]), - timeout: Duration::from_secs(3), - memory_mb: 128, - payload: Bytes::from_static(b"{}"), - capture_logs: true, - }; - - let wire = SquibInvokeRequestWire::from_request(&req).unwrap(); + let mut req = InvokeRequest::fixture(); + req.payload = Bytes::from_static(b"{}"); + req.capture_logs = true; + let wire = SquibInvokeRequestWire::from_request(&req, Uuid::new_v4()).unwrap(); assert_eq!(wire.protocol_version, PROTOCOL_VERSION); - assert_eq!(wire.code_root.as_deref(), Some("/tmp/code")); - assert_eq!(wire.code_zip_base64, "UEsDBHppcA=="); + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { + panic!("unit fixture is ZIP"); + }; + assert_eq!( + wire.code_root, + Some(artifact.code_root().display().to_string()) + ); + assert_eq!( + STANDARD.decode(&wire.code_zip_base64).unwrap(), + *artifact.zip_bytes() + ); assert_eq!(wire.timeout_ms, 3000); assert_eq!(wire.payload_base64, "e30="); } @@ -785,23 +892,8 @@ mod tests { #[test] fn test_should_reject_non_arm64_request() { - let req = InvokeRequest { - function_arn: String::new(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: None, - handler: None, - architectures: vec!["x86_64".to_owned()], - package_type: PackageType::Zip, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04zip")), - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_secs(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, - }; + let mut req = InvokeRequest::fixture(); + Arc::make_mut(&mut req.revision).architecture = Architecture::X86_64; assert!(matches!( validate_request(&req), diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index 5ad04574..62918375 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -1,8 +1,17 @@ //! Shared executor request/response/config types. -use std::{collections::HashMap, path::PathBuf, str::FromStr, time::Duration}; +use std::{collections::HashMap, fmt, str::FromStr, sync::Arc, time::Duration}; use bytes::Bytes; +use rustack_lambda_model::types::ImageConfig; +use typed_builder::TypedBuilder; + +#[cfg(test)] +use crate::artifact::ValidatedArtifact; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + identity::FunctionResourceId, +}; /// Selects which execution backend the provider should construct at startup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -36,65 +45,139 @@ impl FromStr for ExecutorBackend { } } -/// Deployment package kind, mirroring the Lambda API. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PackageType { - /// Unzipped function code on the local filesystem. - Zip, - /// Container image referenced by URI. - Image, -} - -impl PackageType { - /// Parse from the wire string used by the Lambda model. - #[must_use] - pub fn from_wire(s: &str) -> Self { - if s.eq_ignore_ascii_case("Image") { - Self::Image - } else { - Self::Zip - } - } -} - -/// Everything an executor needs to run one invocation. -#[derive(Debug, Clone)] -pub struct InvokeRequest { - /// Qualified ARN — used for `Lambda-Runtime-Invoked-Function-Arn`. - pub function_arn: String, +/// Validated function configuration captured before invocation admission. +/// +/// Requests share this value through Arc; updates create another captured revision. +#[derive(Clone, TypedBuilder)] +#[non_exhaustive] +pub struct ExecutionRevision { + /// Stable function lifetime identity. + pub resource_id: FunctionResourceId, + /// Captured configuration revision identifier. + pub revision_id: String, + /// Backend selection policy captured at admission. + pub backend: ExecutorBackend, /// Bare function name. pub function_name: String, - /// Resolved version string, e.g. `"$LATEST"` or `"3"`. + /// Resolved numeric version or `$LATEST`, never an invoked alias. pub qualifier: String, - /// Lambda runtime identifier, e.g. `"provided.al2023"`. + /// Runtime identifier for ZIP code. pub runtime: Option, - /// Handler string from the function config. + /// Configured ZIP handler. pub handler: Option, - /// Architectures the function was created with, e.g. `["x86_64"]`. - pub architectures: Vec, - /// Package type — Zip uses `code_root`, Image uses `image_uri`. - pub package_type: PackageType, - /// Filesystem path to the unzipped code root (for Zip packages). - pub code_root: Option, - /// Raw uploaded Zip bytes (for microVM staging). - pub code_zip: Option, - /// Image URI (for Image packages). - pub image_uri: Option, - /// User-supplied environment variables. + /// One validated architecture. + pub architecture: Architecture, + /// One immutable artifact, with shared lifetime ownership. + pub artifact: DeploymentArtifact, + /// Complete image entry configuration; missing and empty argv differ. + pub image_config: Option, + /// Captured user and provider-injected environment. pub environment: HashMap, - /// Function timeout. Excess => `Sandbox.Timedout`. + /// Function handler time limit. pub timeout: Duration, - /// Memory size in MB (used as the container memory cap). + /// Function memory in MiB. pub memory_mb: u32, - /// Raw invocation payload (typically JSON, but not required). + /// Ephemeral storage in MiB. + pub ephemeral_storage_mb: u32, +} + +impl fmt::Debug for ExecutionRevision { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecutionRevision") + .field("resource_id", &self.resource_id) + .field("revision_id", &self.revision_id) + .field("backend", &self.backend) + .field("function_name", &self.function_name) + .field("qualifier", &self.qualifier) + .field("artifact", &self.artifact) + .field("architecture", &self.architecture) + .field("environment", &"[REDACTED]") + .field("image_config", &"[REDACTED]") + .field("timeout", &self.timeout) + .field("memory_mb", &self.memory_mb) + .field("ephemeral_storage_mb", &self.ephemeral_storage_mb) + .finish_non_exhaustive() + } +} + +/// One invocation of an immutable captured revision. +#[derive(Clone)] +#[non_exhaustive] +pub struct InvokeRequest { + /// Shared captured configuration and artifact ownership. + pub revision: Arc, + /// Canonical requested ARN, preserving the invoked qualifier. + pub function_arn: String, + /// Original event bytes. pub payload: Bytes, - /// When set, the executor captures the last 4KB of stderr as `log_tail`. + /// Whether to return the final runtime log bytes. pub capture_logs: bool, } +impl InvokeRequest { + /// Create an invocation without copying its captured configuration. + /// + /// ``` + /// # use std::sync::Arc; + /// # use bytes::Bytes; + /// # use rustack_lambda_core::executor::{ExecutionRevision, InvokeRequest}; + /// # fn capture(revision: Arc, arn: String) { + /// let request = InvokeRequest::new(Arc::clone(&revision), arn, Bytes::new()); + /// assert!(Arc::ptr_eq(&request.revision, &revision)); + /// # } + /// ``` + #[must_use] + pub fn new(revision: Arc, function_arn: String, payload: Bytes) -> Self { + Self { + revision, + function_arn, + payload, + capture_logs: false, + } + } +} + +impl fmt::Debug for InvokeRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvokeRequest") + .field("revision", &self.revision) + .field("function_arn", &self.function_arn) + .field("payload", &"[REDACTED]") + .field("capture_logs", &self.capture_logs) + .finish() + } +} + +#[cfg(test)] +impl InvokeRequest { + pub(crate) fn fixture() -> Self { + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::default()) + .revision_id("fixture-revision".into()) + .backend(ExecutorBackend::Native) + .function_name("fixture".into()) + .qualifier("$LATEST".into()) + .runtime(Some("provided.al2023".into())) + .handler(Some("bootstrap".into())) + .architecture(Architecture::Arm64) + .artifact(DeploymentArtifact::Zip(ValidatedArtifact::fixture())) + .image_config(None) + .environment(HashMap::new()) + .timeout(Duration::from_secs(3)) + .memory_mb(128) + .ephemeral_storage_mb(512) + .build(); + Self::new( + Arc::new(revision), + "arn:aws:lambda:us-east-1:000000000000:function:fixture".into(), + Bytes::new(), + ) + } +} + /// Outcome of a successful round-trip with the user code (which may itself /// have reported a function error). -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct InvokeResponse { /// HTTP status to mirror back to the caller (typically 200). pub status: u16, @@ -102,12 +185,25 @@ pub struct InvokeResponse { pub payload: Bytes, /// `Some("Unhandled")` when the bootstrap posted to `/error`. pub function_error: Option, - /// Base64-encoded last 4KB of stderr, if `capture_logs` was set. + /// Base64-encoded final bounded log bytes when requested. Docker combines + /// observed stdout/stderr in its invocation window; Native uses stderr. pub log_tail: Option, /// What the executor actually invoked (echoes back the qualifier). pub executed_version: String, } +impl fmt::Debug for InvokeResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvokeResponse") + .field("status", &self.status) + .field("function_error", &self.function_error.is_some()) + .field("executed_version", &self.executed_version) + .field("payload", &"[REDACTED]") + .field("log_tail", &"[REDACTED]") + .finish() + } +} + impl InvokeResponse { /// Convenience constructor for a plain JSON success response. #[must_use] @@ -126,6 +222,39 @@ impl InvokeResponse { mod tests { use super::*; + #[test] + fn test_should_share_captured_revisions_and_isolate_separately_updated_configuration() { + let original = InvokeRequest::fixture(); + let mut updated = original.clone(); + assert!(Arc::ptr_eq(&original.revision, &updated.revision)); + Arc::make_mut(&mut updated.revision).memory_mb = 512; + assert_eq!(original.revision.memory_mb, 128); + assert_eq!(updated.revision.memory_mb, 512); + assert!(!Arc::ptr_eq(&original.revision, &updated.revision)); + } + + #[test] + fn test_should_redact_invocation_request_and_response_debug() { + let secret = "fixture-secret-should-never-be-rendered"; + let mut request = InvokeRequest::fixture(); + Arc::make_mut(&mut request.revision) + .environment + .insert("SECRET".into(), secret.into()); + request.payload = Bytes::from(secret); + Arc::make_mut(&mut request.revision).image_config = Some(ImageConfig { + entry_point: Some(vec![secret.into()]), + command: Some(vec![secret.into()]), + working_directory: Some(format!("/{secret}")), + }); + let mut response = InvokeResponse::success(Bytes::from(secret), "1".into()); + response.function_error = Some(secret.into()); + response.log_tail = Some(secret.into()); + for rendered in [format!("{request:?}"), format!("{response:?}")] { + assert!(!rendered.contains(secret)); + assert!(rendered.contains("[REDACTED]")); + } + } + #[test] fn test_should_parse_executor_backend_strings() { assert_eq!( @@ -154,13 +283,4 @@ mod tests { ); assert!("nope".parse::().is_err()); } - - #[test] - fn test_should_parse_package_type_from_wire() { - assert_eq!(PackageType::from_wire("Image"), PackageType::Image); - assert_eq!(PackageType::from_wire("image"), PackageType::Image); - assert_eq!(PackageType::from_wire("Zip"), PackageType::Zip); - // Anything unrecognized falls back to Zip — matches the provider default. - assert_eq!(PackageType::from_wire(""), PackageType::Zip); - } } diff --git a/crates/rustack-lambda-core/src/function_config.rs b/crates/rustack-lambda-core/src/function_config.rs new file mode 100644 index 00000000..64dee520 --- /dev/null +++ b/crates/rustack-lambda-core/src/function_config.rs @@ -0,0 +1,416 @@ +//! Validate captured function settings before storage or execution admission. + +use std::{collections::HashMap, fmt}; + +use rustack_lambda_model::{input::CreateFunctionInput, types::ImageConfig}; +use typed_builder::TypedBuilder; + +use crate::{ + artifact::{Architecture, ImageReference, ResolvedImage}, + error::LambdaServiceError, + storage::{VersionRecord, VersionRecordSnapshot}, +}; + +fn invalid(message: &'static str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} + +#[derive(TypedBuilder)] +struct Settings<'a> { + package_type: &'a str, + runtime: Option<&'a str>, + handler: Option<&'a str>, + architectures: Option<&'a [String]>, + environment: Option<&'a HashMap>, + image_config: Option<&'a ImageConfig>, + image_uri: Option<&'a str>, + timeout: u32, + memory: u32, + ephemeral: u32, +} + +impl fmt::Debug for Settings<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Settings") + .field("package_type", &self.package_type) + .field("environment", &"[REDACTED]") + .field("image_config", &"[REDACTED]") + .finish_non_exhaustive() + } +} + +impl Settings<'_> { + fn validate(&self) -> Result { + if !(1..=900).contains(&self.timeout) + || !(128..=10240).contains(&self.memory) + || !(512..=10240).contains(&self.ephemeral) + { + return Err(invalid( + "Invalid function timeout, memory or ephemeral storage", + )); + } + let architecture = match self.architectures { + None => Architecture::X86_64, + Some([architecture]) => architecture + .parse() + .map_err(|_| invalid("Invalid Lambda architecture"))?, + Some(_) => return Err(invalid("Exactly one Lambda architecture is required")), + }; + match self.package_type { + "Zip" => { + if self.runtime.is_none_or(|runtime| { + runtime.is_empty() || runtime.len() > 128 || runtime.contains('\0') + }) || self.handler.is_none_or(|handler| { + handler.is_empty() || handler.len() > 256 || handler.contains('\0') + }) || self.image_uri.is_some() + || self.image_config.is_some() + { + return Err(invalid("Invalid ZIP runtime, handler or image fields")); + } + } + "Image" => { + let reference = self + .image_uri + .ok_or_else(|| invalid("ImageUri is required"))?; + reference + .parse::() + .map_err(|_| invalid("Invalid ImageUri"))?; + if self.runtime.is_some() || self.handler.is_some() { + return Err(invalid( + "Image functions cannot contain ZIP runtime or handler fields", + )); + } + if let Some(config) = self.image_config { + validate_image_config(config)?; + } + } + _ => return Err(invalid("Unsupported deployment package type")), + } + if let Some(environment) = self.environment { + validate_environment(environment)?; + } + Ok(architecture) + } +} + +pub(crate) fn validate_provided_settings( + runtime: Option<&str>, + handler: Option<&str>, +) -> Result<(), LambdaServiceError> { + if runtime != Some("provided.al2023") + || handler.is_none_or(|value| { + value.is_empty() + || value.len() > 128 + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_./:-".contains(&byte)) + }) + { + return Err(invalid( + "Docker ZIP requires provided.al2023 and a supported handler", + )); + } + Ok(()) +} + +pub(crate) fn validate_create_settings( + input: &CreateFunctionInput, +) -> Result { + Settings::builder() + .package_type(input.package_type.as_deref().unwrap_or("Zip")) + .runtime(input.runtime.as_deref()) + .handler(input.handler.as_deref()) + .architectures(input.architectures.as_deref()) + .environment( + input + .environment + .as_ref() + .and_then(|environment| environment.variables.as_ref()), + ) + .image_config(input.image_config.as_ref()) + .image_uri(input.code.image_uri.as_deref()) + .timeout(input.timeout.unwrap_or(3)) + .memory(input.memory_size.unwrap_or(128)) + .ephemeral( + input + .ephemeral_storage + .as_ref() + .map_or(512, |storage| storage.size), + ) + .build() + .validate() +} + +pub(crate) fn validate_execution_settings( + version: &VersionRecord, +) -> Result { + let architecture = Settings::builder() + .package_type(&version.package_type) + .runtime(version.runtime.as_deref()) + .handler(version.handler.as_deref()) + .architectures(Some(&version.architectures)) + .environment(Some(&version.environment)) + .image_config(version.image_config.as_ref()) + .image_uri(version.image_uri.as_deref()) + .timeout(version.timeout) + .memory(version.memory_size) + .ephemeral(version.ephemeral_storage_size) + .build() + .validate()?; + validate_layout( + &version.package_type, + version.zip_bytes.is_some(), + version.code_path.is_some(), + version.image.resolved(), + architecture, + version.image_uri.as_deref(), + )?; + validate_image_fingerprint( + version.image.resolved(), + &version.code_sha256, + version.code_size, + version + .image + .pinned() + .map(|image| image.unpacked_size().get()), + )?; + Ok(architecture) +} + +pub(crate) fn validate_snapshot_settings( + version: &VersionRecordSnapshot, +) -> Result<(), LambdaServiceError> { + let architecture = Settings::builder() + .package_type(&version.package_type) + .runtime(version.runtime.as_deref()) + .handler(version.handler.as_deref()) + .architectures(Some(&version.architectures)) + .environment(Some(&version.environment)) + .image_config(version.image_config.as_ref()) + .image_uri(version.image_uri.as_deref()) + .timeout(version.timeout) + .memory(version.memory_size) + .ephemeral(version.ephemeral_storage_size) + .build() + .validate()?; + validate_layout( + &version.package_type, + version.zip_bytes_base64.is_some(), + false, + version.resolved_image.as_ref(), + architecture, + version.image_uri.as_deref(), + )?; + validate_image_fingerprint( + version.resolved_image.as_ref(), + &version.code_sha256, + version.code_size, + None, + ) +} + +fn validate_image_fingerprint( + image: Option<&ResolvedImage>, + hash: &str, + size: u64, + pinned_size: Option, +) -> Result<(), LambdaServiceError> { + if let Some(image) = image { + if hash + != image + .code_sha256() + .map_err(|_| invalid("Invalid image fingerprint"))? + || !(1..=crate::artifact::MAX_IMAGE_BYTES).contains(&size) + || pinned_size.is_some_and(|proven| proven != size) + { + return Err(invalid("Image deployment metadata mismatch")); + } + } + Ok(()) +} + +fn validate_layout( + package_type: &str, + has_zip: bool, + has_extracted: bool, + image: Option<&ResolvedImage>, + architecture: Architecture, + reference: Option<&str>, +) -> Result<(), LambdaServiceError> { + if (package_type == "Zip" && image.is_some()) + || (package_type == "Image" && (has_extracted || has_zip)) + { + return Err(invalid("Deployment package fields are inconsistent")); + } + if let Some(image) = image { + if image.metadata().architecture != architecture + || Some(image.metadata().requested_reference.as_str()) != reference + { + return Err(invalid( + "Resolved image does not match function configuration", + )); + } + } + Ok(()) +} + +pub(crate) fn validate_image_config(config: &ImageConfig) -> Result<(), LambdaServiceError> { + let mut bytes = 0usize; + for args in [config.entry_point.as_deref(), config.command.as_deref()] + .into_iter() + .flatten() + { + if args.len() > 1500 { + return Err(invalid("Image argv count exceeds the local limit")); + } + for arg in args { + if arg.len() > 4096 || arg.contains('\0') { + return Err(invalid("Invalid image argument")); + } + bytes = bytes + .checked_add(arg.len()) + .ok_or_else(|| invalid("Image argv size overflow"))?; + if bytes > 64 * 1024 { + return Err(invalid("Image argv exceeds the local byte limit")); + } + } + } + if config + .working_directory + .as_ref() + .is_some_and(|path| !path.starts_with('/') || path.len() > 1000 || path.contains('\0')) + { + return Err(invalid( + "Image working directory must be a bounded absolute path", + )); + } + Ok(()) +} + +fn validate_environment(environment: &HashMap) -> Result<(), LambdaServiceError> { + validate_environment_entries(environment) +} + +pub(crate) fn validate_environment_entries<'a>( + environment: impl IntoIterator, +) -> Result<(), LambdaServiceError> { + let mut count = 0_u16; + let mut bytes = 0usize; + for (key, value) in environment { + count = count.saturating_add(1); + if count > 128 { + return Err(invalid("Environment has too many variables")); + } + if !(2..=64).contains(&key.len()) + || !key.as_bytes().first().is_some_and(u8::is_ascii_alphabetic) + || !key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + || value.len() > 4096 + || value.contains('\0') + || is_reserved(key) + { + return Err(invalid("Invalid or reserved environment variable")); + } + bytes = bytes + .checked_add(key.len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .ok_or_else(|| invalid("Environment size overflow"))?; + if bytes > 4096 { + return Err(invalid("Environment exceeds 4 KiB")); + } + } + Ok(()) +} + +fn is_reserved(key: &str) -> bool { + matches!( + key, + "_HANDLER" + | "_X_AMZN_TRACE_ID" + | "AWS_DEFAULT_REGION" + | "AWS_REGION" + | "AWS_EXECUTION_ENV" + | "AWS_LAMBDA_FUNCTION_NAME" + | "AWS_LAMBDA_FUNCTION_MEMORY_SIZE" + | "AWS_LAMBDA_FUNCTION_VERSION" + | "AWS_LAMBDA_FUNCTION_TIMEOUT" + | "AWS_LAMBDA_INITIALIZATION_TYPE" + | "AWS_LAMBDA_LOG_GROUP_NAME" + | "AWS_LAMBDA_LOG_STREAM_NAME" + | "AWS_ACCESS_KEY" + | "AWS_ACCESS_KEY_ID" + | "AWS_SECRET_ACCESS_KEY" + | "AWS_SESSION_TOKEN" + | "AWS_LAMBDA_RUNTIME_API" + | "LAMBDA_TASK_ROOT" + | "LAMBDA_RUNTIME_DIR" + | "AWS_LAMBDA_MAX_CONCURRENCY" + | "AWS_LAMBDA_METADATA_API" + | "AWS_LAMBDA_METADATA_TOKEN" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_bound_environment_bytes_and_reject_reserved_or_invalid_keys() { + let at_limit = HashMap::from([("KEY".into(), "v".repeat(4093))]); + assert!(validate_environment(&at_limit).is_ok()); + let over_limit = HashMap::from([("KEY".into(), "v".repeat(4094))]); + assert!(validate_environment(&over_limit).is_err()); + for key in [ + "AWS_REGION", + "AWS_LAMBDA_RUNTIME_API", + "AWS_SECRET_ACCESS_KEY", + "LAMBDA_TASK_ROOT", + "A", + "A-B", + "ÄB", + ] { + let environment = HashMap::from([(key.into(), "value".into())]); + assert!(validate_environment(&environment).is_err(), "{key}"); + } + for key in ["AWS_ENDPOINT_URL", "TZ", "PATH"] { + assert!(validate_environment(&HashMap::from([(key.into(), "value".into())])).is_ok()); + } + } + + #[test] + fn test_should_validate_image_argv_without_collapsing_empty_and_missing() { + assert!(validate_image_config(&ImageConfig::default()).is_ok()); + assert!( + validate_image_config(&ImageConfig { + command: Some(vec![]), + ..ImageConfig::default() + }) + .is_ok() + ); + for config in [ + ImageConfig { + command: Some(vec!["x".repeat(4097)]), + ..ImageConfig::default() + }, + ImageConfig { + command: Some(vec![String::new(); 1501]), + ..ImageConfig::default() + }, + ImageConfig { + command: Some(vec!["x".repeat(4096); 17]), + ..ImageConfig::default() + }, + ImageConfig { + entry_point: Some(vec!["x\0".into()]), + ..ImageConfig::default() + }, + ImageConfig { + working_directory: Some("relative".into()), + ..ImageConfig::default() + }, + ] { + assert!(validate_image_config(&config).is_err()); + } + } +} diff --git a/crates/rustack-lambda-core/src/handler.rs b/crates/rustack-lambda-core/src/handler.rs index 3339f584..77af70f0 100644 --- a/crates/rustack-lambda-core/src/handler.rs +++ b/crates/rustack-lambda-core/src/handler.rs @@ -30,7 +30,10 @@ use rustack_lambda_model::{ types::InvocationType, }; -use crate::provider::{InvokeKind, InvokeOutcome, RustackLambda}; +use crate::{ + deployment_input::{Profile, decode}, + provider::{InvokeKind, InvokeOutcome, RustackLambda}, +}; /// Handler that bridges the HTTP layer to the Lambda provider. #[derive(Debug)] @@ -121,7 +124,7 @@ async fn dispatch( path_params: &PathParams, query: &str, headers: &http::HeaderMap, - body: &[u8], + body: &Bytes, ) -> Result, LambdaError> { let query_params = parse_query_params(query); let qualifier = get_query_param(&query_params, "Qualifier"); @@ -129,9 +132,8 @@ async fn dispatch( match op { // ---- Phase 0: Function CRUD ---- LambdaOperation::CreateFunction => { - let input: CreateFunctionInput = serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, _input_permit): (CreateFunctionInput, _) = + decode(body, Profile::Create).await?; let config = provider .create_function(input) .await @@ -165,11 +167,10 @@ async fn dispatch( LambdaOperation::UpdateFunctionCode => { let function_name = require_path_param(path_params, "FunctionName")?; - let input: UpdateFunctionCodeInput = serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, input_permit): (UpdateFunctionCodeInput, _) = + decode(body, Profile::Code).await?; let config = provider - .update_function_code(function_name, input) + .update_function_code_parsed(function_name, input, input_permit) .await .map_err(LambdaError::from)?; wrap_json_response(200, &config) @@ -177,10 +178,8 @@ async fn dispatch( LambdaOperation::UpdateFunctionConfiguration => { let function_name = require_path_param(path_params, "FunctionName")?; - let input: UpdateFunctionConfigurationInput = - serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, _input_permit): (UpdateFunctionConfigurationInput, _) = + decode(body, Profile::Configuration).await?; let config = provider .update_function_configuration(function_name, &input) .map_err(LambdaError::from)?; @@ -209,11 +208,21 @@ async fn dispatch( let function_name = require_path_param(path_params, "FunctionName")?; // Parse InvocationType from header. - let invocation_type = headers - .get("x-amz-invocation-type") - .and_then(|v| v.to_str().ok()) - .and_then(InvocationType::from_str_value) - .unwrap_or(InvocationType::RequestResponse); + let invocation_type = match single_invoke_header(headers, "x-amz-invocation-type")? { + None => InvocationType::RequestResponse, + Some(value) => value + .to_str() + .ok() + .and_then(InvocationType::from_str_value) + .ok_or_else(|| LambdaError::invalid_parameter("Invalid InvocationType"))?, + }; + let capture_logs = match single_invoke_header(headers, "x-amz-log-type")? + .map(http::HeaderValue::as_bytes) + { + None | Some(b"None") => false, + Some(b"Tail") => true, + Some(_) => return Err(LambdaError::invalid_parameter("Invalid LogType")), + }; let kind = match invocation_type { InvocationType::RequestResponse => InvokeKind::RequestResponse, InvocationType::Event => InvokeKind::Event, @@ -221,7 +230,7 @@ async fn dispatch( }; let outcome = provider - .invoke(function_name, qualifier, body, kind) + .invoke_with_logs(function_name, qualifier, body, kind, capture_logs) .await .map_err(LambdaError::from)?; @@ -713,6 +722,20 @@ fn require_path_param<'a>(params: &'a PathParams, name: &str) -> Result<&'a str, .ok_or_else(|| LambdaError::invalid_parameter(format!("Missing path parameter: {name}"))) } +fn single_invoke_header<'a>( + headers: &'a http::HeaderMap, + name: &'static str, +) -> Result, LambdaError> { + let mut values = headers.get_all(name).iter(); + let value = values.next(); + if values.next().is_some() { + return Err(LambdaError::invalid_parameter( + "Duplicate invocation header", + )); + } + Ok(value) +} + /// Wrap a serializable value into a JSON `LambdaResponseBody` response. fn wrap_json_response( status: u16, @@ -738,7 +761,64 @@ fn wrap_empty_response(status: u16) -> Result #[cfg(test)] mod tests { + use http::{HeaderMap, HeaderValue, Method}; + use rustack_lambda_http::router::resolve_operation; + use rustack_lambda_model::error::LambdaErrorCode; + use super::*; + use crate::config::LambdaConfig; + + #[rstest::rstest] + #[case::log_case("x-amz-log-type", b"tail".as_slice())] + #[case::log_unknown("x-amz-log-type", b"private-invalid-log-type".as_slice())] + #[case::log_nonascii("x-amz-log-type", b"\xff".as_slice())] + #[case::invocation_case("x-amz-invocation-type", b"event".as_slice())] + #[case::invocation_unknown("x-amz-invocation-type", b"private-invalid-invocation-type".as_slice())] + #[case::invocation_nonascii("x-amz-invocation-type", b"\xff".as_slice())] + #[tokio::test] + async fn test_should_reject_invalid_invoke_headers_before_resource_lookup( + #[case] key: &'static str, + #[case] value: &[u8], + ) { + let provider = RustackLambda::new(LambdaConfig::default()); + let (operation, params, _) = + resolve_operation(&Method::POST, "/2015-03-31/functions/missing/invocations").unwrap(); + let mut headers = HeaderMap::new(); + headers.insert(key, HeaderValue::from_bytes(value).unwrap()); + let error = dispatch(&provider, operation, ¶ms, "", &headers, &Bytes::new()) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::InvalidParameterValueException); + assert!(!format!("{error:?}").contains("private-invalid")); + provider.shutdown().await; + } + + #[rstest::rstest] + #[case::tail_none("x-amz-log-type", "Tail", "None")] + #[case::none_tail("x-amz-log-type", "None", "Tail")] + #[case::tail_invalid("x-amz-log-type", "Tail", "private-invalid-value")] + #[case::sync_event("x-amz-invocation-type", "RequestResponse", "Event")] + #[case::event_sync("x-amz-invocation-type", "Event", "RequestResponse")] + #[case::sync_invalid("x-amz-invocation-type", "RequestResponse", "private-invalid-value")] + #[tokio::test] + async fn test_should_reject_duplicate_invoke_enum_headers_before_resource_lookup( + #[case] key: &'static str, + #[case] first: &'static str, + #[case] second: &'static str, + ) { + let provider = RustackLambda::new(LambdaConfig::default()); + let (operation, params, _) = + resolve_operation(&Method::POST, "/2015-03-31/functions/missing/invocations").unwrap(); + let mut headers = HeaderMap::new(); + headers.append(key, HeaderValue::from_static(first)); + headers.append(key, HeaderValue::from_static(second)); + let error = dispatch(&provider, operation, ¶ms, "", &headers, &Bytes::new()) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::InvalidParameterValueException); + assert!(!format!("{error:?}").contains("private-invalid")); + provider.shutdown().await; + } #[test] fn test_should_parse_empty_query() { diff --git a/crates/rustack-lambda-core/src/identity.rs b/crates/rustack-lambda-core/src/identity.rs new file mode 100644 index 00000000..747496e4 --- /dev/null +++ b/crates/rustack-lambda-core/src/identity.rs @@ -0,0 +1,133 @@ +//! Stable function resource identity, independent of mutable names and revisions. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use uuid::Uuid; + +/// Identity of one function's lifetime, preserved across updates and snapshots. +/// +/// Deleting and recreating a name must allocate a new identity. +/// +/// ``` +/// use rustack_lambda_core::identity::FunctionResourceId; +/// let identity = FunctionResourceId::default(); +/// assert_eq!(identity.to_string().parse::()?, identity); +/// # Ok::<(), rustack_lambda_core::identity::ResourceIdError>(()) +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct FunctionResourceId(Uuid); + +/// A saved resource identity is malformed or absent of meaningful identity. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ResourceIdError { + /// Only the 36-byte hyphenated UUID representation is accepted. + #[error("function resource identity must be a 36-byte UUID")] + Length, + /// The input is not a UUID. + #[error("invalid function resource UUID")] + Invalid { + /// UUID parser failure, without the original input. + #[source] + source: uuid::Error, + }, + /// A nil UUID cannot identify a function lifetime. + #[error("function resource UUID must not be nil")] + Nil, +} + +impl Default for FunctionResourceId { + fn default() -> Self { + Self::new() + } +} + +impl FunctionResourceId { + /// Allocate a random identity for a new function resource. + /// + /// ``` + /// use rustack_lambda_core::identity::FunctionResourceId; + /// let first = FunctionResourceId::new(); + /// assert_ne!(first, FunctionResourceId::new()); + /// ``` + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl fmt::Display for FunctionResourceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl FromStr for FunctionResourceId { + type Err = ResourceIdError; + + fn from_str(value: &str) -> Result { + if value.len() != 36 { + return Err(ResourceIdError::Length); + } + let uuid = Uuid::parse_str(value).map_err(|source| ResourceIdError::Invalid { source })?; + if uuid.is_nil() { + return Err(ResourceIdError::Nil); + } + Ok(Self(uuid)) + } +} + +impl Serialize for FunctionResourceId { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for FunctionResourceId { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + impl de::Visitor<'_> for Visitor { + type Value = FunctionResourceId; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a nonnil hyphenated function resource UUID") + } + + fn visit_str(self, value: &str) -> Result { + value.parse().map_err(E::custom) + } + } + deserializer.deserialize_str(Visitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_generate_distinct_nonnil_serializable_identities() + -> Result<(), Box> { + let first = FunctionResourceId::default(); + let second = FunctionResourceId::default(); + assert_ne!(first, second); + let saved = serde_json::to_string(&first)?; + assert_eq!(serde_json::from_str::(&saved)?, first); + Ok(()) + } + + #[test] + fn test_should_reject_nil_malformed_and_oversized_resource_identity() { + assert!(matches!( + "00000000-0000-0000-0000-000000000000".parse::(), + Err(ResourceIdError::Nil) + )); + for value in ["x".repeat(36), "x".repeat(4096), String::new()] { + assert!(value.parse::().is_err()); + let saved = serde_json::Value::String(value); + assert!(serde_json::from_value::(saved).is_err()); + } + } +} diff --git a/crates/rustack-lambda-core/src/invocation.rs b/crates/rustack-lambda-core/src/invocation.rs new file mode 100644 index 00000000..fbfc898a --- /dev/null +++ b/crates/rustack-lambda-core/src/invocation.rs @@ -0,0 +1,769 @@ +//! Attempt identity, cooperative cancellation and supervised invocation handles. +//! +//! Cancellation signals do not drop backend futures. The work supervisor owns +//! those futures until their resources are settled. + +use std::{ + fmt, + future::{Future, pending}, + sync::Arc, +}; + +use parking_lot::Mutex as OwnerMutex; +use tokio::{ + runtime::Handle, + sync::{Mutex, oneshot, watch}, + task::JoinHandle, + time::{Instant, sleep_until, timeout_at}, +}; +use uuid::Uuid; + +use crate::{ + error::LambdaServiceError, + executor::{ExecutorError, InvokeResponse}, +}; + +/// Why an invocation must stop accepting a result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CancellationReason { + /// Explicit cancellation or loss of the attached caller. + Requested, + /// The absolute invocation budget expired. + Deadline, +} + +impl CancellationReason { + pub(crate) fn error(self) -> ExecutorError { + match self { + Self::Requested => ExecutorError::Cancelled, + Self::Deadline => ExecutorError::DeadlineExceeded, + } + } +} + +/// Immutable attempt metadata and a cooperative cancellation receiver. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct InvocationContext { + id: Uuid, + deadline: Instant, + cancellation: watch::Receiver, + external: Option>, +} + +impl InvocationContext { + #[cfg(test)] + pub(crate) fn channel(deadline: Instant) -> (watch::Sender, Self) { + Self::channel_with_cancellation(deadline, None) + } + + pub(crate) fn channel_with_cancellation( + deadline: Instant, + external: Option>, + ) -> (watch::Sender, Self) { + let (sender, cancellation) = watch::channel(false); + ( + sender, + Self { + id: Uuid::new_v4(), + deadline, + cancellation, + external, + }, + ) + } + + /// Attempt identity shared with the Runtime API and invocation handle. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// assert!(!context.id().is_nil()); + /// # } + /// ``` + #[must_use] + pub fn id(&self) -> Uuid { + self.id + } + + /// Absolute monotonic budget, including queue and cold start. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let deadline = context.deadline(); + /// # } + /// ``` + #[must_use] + pub fn deadline(&self) -> Instant { + self.deadline + } + + /// Check cancellation before allocating a runtime resource. + /// + /// # Errors + /// Returns cancellation or deadline expiry, without cancelling cleanup. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let still_active = context.check().is_ok(); + /// # } + /// ``` + pub fn check(&self) -> Result<(), ExecutorError> { + if Instant::now() >= self.deadline { + return Err(ExecutorError::DeadlineExceeded); + } + if *self.cancellation.borrow() + || self.cancellation.has_changed().is_err() + || self + .external + .as_ref() + .is_some_and(|signal| *signal.borrow() || signal.has_changed().is_err()) + { + return Err(ExecutorError::Cancelled); + } + Ok(()) + } + + /// Wait for an explicit signal or the absolute deadline. + /// + /// ```no_run + /// # async fn stop(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let reason = context.cancelled().await; + /// # } + /// ``` + pub async fn cancelled(&self) -> CancellationReason { + let mut cancellation = self.cancellation.clone(); + let mut external = self.external.clone(); + match self.check() { + Err(ExecutorError::DeadlineExceeded) => return CancellationReason::Deadline, + Err(_) => return CancellationReason::Requested, + Ok(()) => {} + } + tokio::select! { + biased; + () = sleep_until(self.deadline) => CancellationReason::Deadline, + _ = cancellation.wait_for(|cancelled| *cancelled) => { + if Instant::now() >= self.deadline { CancellationReason::Deadline } else { CancellationReason::Requested } + } + () = external_cancelled(&mut external) => { + if Instant::now() >= self.deadline { CancellationReason::Deadline } else { CancellationReason::Requested } + } + } + } +} + +async fn external_cancelled(external: &mut Option>) { + if let Some(signal) = external { + let _ = signal.wait_for(|cancelled| *cancelled).await; + } else { + pending::<()>().await; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Settlement { + Pending, + Clean, + Uncertain, +} + +/// Attached ownership of an admitted invocation; Drop requests cancellation. +#[non_exhaustive] +pub struct InvocationHandle { + context: InvocationContext, + cancellation: watch::Sender, + response: Option>>, + settlement: watch::Receiver, + cancel_on_drop: bool, +} + +impl fmt::Debug for InvocationHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvocationHandle") + .field("id", &self.context.id) + .field("deadline", &self.context.deadline) + .field("settlement", &*self.settlement.borrow()) + .finish_non_exhaustive() + } +} + +impl InvocationHandle { + pub(crate) fn new( + context: InvocationContext, + cancellation: watch::Sender, + response: oneshot::Receiver>, + settlement: watch::Receiver, + ) -> Self { + Self { + context, + cancellation, + response: Some(response), + settlement, + cancel_on_drop: true, + } + } + + /// UUID used for this admitted attempt. + /// + /// ``` + /// # fn inspect(handle: &rustack_lambda_core::invocation::InvocationHandle) { + /// assert!(!handle.id().is_nil()); + /// # } + /// ``` + #[must_use] + pub fn id(&self) -> Uuid { + self.context.id + } + + /// Signal cancellation while retaining the ability to await cleanup. + /// + /// ``` + /// # fn stop(handle: &rustack_lambda_core::invocation::InvocationHandle) { + /// handle.cancel(); + /// # } + /// ``` + pub fn cancel(&self) { + self.cancellation.send_replace(true); + } + + /// Await the backend outcome within the admitted invocation deadline. + /// + /// # Errors + /// Returns backend errors, deadline expiry, or an uncertain worker failure. + /// Deadline failure signals cancellation; the supervisor still owns cleanup. + /// + /// ```no_run + /// # async fn finish(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// let response = handle.wait().await; + /// # } + /// ``` + pub async fn wait(mut self) -> Result { + let result = self.wait_response().await; + if self.response.is_none() { + self.cancel_on_drop = false; + } + result + } + + /// Read the business response while retaining the same cleanup handle. + /// + /// A dropped waiter can resume. A consumed result cannot be read twice. + /// + /// # Errors + /// Returns cancellation/deadline/backend failure, lost owner or consumed result. + /// + /// ```no_run + /// # async fn read(handle:&mut rustack_lambda_core::invocation::InvocationHandle) { + /// let response=handle.wait_response().await; + /// # } + /// ``` + pub async fn wait_response(&mut self) -> Result { + if let Err(error) = self.context.check() { + self.cancel(); + return Err(error); + } + let Some(receiver) = &mut self.response else { + return Err(ExecutorError::CleanupFailed( + "Invocation response already consumed".into(), + )); + }; + let response = tokio::select! { + biased; + reason=self.context.cancelled()=>{self.cancel();return Err(reason.error());}, + result=receiver=>result, + }; + self.response = None; + if let Err(error) = self.context.check() { + self.cancel(); + return Err(error); + } + response.map_err(|_| { + ExecutorError::CleanupFailed("Invocation worker terminated without settlement".into()) + })? + } + + /// Await confirmed cleanup independently of response/deadline/cancellation. + /// + /// Expiry or dropping this waiter leaves the handle resumable and owned. + /// + /// # Errors + /// Returns `CleanupFailed` on expiry, uncertainty or a closed settlement channel. + /// + /// ```no_run + /// # async fn join(handle:&mut rustack_lambda_core::invocation::InvocationHandle,deadline:tokio::time::Instant) { + /// let cleanup=handle.wait_settlement(deadline).await; + /// # } + /// ``` + pub async fn wait_settlement(&mut self, deadline: Instant) -> Result<(), ExecutorError> { + timeout_at(deadline, self.wait_cleanup()) + .await + .map_err(|_| { + ExecutorError::CleanupFailed("Invocation cleanup observer expired".into()) + })? + } + + /// Retain ownership while joining cleanup without an observation deadline. + /// + /// This waits only on the supervisor's state channel; backend IO retains its + /// own deadlines. Use `wait_settlement` for a bounded observer of the same work. + /// + /// # Errors + /// Returns `CleanupFailed` if the owner closes or reports uncertain settlement. + /// + /// ```no_run + /// # async fn join(handle:&mut rustack_lambda_core::invocation::InvocationHandle) { + /// let settled=handle.wait_cleanup().await; + /// # } + /// ``` + pub async fn wait_cleanup(&mut self) -> Result<(), ExecutorError> { + match self + .settlement + .wait_for(|state| *state != Settlement::Pending) + .await + { + Ok(state) if *state == Settlement::Clean => { + self.cancel_on_drop = false; + Ok(()) + } + _ => Err(ExecutorError::CleanupFailed( + "Invocation cleanup is not confirmed".into(), + )), + } + } + + /// Request cancellation and await confirmed cleanup, independently of the call deadline. + /// + /// # Errors + /// Returns `CleanupFailed` when the cleanup waiter expires or settlement is uncertain. + /// Cleanup remains owned after this method returns an error or is dropped. + /// + /// ```no_run + /// # async fn stop(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + /// let settled = handle.cancel_and_wait(deadline).await; + /// # } + /// ``` + pub async fn cancel_and_wait(mut self, deadline: Instant) -> Result<(), ExecutorError> { + self.cancel(); + self.wait_settlement(deadline).await + } + + /// Transfer Event invocation ownership to the supervisor after successful admission. + /// + /// ``` + /// # fn accept(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// handle.detach(); + /// # } + /// ``` + pub fn detach(mut self) { + self.cancel_on_drop = false; + } +} + +impl Drop for InvocationHandle { + fn drop(&mut self) { + if self.cancel_on_drop { + self.cancel(); + } + } +} + +/// Owned queued submission, retaining cleanup even when admission observation fails. +#[non_exhaustive] +pub struct InvocationSubmission { + handle: InvocationHandle, + accepted: Option>>, + admitted: bool, +} +impl fmt::Debug for InvocationSubmission { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvocationSubmission") + .field("handle", &self.handle) + .field("admitted", &self.admitted) + .finish_non_exhaustive() + } +} +impl InvocationSubmission { + pub(crate) fn new( + handle: InvocationHandle, + accepted: oneshot::Receiver>, + ) -> Self { + Self { + handle, + accepted: Some(accepted), + admitted: false, + } + } + /// Await execution admission while keeping the cleanup receipt on every path. + /// + /// # Errors + /// Returns service rejection, cancellation/deadline or a lost admission owner. + /// Dropping this waiter does not drop the submission or restart admission. + /// + /// ```no_run + /// # async fn accepted(submission:&mut rustack_lambda_core::invocation::InvocationSubmission) { + /// let result=submission.admitted().await; + /// # } + /// ``` + pub async fn admitted(&mut self) -> Result<(), LambdaServiceError> { + if self.admitted { + return Ok(()); + } + let Some(receiver) = &mut self.accepted else { + return Err(admission_error( + "Invocation admission result already consumed", + )); + }; + let result = tokio::select! { + biased; + reason=self.handle.context.cancelled()=>{self.handle.cancel();return Err(reason.error().into());}, + result=receiver=>result, + }; + self.accepted = None; + result.map_err(|_| admission_error("Invocation admission owner lost"))??; + self.handle + .context + .check() + .map_err(LambdaServiceError::from)?; + self.admitted = true; + Ok(()) + } + /// Signal cancellation before admission completes without consuming ownership. + /// + /// ``` + /// # fn cancel(submission:&rustack_lambda_core::invocation::InvocationSubmission) {submission.cancel();} + /// ``` + pub fn cancel(&self) { + self.handle.cancel(); + } + /// Transfer the invocation handle even after admission rejection or timeout. + /// + /// ``` + /// # fn transfer(submission:rustack_lambda_core::invocation::InvocationSubmission) { + /// let retained=submission.into_handle(); + /// # } + /// ``` + #[must_use] + pub fn into_handle(self) -> InvocationHandle { + self.handle + } +} +fn admission_error(message: &str) -> LambdaServiceError { + LambdaServiceError::ResourceNotReady { + message: message.into(), + } +} + +/// Single cleanup operation whose future survives cancellation of any waiter. +#[derive(Default)] +pub(crate) struct CleanupTask { + state: OwnerMutex>>>, +} + +enum CleanupState { + Running(JoinHandle>), + Complete(Result<(), ExecutorError>), +} + +impl fmt::Debug for CleanupTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CleanupTask") + .field("started", &self.state.lock().is_some()) + .finish() + } +} +impl fmt::Debug for CleanupState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Running(_) => "Running", + Self::Complete(Ok(())) => "Clean", + Self::Complete(Err(_)) => "Failed", + }) + } +} + +impl CleanupTask { + #[cfg(test)] + pub(crate) fn start(&self, factory: F) -> bool + where + F: FnOnce() -> Fut, + Fut: Future> + Send + 'static, + { + self.start_on(&Handle::current(), factory) + } + + pub(crate) fn start_on(&self, runtime: &Handle, factory: F) -> bool + where + F: FnOnce() -> Fut, + Fut: Future> + Send + 'static, + { + let mut state = self.state.lock(); + if state.is_some() { + return false; + } + // Retain the JoinHandle before returning; dropping a waiter never drops this future. + *state = Some(Arc::new(Mutex::new(CleanupState::Running( + runtime.spawn(factory()), + )))); + true + } + + pub(crate) fn is_finished(&self) -> bool { + let state = self.state.lock().clone(); + let Some(state) = state else { + return true; + }; + match state.try_lock() { + Ok(state) => match &*state { + CleanupState::Complete(_) => true, + CleanupState::Running(task) => task.is_finished(), + }, + Err(_) => false, + } + } + + pub(crate) async fn wait(&self) -> Result<(), ExecutorError> { + let state = self.state.lock().clone(); + let Some(state) = state else { + return Ok(()); + }; + let mut state = state.lock().await; + if let CleanupState::Running(task) = &mut *state { + let result = task.await.unwrap_or_else(|_| { + Err(ExecutorError::CleanupFailed( + "Cleanup supervisor task failed".into(), + )) + }); + *state = CleanupState::Complete(result); + } + match &*state { + CleanupState::Complete(result) => result.clone(), + CleanupState::Running(_) => Err(ExecutorError::CleanupFailed( + "Cleanup supervisor is unsettled".into(), + )), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use bytes::Bytes; + use tokio::sync::Semaphore; + + use super::*; + + #[tokio::test(start_paused = true)] + async fn test_should_prioritize_absolute_expiry_and_notice_closed_cancellation_owner() { + let deadline = Instant::now() + Duration::from_secs(2); + let (cancel, context) = InvocationContext::channel(deadline); + assert!(context.check().is_ok()); + cancel.send_replace(true); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(context.cancelled().await, CancellationReason::Deadline); + assert!(matches!( + context.check(), + Err(ExecutorError::DeadlineExceeded) + )); + + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(2)); + drop(cancel); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + assert!(matches!(context.check(), Err(ExecutorError::Cancelled))); + } + + #[tokio::test(start_paused = true)] + async fn test_should_discard_a_result_received_after_the_attempt_deadline() { + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(1)); + let (response, received) = oneshot::channel(); + let (_, settled) = watch::channel(Settlement::Clean); + let handle = InvocationHandle::new(context, cancel.clone(), received, settled); + response + .send(Ok(InvokeResponse::success(Bytes::new(), "$LATEST".into()))) + .unwrap(); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!( + handle.wait().await, + Err(ExecutorError::DeadlineExceeded) + )); + assert!( + *cancel.borrow(), + "expiry must request owned backend cleanup" + ); + } + + #[tokio::test(start_paused = true)] + async fn test_should_wait_for_cleanup_after_expiry_and_detach_only_explicitly() { + let (cancel, context) = InvocationContext::channel(Instant::now()); + let (_, received) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let handle = InvocationHandle::new(context, cancel.clone(), received, settled); + settlement.send_replace(Settlement::Clean); + handle + .cancel_and_wait(Instant::now() + Duration::from_secs(2)) + .await + .unwrap(); + assert!(*cancel.borrow()); + + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(2)); + let (_, received) = oneshot::channel(); + let (_, settled) = watch::channel(Settlement::Pending); + InvocationHandle::new(context, cancel.clone(), received, settled).detach(); + assert!(!*cancel.borrow(), "Event detach preserves admitted work"); + } + + #[tokio::test] + async fn test_should_retain_cleanup_resources_when_a_waiter_is_aborted() { + let cleanup = Arc::new(CleanupTask::default()); + let resources = Arc::new(Semaphore::new(1)); + let reservation = resources.clone().acquire_owned().await.unwrap(); + let release = Arc::new(Semaphore::new(0)); + let ready = Arc::new(Semaphore::new(0)); + let gate = Arc::clone(&release); + let started = Arc::clone(&ready); + assert!(cleanup.start(move || async move { + let _reservation = reservation; + started.add_permits(1); + gate.acquire().await.unwrap().forget(); + Ok(()) + })); + ready.acquire().await.unwrap().forget(); + let owner = Arc::clone(&cleanup); + let waiter = tokio::spawn(async move { owner.wait().await }); + tokio::task::yield_now().await; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!(resources.available_permits(), 0); + assert!(!cleanup.start(|| async { Ok(()) })); + + release.add_permits(1); + let (first, second) = tokio::join!(cleanup.wait(), cleanup.wait()); + first.unwrap(); + second.unwrap(); + assert_eq!(resources.available_permits(), 1); + cleanup.wait().await.unwrap(); + } + + #[tokio::test] + async fn test_should_preserve_failed_cleanup_and_supervised_panics_for_all_waiters() { + let cleanup = CleanupTask::default(); + assert!(cleanup.start(|| async { Err(ExecutorError::CleanupFailed("failed".into())) })); + for _ in 0..2 { + assert!( + matches!(cleanup.wait().await, Err(ExecutorError::CleanupFailed(message)) if message == "failed") + ); + } + let cleanup = CleanupTask::default(); + assert!(cleanup.start(|| async { panic!("intentional cleanup task panic") })); + for _ in 0..2 { + assert!(matches!( + cleanup.wait().await, + Err(ExecutorError::CleanupFailed(_)) + )); + } + } +} + +#[cfg(test)] +mod submission_tests { + use std::time::Duration; + + use bytes::Bytes; + + use super::*; + #[tokio::test(start_paused = true)] + async fn test_should_propagate_external_cancellation_and_keep_deadline_precedence() { + let (external, signal) = watch::channel(false); + let (_own, context) = InvocationContext::channel_with_cancellation( + Instant::now() + Duration::from_secs(1), + Some(signal), + ); + external.send_replace(true); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(context.cancelled().await, CancellationReason::Deadline); + let (external, signal) = watch::channel(false); + let (_own, context) = InvocationContext::channel_with_cancellation( + Instant::now() + Duration::from_secs(1), + Some(signal), + ); + drop(external); + assert!(matches!(context.check(), Err(ExecutorError::Cancelled))); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + } + #[tokio::test(start_paused = true)] + async fn test_should_keep_cleanup_receipt_after_rejected_admission_and_waiter_timeout() { + let (cancel, context) = + InvocationContext::channel(Instant::now() + Duration::from_secs(10)); + let (_response, receiver) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let handle = InvocationHandle::new(context, cancel, receiver, settled); + let (admitted, accepted) = oneshot::channel(); + let mut submission = InvocationSubmission::new(handle, accepted); + admitted + .send(Err(LambdaServiceError::TooManyRequests)) + .unwrap(); + assert!(matches!( + submission.admitted().await, + Err(LambdaServiceError::TooManyRequests) + )); + let mut handle = submission.into_handle(); + assert!( + handle + .wait_settlement(Instant::now() + Duration::from_secs(1)) + .await + .is_err() + ); + settlement.send_replace(Settlement::Clean); + handle.wait_cleanup().await.unwrap(); + } + #[tokio::test(start_paused = true)] + async fn test_should_resume_response_wait_and_reject_second_consumption_without_panic() { + let (cancel, context) = + InvocationContext::channel(Instant::now() + Duration::from_secs(10)); + let (response, receiver) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let mut handle = InvocationHandle::new(context, cancel.clone(), receiver, settled); + assert!( + timeout_at( + Instant::now() + Duration::from_secs(1), + handle.wait_response() + ) + .await + .is_err() + ); + assert!(!*cancel.borrow()); + response + .send(Ok(InvokeResponse::success( + Bytes::from_static(b"5"), + "$LATEST".into(), + ))) + .unwrap(); + assert_eq!( + handle.wait_response().await.unwrap().payload, + Bytes::from_static(b"5") + ); + assert!(matches!( + handle.wait_response().await, + Err(ExecutorError::CleanupFailed(_)) + )); + settlement.send_replace(Settlement::Clean); + handle.wait_cleanup().await.unwrap(); + } + #[tokio::test(start_paused = true)] + async fn test_should_not_certify_uncertain_or_lost_cleanup_from_owned_submission() { + for state in [Settlement::Pending, Settlement::Uncertain] { + let (cancel, context) = + InvocationContext::channel(Instant::now() + Duration::from_secs(10)); + let (_response, receiver) = oneshot::channel(); + let (settlement, settled) = watch::channel(state); + let mut handle = InvocationHandle::new(context, cancel, receiver, settled); + drop(settlement); + assert!(matches!( + handle.wait_cleanup().await, + Err(ExecutorError::CleanupFailed(_)) + )); + } + } +} diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 1ed63610..8f311cb6 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -10,11 +10,20 @@ #![forbid(unsafe_code)] +pub mod artifact; pub mod code; pub mod config; +pub mod deployment; +mod deployment_input; +pub mod diagnostics; +#[cfg(unix)] +pub mod docker; pub mod error; pub mod executor; +mod function_config; pub mod handler; +pub mod identity; +pub mod invocation; pub mod provider; pub mod resolver; pub mod storage; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 7db04184..b7e75c05 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -5,18 +5,21 @@ //! implemented; later phases (versions, aliases, permissions, tags, URLs) //! return appropriate errors until implemented. -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use bytes::Bytes; use serde::{Deserialize, Serialize}; use tracing::info; /// Maximum deployment package size (50 MB zipped, per Appendix C). -const MAX_ZIP_SIZE: u64 = 50 * 1024 * 1024; +const MAX_ZIP_SIZE: usize = crate::artifact::MAX_ZIP_BYTES; /// Maximum synchronous invoke payload size (6 MB, per Appendix C). const MAX_SYNC_PAYLOAD: usize = 6 * 1024 * 1024; +/// Hard upper bound for one code-source fetch before any artifact is staged. +const CODE_FETCH_TIMEOUT: Duration = Duration::from_secs(30); + /// Maximum S3 bucket name length (per S3 naming rules). const MAX_S3_BUCKET_LEN: usize = 63; @@ -50,15 +53,27 @@ use rustack_lambda_model::{ TracingConfigResponse, VpcConfigResponse, }, }; +use tokio::{ + sync::{SemaphorePermit, watch}, + time::{Instant, timeout_at}, +}; use crate::{ + artifact::{Architecture, DeploymentArtifact, ImageArtifact, ImageReference}, code::{S3CodeFetchError, S3CodeFetcher, UnavailableS3CodeFetcher}, config::LambdaConfig, + deployment::{DeploymentState, check_revision}, + diagnostics::InvocationWorkStatus, error::LambdaServiceError, executor::{ - AutoExecutor, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, NoopExecutor, - PackageType, SquibExecutor, + AutoExecutor, ExecutionRevision, Executor, ExecutorBackend, ExecutorError, InvokeRequest, + NativeExecutor, NoopExecutor, SquibExecutor, + }, + function_config::{ + validate_create_settings, validate_execution_settings, validate_provided_settings, }, + identity::FunctionResourceId, + invocation::{InvocationHandle, InvocationSubmission}, resolver::{ alias_arn, function_arn, function_version_arn, layer_arn, layer_version_arn, parse_layer_version_arn, resolve_function_ref, resolve_version, @@ -67,9 +82,14 @@ use crate::{ AliasRecord, EventInvokeConfigRecord, EventSourceMappingRecord, EventSourceMappingStore, EventSourceMappingStoreSnapshot, FunctionRecord, FunctionStore, FunctionStoreSnapshot, FunctionUrlConfigRecord, LayerStore, LayerStoreSnapshot, LayerVersionRecord, - PolicyDocument, PolicyStatement, VersionRecord, compute_sha256, + PolicyDocument, PolicyStatement, StagedArtifact, VersionRecord, compute_sha256, }, }; +#[cfg(unix)] +use crate::{ + deployment::{ImageDeploymentConfig, ImageDeploymentManager}, + docker::{DockerCodeArchive, DockerError, ImageLeasePort}, +}; /// Selects the AWS `Invoke` invocation type. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -84,12 +104,12 @@ pub enum InvokeKind { /// Build the executor backend for the given configuration. /// -/// `Disabled` and unsupported Docker selections fail execution explicitly. +/// `Disabled` and unqualified Docker constructors fail execution explicitly. /// `Auto` is an explicit operator opt-in and chooses the concrete backend per invocation. fn build_executor(config: &LambdaConfig) -> Arc { match config.executor { ExecutorBackend::Disabled => Arc::new(NoopExecutor::new()), - ExecutorBackend::Docker => Arc::new(NoopExecutor::docker_unavailable()), + ExecutorBackend::Docker => Arc::new(NoopExecutor::without_docker_runtime()), ExecutorBackend::Auto => Arc::new(AutoExecutor::new( config.max_warm_instances, config.idle_timeout, @@ -105,6 +125,9 @@ fn build_executor(config: &LambdaConfig) -> Arc { } } +mod snapshot; +pub use snapshot::DockerArchiveProfile; + /// Outcome of an [`RustackLambda::invoke`] call, mapping cleanly to AWS /// `Invoke` response codes. #[derive(Debug, Clone)] @@ -126,13 +149,15 @@ pub enum InvokeOutcome { /// All operations are implemented as methods that return domain types or errors. #[derive(Debug)] pub struct RustackLambda { - store: FunctionStore, - layer_store: LayerStore, - esm_store: EventSourceMappingStore, + store: Arc, + layer_store: Arc, + esm_store: Arc, config: LambdaConfig, executor: Arc, code_fetcher: Arc, work: crate::work::WorkManager, + #[cfg(unix)] + deployments: Option, } /// Serializable Lambda provider snapshot. @@ -165,7 +190,77 @@ struct CodeSource<'a> { image_uri: Option<&'a str>, } +#[derive(Debug)] +enum ProcessedCode<'a> { + Zip(StagedArtifact<'a>), + Image(ImageReference), +} + +impl ProcessedCode<'_> { + fn parts(&self) -> (String, u64, Option, Option, Option) { + match self { + Self::Zip(staged) => { + let artifact = staged.artifact(); + ( + artifact.sha256().to_owned(), + artifact.zip_bytes().len() as u64, + Some(artifact.zip_bytes().clone()), + Some(artifact.code_root().to_path_buf()), + None, + ) + } + Self::Image(reference) => ( + String::new(), + 0, + None, + None, + Some(reference.as_str().into()), + ), + } + } + + fn commit(self) { + if let Self::Zip(staged) = self { + staged.commit(); + } + } +} + +#[derive(Debug)] +struct InvocationOptions { + asynchronous: bool, + deadline: Option, + capture_logs: bool, + cancellation: Option>, +} +fn check_external_cancellation( + signal: Option<&watch::Receiver>, +) -> Result<(), LambdaServiceError> { + if signal.is_some_and(|signal| *signal.borrow() || signal.has_changed().is_err()) { + Err(ExecutorError::Cancelled.into()) + } else { + Ok(()) + } +} + impl RustackLambda { + /// Sample real invocation ownership without admission or supervisor startup. + /// + /// The counters are independently sampled and include retained cleanup and + /// backend pool waiting. They do not count physical running containers. + /// + /// ``` + /// use rustack_lambda_core::{config::LambdaConfig, provider::RustackLambda}; + /// let provider = RustackLambda::new(LambdaConfig::default()); + /// let status = provider.work_status(); + /// assert_eq!(status.active_executions, 0); + /// assert_eq!(status.limits.active_executions, 32); + /// ``` + #[must_use] + pub fn work_status(&self) -> InvocationWorkStatus { + self.work.status() + } + /// Create a new Lambda provider with the given store and config. /// /// The executor is selected from `config.executor`. @@ -173,13 +268,15 @@ impl RustackLambda { pub fn with_store(store: FunctionStore, config: LambdaConfig) -> Self { let executor = build_executor(&config); Self { - store, - layer_store: LayerStore::new(), - esm_store: EventSourceMappingStore::new(), + store: Arc::new(store), + layer_store: Arc::new(LayerStore::new()), + esm_store: Arc::new(EventSourceMappingStore::new()), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -190,13 +287,15 @@ impl RustackLambda { let store = FunctionStore::new(code_dir); let executor = build_executor(&config); Self { - store, - layer_store: LayerStore::new(), - esm_store: EventSourceMappingStore::new(), + store: Arc::new(store), + layer_store: Arc::new(LayerStore::new()), + esm_store: Arc::new(EventSourceMappingStore::new()), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -209,13 +308,15 @@ impl RustackLambda { executor: Arc, ) -> Self { Self { - store, - layer_store: LayerStore::new(), - esm_store: EventSourceMappingStore::new(), + store: Arc::new(store), + layer_store: Arc::new(LayerStore::new()), + esm_store: Arc::new(EventSourceMappingStore::new()), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -229,6 +330,89 @@ impl RustackLambda { self } + /// Attach qualified image preparation after executor readiness probes succeed. + /// + /// # Errors + /// Rejects non-Docker/unavailable executors, duplicate attachment or invalid owner + /// configuration. The runtime must retain the image owner until all deployment/execution + /// consumers settle. + /// + /// ```no_run + /// # fn attach(provider:rustack_lambda_core::provider::RustackLambda, port:rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let provider = provider.with_image_preparation(port, rustack_lambda_core::deployment::ImageDeploymentConfig::default())?; + /// # Ok(()) } + /// ``` + #[cfg(unix)] + pub fn with_image_preparation( + mut self, + port: ImageLeasePort, + config: ImageDeploymentConfig, + ) -> Result { + if self.config.executor != ExecutorBackend::Docker + || self.executor.image_available().is_err() + || self.deployments.is_some() + { + return Err(LambdaServiceError::DockerNotAvailable); + } + self.deployments = Some(ImageDeploymentManager::start( + self.store.clone(), + port, + config, + )?); + Ok(self) + } + + #[cfg(unix)] + fn image_deployments(&self) -> Result<&ImageDeploymentManager, LambdaServiceError> { + if self.config.executor != ExecutorBackend::Docker + || self.executor.image_available().is_err() + { + return Err(LambdaServiceError::DockerNotAvailable); + } + self.deployments + .as_ref() + .ok_or(LambdaServiceError::DockerNotAvailable) + } + + fn create_image_deployment( + &self, + record: FunctionRecord, + ) -> Result { + #[cfg(unix)] + { + self.image_deployments()?.create(record) + } + #[cfg(not(unix))] + { + drop(record); + Err(LambdaServiceError::DockerNotAvailable) + } + } + + fn update_image_deployment( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result { + #[cfg(unix)] + { + let accepted = self.image_deployments()?.update_with_revision( + before, + reference, + architecture, + requested, + )?; + Ok(self.build_function_configuration(&accepted, &accepted.latest)) + } + #[cfg(not(unix))] + { + let _ = (before, reference, architecture, requested); + Err(LambdaServiceError::DockerNotAvailable) + } + } + /// Borrow the executor (e.g. for tests asserting backend behavior). #[must_use] pub fn executor(&self) -> &Arc { @@ -238,10 +422,126 @@ impl RustackLambda { /// Stop all warm executor instances. Wired into the rustack server's /// graceful shutdown path. pub async fn shutdown(&self) { - if let Err(error) = self.work.quiesce(Duration::ZERO).await { - tracing::error!(%error, "Lambda shutdown cancelled unfinished work"); + if let Err(error) = self + .shutdown_before(Instant::now() + Duration::from_secs(10)) + .await + { + tracing::error!(%error, "Lambda shutdown did not confirm cleanup"); + } + } + + /// Stop deployment, invocation and executor owners under one absolute deadline. + /// # Errors + /// Returns retained deployment/work/executor failures or deadline expiry. + /// Repeating a timed-out observation resumes the same lower owners. + /// ```no_run + /// # async fn stop(provider:&rustack_lambda_core::provider::RustackLambda){ + /// let result=provider.shutdown_before(tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await; + /// # } + /// ``` + pub async fn shutdown_before(&self, deadline: Instant) -> Result<(), LambdaServiceError> { + #[cfg(unix)] + let deployment = match &self.deployments { + Some(owner) => owner.shutdown(deadline).await, + None => Ok(()), + }; + self.work.cancel_all(); + let work = self + .work + .quiesce(deadline.saturating_duration_since(Instant::now())) + .await; + let executor = timeout_at(deadline, self.executor.shutdown()) + .await + .map_err(|_| { + ExecutorError::CleanupFailed("Lambda executor cleanup observation expired".into()) + }) + .and_then(|result| result); + #[cfg(unix)] + deployment?; + work?; + executor?; + if Instant::now() >= deadline { + return Err(ExecutorError::DeadlineExceeded.into()); + } + Ok(()) + } + + /// Check selected backend/deployment health without invoking a function. + /// Ordinary handler failures do not poison this observation. + /// ```no_run + /// # fn ready(provider:&rustack_lambda_core::provider::RustackLambda){let ready=provider.is_ready();} + /// ``` + #[must_use] + pub fn is_ready(&self) -> bool { + if self.config.executor != ExecutorBackend::Docker { + return true; + } + if self.executor.image_available().is_err() { + return false; + } + #[cfg(unix)] + { + self.deployments.as_ref().is_some_and(|owner| { + let status = owner.status(); + status.accepting && !status.failed + }) + } + #[cfg(not(unix))] + { + false + } + } + + /// Reject metadata-only image archives before opening ordinary Docker ingress. + /// # Errors + /// Returns an error if usable saved images lack live immutable leases or the + /// qualified backend is unavailable. Failed creations remain inspectable. + /// ```no_run + /// # fn restore(provider:&rustack_lambda_core::provider::RustackLambda){let result=provider.validate_runtime_restore();} + /// ``` + pub fn validate_runtime_restore(&self) -> Result<(), LambdaServiceError> { + if self.config.executor == ExecutorBackend::Docker && self.store.has_unleased_images() { + return Err(invalid_parameter( + "Docker archive contains an unleased usable image", + )); } - self.executor.shutdown().await; + if self.config.executor == ExecutorBackend::Docker { + self.executor.image_available()?; + if let Some(architecture) = self.executor.docker_zip_architecture() { + self.executor.zip_available(architecture)?; + } + } + Ok(()) + } + + /// Configured Docker archive bound, separate from Native archive payloads. + /// + /// ```no_run + /// # fn limit(provider: &rustack_lambda_core::provider::RustackLambda) { + /// let limit = provider.archive_payload_limit(); + /// # } + /// ``` + #[must_use] + pub fn archive_payload_limit(&self) -> Option { + self.docker_archive_profile() + .map(DockerArchiveProfile::payload_limit) + } + + /// Configured Docker archive profile, independently of current readiness. + /// + /// ``` + /// # use rustack_lambda_core::{config::LambdaConfig, provider::RustackLambda}; + /// let provider = RustackLambda::new(LambdaConfig::default()); + /// assert_eq!(provider.docker_archive_profile(), None); + /// ``` + #[must_use] + pub fn docker_archive_profile(&self) -> Option { + (self.config.executor == ExecutorBackend::Docker).then(|| { + self.executor.docker_zip_architecture().map_or( + DockerArchiveProfile::ImageOnly, + DockerArchiveProfile::ProvidedZip, + ) + }) } /// Stop accepting invocation work and drain it before snapshot export. @@ -249,20 +549,36 @@ impl RustackLambda { /// # Errors /// Returns an error after cancelling remaining tasks when the deadline expires. pub async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { - let deadline = tokio::time::Instant::now() + timeout; - let cleanup_budget = (timeout / 10).min(Duration::from_secs(1)); + let deadline = Instant::now() + .checked_add(timeout) + .ok_or_else(|| invalid_parameter("Invalid Lambda quiesce budget"))?; + #[cfg(unix)] + if let Some(owner) = &self.deployments { + if let Err(error) = owner.quiesce(deadline).await { + owner.cancel(); + self.work.cancel_all(); + return Err(error.into()); + } + } + let remaining = deadline.saturating_duration_since(Instant::now()); + let cleanup_budget = (remaining / 10).min(Duration::from_secs(1)); let result = self .work - .quiesce(timeout.saturating_sub(cleanup_budget)) + .quiesce(remaining.saturating_sub(cleanup_budget)) .await; if result.is_err() - && tokio::time::timeout_at(deadline, self.executor.shutdown()) - .await - .is_err() + && !matches!( + timeout_at(deadline, self.executor.shutdown()).await, + Ok(Ok(())) + ) { tracing::error!("Lambda executor cancellation deadline exceeded"); } - result + result?; + if self.config.executor == ExecutorBackend::Docker { + self.executor.image_available()?; + } + Ok(()) } /// Export a point-in-time snapshot of Lambda resources. @@ -275,6 +591,46 @@ impl RustackLambda { } } + /// Export archive metadata only when the installed profile can restore it. + /// + /// # Errors + /// Returns unavailable/unleased Image backend, excessive metadata, worker or + /// deadline failure. Native ZIP export keeps its existing representation. + /// + /// ```no_run + /// # async fn save(provider: &rustack_lambda_core::provider::RustackLambda) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let saved = provider.export_snapshot_before(tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; + /// # Ok(()) } + /// ``` + pub async fn export_snapshot_before( + &self, + deadline: Instant, + ) -> Result { + if self.config.executor == ExecutorBackend::Docker { + self.validate_runtime_restore()?; + self.store.check_snapshot_counts( + snapshot::MAX_IMAGE_FUNCTIONS, + snapshot::MAX_IMAGE_REVISIONS, + )?; + #[cfg(unix)] + { + let saved = snapshot::export_docker( + self, + deadline, + self.docker_archive_profile() + .ok_or(LambdaServiceError::DockerNotAvailable)?, + ) + .await?; + self.validate_runtime_restore()?; + self.image_deployments()?; + return Ok(saved); + } + #[cfg(not(unix))] + return Err(LambdaServiceError::DockerNotAvailable); + } + Ok(self.export_snapshot()) + } + /// Replace Lambda resources from a snapshot. /// /// # Errors @@ -284,6 +640,66 @@ impl RustackLambda { &self, snapshot: LambdaSnapshot, ) -> Result<(), LambdaServiceError> { + let deadline = Instant::now() + .checked_add(Duration::from_secs(300)) + .ok_or_else(|| invalid_parameter("Invalid Lambda restore deadline"))?; + self.import_snapshot_before(snapshot, deadline).await + } + + /// Restore quiescent/startup resources before one captured commit deadline. + /// + /// Docker validates the complete candidate and reacquires exact saved content + /// before replacing any resource. There is no live import HTTP action. + /// + /// # Errors + /// Returns bounded validation, unavailable backend, immutable content, quota, + /// deadline or retained cleanup failure without publishing a failed candidate. + /// + /// ```no_run + /// # async fn restore(provider: &rustack_lambda_core::provider::RustackLambda, saved: rustack_lambda_core::provider::LambdaSnapshot) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// provider.import_snapshot_before(saved, tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; + /// # Ok(()) } + /// ``` + pub async fn import_snapshot_before( + &self, + snapshot: LambdaSnapshot, + deadline: Instant, + ) -> Result<(), LambdaServiceError> { + if self.config.executor == ExecutorBackend::Docker { + #[cfg(unix)] + { + self.image_deployments()?; + let profile = self + .docker_archive_profile() + .ok_or(LambdaServiceError::DockerNotAvailable)?; + if let Some(architecture) = profile.zip_architecture() { + self.executor.zip_available(architecture)?; + } + let snapshot = snapshot::validate_docker( + snapshot, + deadline, + &self.config.default_region, + &self.config.account_id, + profile, + ) + .await?; + let manager = self.image_deployments()?; + let prepared = manager + .prepare_restore(snapshot.functions, deadline, profile) + .await?; + self.image_deployments()?.restore_available()?; + if let Some(architecture) = profile.zip_architecture() { + self.executor.zip_available(architecture)?; + } + prepared.commit_before(deadline)?; + self.layer_store.import_snapshot(snapshot.layers); + self.esm_store + .import_snapshot(snapshot.event_source_mappings); + return Ok(()); + } + #[cfg(not(unix))] + return Err(LambdaServiceError::DockerNotAvailable); + } self.store.import_snapshot(snapshot.functions).await?; self.layer_store.import_snapshot(snapshot.layers); self.esm_store @@ -323,6 +739,11 @@ impl RustackLambda { /// /// Validates the input, stores the deployment package, and inserts /// the function record into the store. + /// + /// # Errors + /// + /// Rejects invalid deployment settings, duplicate functions, unavailable image execution, + /// exhausted deployment admission, or ZIP/S3 staging and storage failures. #[allow(clippy::too_many_lines)] pub async fn create_function( &self, @@ -458,9 +879,19 @@ impl RustackLambda { } } - // Process code. - let (code_sha256, code_size, zip_bytes, code_path, image_uri) = - self.process_code(name, "$LATEST", code_source).await?; + validate_image_publication(&package_type, input.publish.unwrap_or(false))?; + let architecture = validate_create_settings(&input)?; + if package_type == "Zip" { + self.validate_docker_zip_settings( + input.runtime.as_deref(), + input.handler.as_deref(), + architecture, + )?; + } + let processed = self.process_code(name, "$LATEST", code_source).await?; + self.validate_docker_zip_candidate(&processed, architecture) + .await?; + let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); let timeout = input.timeout.unwrap_or(3); let memory_size = input.memory_size.unwrap_or(128); @@ -487,8 +918,9 @@ impl RustackLambda { package_type: package_type.clone(), code_path, image_uri, + image: ImageArtifact::Unresolved, zip_bytes, - state: "Active".to_owned(), + deployment: DeploymentState::Ready, last_modified: now.clone(), architectures, ephemeral_storage_size, @@ -505,6 +937,7 @@ impl RustackLambda { }; let record = FunctionRecord { + resource_id: FunctionResourceId::new(), name: name.clone(), arn: arn.clone(), latest: version_record, @@ -520,8 +953,18 @@ impl RustackLambda { created_at: now, }; + if package_type == "Image" { + let accepted = self.create_image_deployment(record)?; + return Ok(self.build_function_configuration(&accepted, &accepted.latest)); + } let should_publish = input.publish.unwrap_or(false); + self.validate_docker_zip_settings( + record.latest.runtime.as_deref(), + record.latest.handler.as_deref(), + architecture, + )?; self.store.insert(record)?; + processed.commit(); // If publish=true, publish version 1 immediately. let config = if should_publish { @@ -537,6 +980,11 @@ impl RustackLambda { } /// Get function information including configuration, code location, and tags. + /// + /// # Errors + /// + /// Rejects malformed function references or qualifiers, and missing functions, versions or + /// aliases. pub fn get_function( &self, function_ref: &str, @@ -552,15 +1000,16 @@ impl RustackLambda { let version = resolve_version(&record, qualifier)?; let config = self.build_function_configuration(&record, version); + let is_image = version.package_type == "Image"; let code_location = FunctionCodeLocation { - repository_type: Some("S3".to_owned()), - location: Some(format!( + repository_type: Some(if is_image { "ECR" } else { "S3" }.into()), + location: (!is_image).then(|| format!( "https://awslambda-{region}-tasks.s3.{region}.amazonaws.com/snapshots/{account}/{name}", region = self.config.default_region, account = self.config.account_id, )), image_uri: version.image_uri.clone(), - resolved_image_uri: None, + resolved_image_uri: version.image.resolved().map(crate::artifact::ResolvedImage::resolved_uri), }; let tags = if record.tags.is_empty() { @@ -577,6 +1026,11 @@ impl RustackLambda { } /// Get function configuration only. + /// + /// # Errors + /// + /// Rejects malformed function references or qualifiers, and missing functions, versions or + /// aliases. pub fn get_function_configuration( &self, function_ref: &str, @@ -594,6 +1048,10 @@ impl RustackLambda { } /// Get the code signing config association for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn get_function_code_signing_config( &self, function_ref: &str, @@ -608,10 +1066,35 @@ impl RustackLambda { } /// Update function code. + /// + /// # Errors + /// + /// Rejects invalid or immutable references, incompatible code sources/settings, stale + /// revisions, in-progress deployments or unavailable/exhausted image preparation. ZIP/S3 + /// staging and validation failures propagate; `DryRun` preserves metadata and performs no + /// deployment effects. pub async fn update_function_code( &self, function_ref: &str, input: UpdateFunctionCodeInput, + ) -> Result { + self.update_code(function_ref, input, None).await + } + + pub(crate) async fn update_function_code_parsed( + &self, + function_ref: &str, + input: UpdateFunctionCodeInput, + permit: SemaphorePermit<'static>, + ) -> Result { + self.update_code(function_ref, input, Some(permit)).await + } + + async fn update_code( + &self, + function_ref: &str, + mut input: UpdateFunctionCodeInput, + mut admitted: Option>, ) -> Result { let code_source = CodeSource { zip_file_b64: input.zip_file.as_deref(), @@ -622,126 +1105,117 @@ impl RustackLambda { }; validate_code_source(&code_source)?; - // Validate that some code source is provided. - if code_source.zip_file_b64.is_none() - && code_source.image_uri.is_none() - && code_source.s3_bucket.is_none() - { - return Err(LambdaServiceError::InvalidParameter { - message: "Provide at least one of ZipFile, S3Bucket, or ImageUri.".to_owned(), - }); - } - - let (name, _) = resolve_function_ref(function_ref)?; + let name = mutable_function_name(function_ref)?; let should_publish = input.publish.unwrap_or(false); let before = self.get_record(&name)?; - if input.architectures.as_ref().is_some_and(|architectures| { - architectures.len() != 1 - || architectures - .iter() - .any(|architecture| architecture != "arm64" && architecture != "x86_64") - }) { - return Err(invalid_parameter( - "Exactly one supported architecture is required", - )); + let architecture = validate_code_update(&before, &input, code_source)?; + + if input.dry_run.unwrap_or(false) { + if let Some(reference) = code_source.image_uri { + reference + .parse::() + .map_err(|_| invalid_parameter("Invalid ImageUri"))?; + } + let _inline_permit = if let Some(encoded) = input.zip_file.take() { + Some(crate::deployment_input::validate_inline(encoded, admitted.take()).await?) + } else { + None + }; + let current = self.get_record(&name)?; + check_revision(input.revision_id.as_deref(), ¤t.latest.revision_id)?; + check_staged_revision(¤t, &before)?; + current.latest.deployment.ensure_mutable()?; + return Ok(self.build_function_configuration(¤t, ¤t.latest)); } - let (code_sha256, code_size, zip_bytes, code_path, image_uri) = - self.process_code(&name, "$LATEST", code_source).await?; - - let config = - self.store - .update_if_revision(&name, Some(&before.latest.revision_id), |record| { - let now = now_iso8601(); - record.latest.code_sha256 = code_sha256; - record.latest.code_size = code_size; - record.latest.zip_bytes = zip_bytes; - record.latest.code_path = code_path; - record.latest.image_uri = image_uri; - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - - if let Some(archs) = input.architectures.clone() { - record.latest.architectures = archs; - } - if should_publish { - let number = record.next_version; - record.next_version = number.saturating_add(1); - let mut published = record.latest.clone(); - published.version = number.to_string(); - let configuration = self.build_function_configuration(record, &published); - record.versions.insert(number, published); - configuration - } else { - self.build_function_configuration(record, &record.latest) - } - })?; + if before.latest.package_type == "Image" { + let reference = code_source + .image_uri + .ok_or_else(|| invalid_parameter("ImageUri is required"))? + .parse() + .map_err(|_| invalid_parameter("Invalid ImageUri"))?; + return self.update_image_deployment( + &before, + reference, + architecture, + input.revision_id.as_deref(), + ); + } + if before.latest.package_type == "Zip" { + self.validate_docker_zip_settings( + before.latest.runtime.as_deref(), + before.latest.handler.as_deref(), + architecture, + )?; + } + let processed = self.process_code(&name, "$LATEST", code_source).await?; + self.validate_docker_zip_candidate(&processed, architecture) + .await?; + let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); + + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + check_staged_revision(record, &before)?; + record.latest.deployment.ensure_mutable()?; + self.validate_docker_zip_settings( + record.latest.runtime.as_deref(), + record.latest.handler.as_deref(), + architecture, + )?; + let now = now_iso8601(); + record.latest.code_sha256 = code_sha256; + record.latest.code_size = code_size; + record.latest.zip_bytes = zip_bytes; + record.latest.code_path = code_path; + record.latest.image_uri = image_uri; + record.latest.image = ImageArtifact::Unresolved; + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + + if let Some(archs) = input.architectures.clone() { + record.latest.architectures = archs; + } + record.latest.deployment = DeploymentState::Ready; + let configuration = if should_publish { + let number = record.next_version; + record.next_version = number.saturating_add(1); + let mut published = record.latest.clone(); + published.version = number.to_string(); + let configuration = self.build_function_configuration(record, &published); + record.versions.insert(number, published); + configuration + } else { + self.build_function_configuration(record, &record.latest) + }; + Ok(configuration) + })?; + processed.commit(); info!(function_name = %name, "updated Lambda function code"); Ok(config) } /// Update function configuration (handler, runtime, env vars, etc.). + /// + /// # Errors + /// + /// Rejects invalid or immutable references/settings, stale revisions, in-progress deployments, + /// missing functions or changes that violate the stored execution configuration. pub fn update_function_configuration( &self, function_ref: &str, input: &UpdateFunctionConfigurationInput, ) -> Result { - let (name, _) = resolve_function_ref(function_ref)?; + let name = mutable_function_name(function_ref)?; - // Validate handler length (Appendix C: max 128 chars). - if let Some(ref handler) = input.handler { - if handler.len() > 128 { - return Err(LambdaServiceError::InvalidParameter { - message: "Handler must be at most 128 characters".to_owned(), - }); - } - } - // Validate description length (Appendix C: max 256 chars). - if let Some(ref desc) = input.description { - if desc.len() > 256 { - return Err(LambdaServiceError::InvalidParameter { - message: "Description must be at most 256 characters".to_owned(), - }); - } - } - // Validate timeout (Appendix C: 1-900 seconds). - if let Some(timeout) = input.timeout { - if !(1..=900).contains(&timeout) { - return Err(LambdaServiceError::InvalidParameter { - message: "Timeout must be between 1 and 900 seconds".to_owned(), - }); - } - } - // Validate memory size (Appendix C: 128-10240 MB). - if let Some(memory) = input.memory_size { - if !(128..=10_240).contains(&memory) { - return Err(LambdaServiceError::InvalidParameter { - message: "Memory size must be between 128 and 10240 MB".to_owned(), - }); - } - } - // Validate ephemeral storage (Appendix C: 512-10240 MB). - if let Some(ref ephemeral) = input.ephemeral_storage { - if !(512..=10_240).contains(&ephemeral.size) { - return Err(LambdaServiceError::InvalidParameter { - message: "Ephemeral storage must be between 512 and 10240 MB".to_owned(), - }); - } - } - // Validate environment variables total size (Appendix C: 4 KB). - if let Some(ref env) = input.environment { - if let Some(ref vars) = env.variables { - let total_size: usize = vars.iter().map(|(k, v)| k.len() + v.len()).sum(); - if total_size > 4096 { - return Err(LambdaServiceError::InvalidParameter { - message: "Environment variables total size exceeds 4 KB limit".to_owned(), - }); - } - } - } + validate_configuration_update(input)?; - let config = self.store.update(&name, |record| { + let before = self.get_record(&name)?; + check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + check_staged_revision(record, &before)?; + record.latest.deployment.ensure_mutable()?; let now = now_iso8601(); if let Some(runtime) = &input.runtime { @@ -793,7 +1267,29 @@ impl RustackLambda { record.latest.last_modified = now; record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - self.build_function_configuration(record, &record.latest) + if matches!(record.latest.deployment, DeploymentState::UpdateFailed(_)) { + record.latest.deployment = DeploymentState::Ready; + } + if let Some(image) = record.latest.image.resolved() { + image + .metadata() + .execution + .plan(record.latest.image_config.as_ref()) + .map_err(|_| { + invalid_parameter("Invalid effective image entry configuration") + })?; + } + if record.latest.package_type == "Zip" + && self.config.executor == ExecutorBackend::Docker + { + let architecture = validate_execution_settings(&record.latest)?; + self.validate_docker_zip_settings( + record.latest.runtime.as_deref(), + record.latest.handler.as_deref(), + architecture, + )?; + } + Ok(self.build_function_configuration(record, &record.latest)) })?; info!(function_name = %name, "updated Lambda function configuration"); @@ -801,6 +1297,12 @@ impl RustackLambda { } /// Delete a function. + /// + /// # Errors + /// + /// Rejects malformed references, nonnumeric deletion qualifiers, missing functions/versions, + /// or artifact-root preflight failures. Final cleanup revalidates the root and can fail + /// after metadata removal if the root changes or filesystem inspection fails. pub async fn delete_function( &self, function_ref: &str, @@ -877,104 +1379,333 @@ impl RustackLambda { } } - // --------------------------------------------------------------- - // Phase 0: Invoke - // --------------------------------------------------------------- + // --------------------------------------------------------------- + // Phase 0: Invoke + // --------------------------------------------------------------- + + /// Invoke a function through shared admission and supervised execution. + /// + /// # Errors + /// Rejects invalid payload/reference, missing resources, unavailable execution, + /// exhausted capacity and mapped backend errors. `DryRun` starts no runtime. + pub async fn invoke( + &self, + function_ref: &str, + qualifier: Option<&str>, + payload: &[u8], + invocation_type: InvokeKind, + ) -> Result { + self.invoke_with_logs(function_ref, qualifier, payload, invocation_type, false) + .await + } + + /// Invoke through shared admission with an optional synchronous log tail. + /// + /// Event and `DryRun` calls never capture response logs. The HTTP adapter passes + /// only its already-validated `LogType` choice; internal adapters default to None. + /// + /// # Errors + /// Returns the same validation, admission and backend errors as `invoke`. + /// + /// ```no_run + /// # async fn call(provider: &rustack_lambda_core::provider::RustackLambda) { + /// use rustack_lambda_core::provider::InvokeKind; + /// let result = provider.invoke_with_logs("worker", None, b"{}", InvokeKind::RequestResponse, true).await; + /// # } + /// ``` + pub async fn invoke_with_logs( + &self, + function_ref: &str, + qualifier: Option<&str>, + payload: &[u8], + invocation_type: InvokeKind, + capture_logs: bool, + ) -> Result { + validate_invoke_payload( + payload, + if invocation_type == InvokeKind::Event { + 1024 * 1024 + } else { + MAX_SYNC_PAYLOAD + }, + )?; + let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; + resolve_version(&record, qualifier.as_deref())?; + if invocation_type == InvokeKind::DryRun { + return Ok(InvokeOutcome::DryRun); + } + let handle = self + .admit_invocation( + &record, + qualifier.as_deref(), + payload, + invocation_type == InvokeKind::Event, + None, + capture_logs && invocation_type == InvokeKind::RequestResponse, + ) + .await?; + if invocation_type == InvokeKind::Event { + let request_id = handle.id().to_string(); + handle.detach(); + return Ok(InvokeOutcome::Async { request_id }); + } + handle + .wait() + .await + .map(InvokeOutcome::Sync) + .map_err(LambdaServiceError::from) + } + + /// Admit a cancellable synchronous invocation for an internal service adapter. + /// + /// Shares version resolution, validation and quotas with ordinary HTTP Invoke. + /// The optional absolute deadline bounds capture, queueing and cold start. + /// + /// # Errors + /// Rejects invalid input, missing resources, capture timeout, unavailable backend + /// or exhausted admission/execution capacity. Runtime outcomes use the handle. + /// + /// ```no_run + /// # async fn invoke(provider: &rustack_lambda_core::provider::RustackLambda) + /// # -> Result<(), Box> { + /// let handle = provider.start_invocation("worker", None, b"{}", None).await?; + /// let result = handle.wait().await?; + /// # Ok(()) } + /// ``` + pub async fn start_invocation( + &self, + function_ref: &str, + qualifier: Option<&str>, + payload: &[u8], + deadline: Option, + ) -> Result { + validate_invoke_payload(payload, MAX_SYNC_PAYLOAD)?; + let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; + self.admit_invocation( + &record, + qualifier.as_deref(), + payload, + false, + deadline, + false, + ) + .await + } - /// Invoke a function. + /// Capture and queue an invocation, returning ownership before admission waits. /// - /// Resolves the target version, validates the payload size, then routes - /// to the configured [`Executor`]. `DryRun` short-circuits before the - /// executor is touched. `Event` returns immediately with a synthetic - /// request id only after a slot in the bounded, supervised work set is acquired. - pub async fn invoke( + /// An error means no call was queued. After success, retain the submission or + /// its handle through cleanup even if `admitted()` fails or the caller stops. + /// External cancellation is checked across capture, queueing and execution. + /// + /// # Errors + /// Rejects invalid input/scope/revision, capture failure, cancellation or full queue. + /// + /// ```no_run + /// # async fn begin(provider:&rustack_lambda_core::provider::RustackLambda,signal:tokio::sync::watch::Receiver) -> Result<(),Box> { + /// let mut submission=provider.start_invocation_owned("worker",None,b"{}",None,signal).await?; + /// let admitted=submission.admitted().await; + /// let handle=submission.into_handle(); + /// # Ok(()) } + /// ``` + pub async fn start_invocation_owned( &self, function_ref: &str, qualifier: Option<&str>, payload: &[u8], - invocation_type: InvokeKind, - ) -> Result { - // Validate synchronous payload size (Appendix C: 6 MB). - let payload_limit = if invocation_type == InvokeKind::Event { - 1024 * 1024 - } else { - MAX_SYNC_PAYLOAD - }; - if payload.len() > payload_limit { - let payload_len = payload.len(); - return Err(LambdaServiceError::RequestTooLarge { - message: format!( - "Request payload size {payload_len} exceeds the invoke limit of \ - {payload_limit} bytes", - ), - }); - } + deadline: Option, + cancellation: watch::Receiver, + ) -> Result { + validate_invoke_payload(payload, MAX_SYNC_PAYLOAD)?; + check_external_cancellation(Some(&cancellation))?; + let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; + self.prepare_invocation( + &record, + qualifier.as_deref(), + payload, + InvocationOptions { + asynchronous: false, + deadline, + capture_logs: false, + cancellation: Some(cancellation), + }, + ) + .await + } - let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let qualifier = qualifier.or(ref_qualifier.as_deref()); - if let Some(q) = qualifier { - crate::resolver::Qualifier::parse(q)?; + fn resolve_invocation( + &self, + reference: &str, + qualifier: Option<&str>, + ) -> Result<(FunctionRecord, Option), LambdaServiceError> { + if reference + .split_once(':') + .is_some_and(|(account, resource)| { + account.len() == 12 + && account.bytes().all(|b| b.is_ascii_digit()) + && resource.starts_with("function:") + && account != self.config.account_id + }) + { + return Err(invalid_parameter( + "Invocation partial ARN does not match this Lambda scope", + )); } - - // Validate function exists and qualifier resolves. - let record = self.get_record(&name)?; - let version = resolve_version(&record, qualifier)?; - - if invocation_type == InvokeKind::DryRun { - return Ok(InvokeOutcome::DryRun); + let (name, embedded) = resolve_function_ref(reference)?; + if qualifier + .zip(embedded.as_deref()) + .is_some_and(|(explicit, embedded)| explicit != embedded) + { + return Err(invalid_parameter("Conflicting invocation qualifiers")); } - - if self.config.executor == ExecutorBackend::Docker { - return Err(LambdaServiceError::Internal { - message: "Docker execution is not implemented in this build".into(), - }); + let qualifier = qualifier.map(str::to_owned).or(embedded.clone()); + if let Some(qualifier) = &qualifier { + crate::resolver::Qualifier::parse(qualifier)?; } - self.store.validate_root().await?; - if let Some(path) = &version.code_path { - self.store.validate_artifact(path).await?; + let record = self.get_record(&name)?; + if reference.starts_with("arn:") { + let expected = embedded.as_ref().map_or_else( + || record.arn.clone(), + |qualifier| format!("{}:{qualifier}", record.arn), + ); + if reference != expected { + return Err(invalid_parameter( + "Invocation ARN does not match this Lambda scope", + )); + } } - let req = self.build_invoke_request(&record, version, payload); + Ok((record, qualifier)) + } - let (response, handle) = self - .work - .submit( - Arc::clone(&self.executor), - req, - record.reserved_concurrent_executions, - invocation_type == InvokeKind::Event, + async fn admit_invocation( + &self, + record: &FunctionRecord, + qualifier: Option<&str>, + payload: &[u8], + asynchronous: bool, + deadline: Option, + capture_logs: bool, + ) -> Result { + let mut submission = self + .prepare_invocation( + record, + qualifier, + payload, + InvocationOptions { + asynchronous, + deadline, + capture_logs, + cancellation: None, + }, ) .await?; - if invocation_type == InvokeKind::Event { - return Ok(InvokeOutcome::Async { - request_id: uuid::Uuid::new_v4().to_string(), - }); - } - let _cancel = crate::work::CancelOnDrop(handle); - let response = response - .await - .map_err(|error| LambdaServiceError::Internal { - message: format!("Lambda worker cancelled: {error}"), - })??; - Ok(InvokeOutcome::Sync(response)) + submission.admitted().await?; + Ok(submission.into_handle()) + } + + async fn prepare_invocation( + &self, + record: &FunctionRecord, + qualifier: Option<&str>, + payload: &[u8], + options: InvocationOptions, + ) -> Result { + let InvocationOptions { + asynchronous, + deadline, + capture_logs, + cancellation, + } = options; + check_external_cancellation(cancellation.as_ref())?; + let version = resolve_version(record, qualifier)?; + let capture = self.build_invoke_request(record, version, qualifier, payload); + let mut request = if let Some(deadline) = deadline { + timeout_at(deadline, capture).await.map_err(|_| { + LambdaServiceError::from(crate::executor::ExecutorError::DeadlineExceeded) + })?? + } else { + capture.await? + }; + request.capture_logs = capture_logs; + check_external_cancellation(cancellation.as_ref())?; + self.work.submit_owned( + Arc::clone(&self.executor), + request, + record.reserved_concurrent_executions, + asynchronous, + deadline, + cancellation, + ) } /// Build the executor request from a resolved function/version record. - fn build_invoke_request( + async fn build_invoke_request( &self, record: &FunctionRecord, version: &VersionRecord, + invoked_qualifier: Option<&str>, payload: &[u8], - ) -> InvokeRequest { - let function_arn = if version.version == "$LATEST" { - record.arn.clone() - } else { - function_version_arn( - &self.config.default_region, - &self.config.account_id, - &record.name, - &version.version, - ) + ) -> Result { + let architecture = validate_execution_settings(version)?; + if version.deployment.state() != "Active" { + return Err(LambdaServiceError::ResourceNotReady { + message: "Function revision is not Active".into(), + }); + } + let artifact = match version.package_type.as_str() { + "Zip" => { + let path = version + .code_path + .as_ref() + .ok_or_else(|| invalid_parameter("ZIP code has not been staged"))?; + let artifact = self.store.capture_artifact(path).await?; + if artifact.sha256() != version.code_sha256 + || artifact.zip_bytes().len() as u64 != version.code_size + { + return Err(invalid_parameter( + "Captured ZIP metadata does not match the revision", + )); + } + DeploymentArtifact::Zip(artifact) + } + "Image" => { + let image = version.image.resolved().ok_or_else(|| { + LambdaServiceError::ResourceNotReady { + message: "Image deployment has not been resolved".into(), + } + })?; + if image.metadata().architecture != architecture + || version.image_uri.as_deref() + != Some(image.metadata().requested_reference.as_str()) + || version.runtime.is_some() + || version.handler.is_some() + { + return Err(invalid_parameter( + "Image metadata does not match the captured revision", + )); + } + image + .metadata() + .execution + .plan(version.image_config.as_ref()) + .map_err(|_| { + invalid_parameter("Invalid effective image entry configuration") + })?; + let lease = version + .image + .pinned() + .filter(|image| image.is_live()) + .ok_or_else(|| LambdaServiceError::ResourceNotReady { + message: "Image availability has not been acquired".into(), + })?; + DeploymentArtifact::Image(lease.clone()) + } + _ => return Err(invalid_parameter("Unsupported deployment package type")), }; + let function_arn = invoked_qualifier.map_or_else( + || record.arn.clone(), + |qualifier| format!("{}:{qualifier}", record.arn), + ); let mut env = version.environment.clone(); // Always make the rustack region/account visible inside the runtime so // SDK calls back to the same rustack instance work without extra setup. @@ -982,23 +1713,27 @@ impl RustackLambda { .or_insert_with(|| self.config.default_region.clone()); env.entry("AWS_DEFAULT_REGION".to_owned()) .or_insert_with(|| self.config.default_region.clone()); - InvokeRequest { + let revision = ExecutionRevision::builder() + .resource_id(record.resource_id) + .revision_id(version.revision_id.clone()) + .image_config(version.image_config.clone()) + .ephemeral_storage_mb(version.ephemeral_storage_size) + .function_name(record.name.clone()) + .qualifier(version.version.clone()) + .runtime(version.runtime.clone()) + .handler(version.handler.clone()) + .architecture(architecture) + .artifact(artifact) + .environment(env) + .timeout(Duration::from_secs(u64::from(version.timeout))) + .memory_mb(version.memory_size) + .backend(self.config.executor) + .build(); + Ok(InvokeRequest::new( + Arc::new(revision), function_arn, - function_name: record.name.clone(), - qualifier: version.version.clone(), - runtime: version.runtime.clone(), - handler: version.handler.clone(), - architectures: version.architectures.clone(), - package_type: PackageType::from_wire(&version.package_type), - code_root: version.code_path.clone(), - code_zip: version.zip_bytes.clone(), - image_uri: version.image_uri.clone(), - environment: env, - timeout: Duration::from_secs(u64::from(version.timeout)), - memory_mb: version.memory_size, - payload: Bytes::copy_from_slice(payload), - capture_logs: false, - } + Bytes::copy_from_slice(payload), + )) } // --------------------------------------------------------------- @@ -1006,6 +1741,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Publish a version from `$LATEST`. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions, stale revision/hash preconditions, + /// unsuccessful deployments, unavailable image leases or invalid stored publication settings. pub fn publish_version( &self, function_ref: &str, @@ -1013,7 +1753,31 @@ impl RustackLambda { ) -> Result { let (name, _) = resolve_function_ref(function_ref)?; - let config = self.store.update(&name, |record| { + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + if record.latest.deployment != DeploymentState::Ready { + return Err(LambdaServiceError::ResourceConflict { + message: "Function deployment must be successful before publication".into(), + }); + } + if record.latest.package_type == "Image" + && record + .latest + .image + .pinned() + .is_none_or(|image| !image.is_live()) + { + return Err(LambdaServiceError::ResourceNotReady { + message: "Image availability must be acquired before publication".into(), + }); + } + if input + .code_sha256 + .as_ref() + .is_some_and(|hash| hash != &record.latest.code_sha256) + { + return Err(LambdaServiceError::PreconditionFailed); + } let version_num = record.next_version; record.next_version += 1; @@ -1027,7 +1791,7 @@ impl RustackLambda { let config = self.build_function_configuration(record, &published); record.versions.insert(version_num, published); - config + Ok(config) })?; info!(function_name = %name, "published Lambda function version"); @@ -1035,6 +1799,10 @@ impl RustackLambda { } /// List versions of a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_versions_by_function( &self, function_ref: &str, @@ -1076,6 +1844,11 @@ impl RustackLambda { } /// Create an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions, duplicate aliases, invalid target versions + /// or targets that do not exist; invalid resulting store records also reject. pub fn create_alias( &self, function_ref: &str, @@ -1145,6 +1918,10 @@ impl RustackLambda { } /// Get an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions and missing aliases. pub fn get_alias( &self, function_ref: &str, @@ -1188,6 +1965,11 @@ impl RustackLambda { } /// Update an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions/aliases, invalid target versions + /// or targets that do not exist; invalid resulting store records also reject. pub fn update_alias( &self, function_ref: &str, @@ -1263,6 +2045,10 @@ impl RustackLambda { } /// Delete an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions and missing aliases. pub fn delete_alias( &self, function_ref: &str, @@ -1284,6 +2070,10 @@ impl RustackLambda { } /// List aliases for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_aliases( &self, function_ref: &str, @@ -1348,6 +2138,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Add a permission to a function's resource policy. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, missing required permission + /// fields or duplicate statement IDs; store validation failures also propagate. pub fn add_permission( &self, function_ref: &str, @@ -1424,6 +2219,10 @@ impl RustackLambda { } /// Remove a permission from a function's resource policy. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent statement IDs. pub fn remove_permission( &self, function_ref: &str, @@ -1451,6 +2250,10 @@ impl RustackLambda { } /// Get the resource policy for a function. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and empty resource policies. pub fn get_policy( &self, function_ref: &str, @@ -1496,6 +2299,11 @@ impl RustackLambda { } /// Tag a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references, missing functions, a merged tag count above50, + /// or resulting records that fail store validation. pub fn tag_resource( &self, resource_arn: &str, @@ -1524,6 +2332,10 @@ impl RustackLambda { } /// Remove tags from a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references, missing functions and store validation failures. pub fn untag_resource( &self, resource_arn: &str, @@ -1541,6 +2353,10 @@ impl RustackLambda { } /// List tags for a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references and missing functions. pub fn list_tags(&self, resource_arn: &str) -> Result { let name = Self::extract_function_name_from_arn(resource_arn)?; let record = self.get_record(&name)?; @@ -1580,6 +2396,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Create a function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, duplicate URL configurations + /// or resulting records that fail store validation. pub fn create_function_url_config( &self, function_ref: &str, @@ -1647,6 +2468,10 @@ impl RustackLambda { } /// Get function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent URL configurations. pub fn get_function_url_config( &self, function_ref: &str, @@ -1682,6 +2507,11 @@ impl RustackLambda { } /// Update function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, absent URL configurations + /// or resulting records that fail store validation. pub fn update_function_url_config( &self, function_ref: &str, @@ -1735,6 +2565,10 @@ impl RustackLambda { } /// Delete function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent URL configurations. pub fn delete_function_url_config( &self, function_ref: &str, @@ -1761,6 +2595,10 @@ impl RustackLambda { } /// List function URL configurations. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_function_url_configs( &self, function_ref: &str, @@ -1795,6 +2633,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Publish a new layer version. + /// + /// # Errors + /// + /// Rejects empty/oversized layer names, oversized license text, invalid inline base64, + /// or a layer version that disappears before its assigned metadata is updated. pub fn publish_layer_version( &self, layer_name: &str, @@ -1894,6 +2737,10 @@ impl RustackLambda { } /// Get a specific layer version. + /// + /// # Errors + /// + /// Returns `InvalidParameter` when the requested layer version does not exist. pub fn get_layer_version( &self, layer_name: &str, @@ -1910,6 +2757,10 @@ impl RustackLambda { } /// Get a layer version by its full ARN. + /// + /// # Errors + /// + /// Rejects malformed layer-version ARNs and missing layer versions. pub fn get_layer_version_by_arn( &self, arn: &str, @@ -1919,6 +2770,10 @@ impl RustackLambda { } /// List versions of a layer. + /// + /// # Errors + /// + /// This implementation returns no errors; an unknown layer produces an empty page. pub fn list_layer_versions( &self, layer_name: &str, @@ -2036,6 +2891,10 @@ impl RustackLambda { } /// Delete a layer version. + /// + /// # Errors + /// + /// This implementation returns no errors; deleting an absent version succeeds. pub fn delete_layer_version( &self, layer_name: &str, @@ -2048,6 +2907,10 @@ impl RustackLambda { } /// Add a permission to a layer version's resource policy. + /// + /// # Errors + /// + /// Rejects missing required fields, duplicate statement IDs or missing layer versions. pub fn add_layer_version_permission( &self, layer_name: &str, @@ -2125,6 +2988,10 @@ impl RustackLambda { } /// Get the resource policy for a layer version. + /// + /// # Errors + /// + /// Rejects missing layer versions and empty layer resource policies. pub fn get_layer_version_policy( &self, layer_name: &str, @@ -2171,6 +3038,10 @@ impl RustackLambda { } /// Remove a permission from a layer version's resource policy. + /// + /// # Errors + /// + /// Rejects missing layer versions and absent statement IDs. pub fn remove_layer_version_permission( &self, layer_name: &str, @@ -2474,6 +3345,10 @@ impl RustackLambda { // ----------------------------------------------------------------- /// Put (set) reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references, missing functions and store validation failures. pub fn put_function_concurrency( &self, function_ref: &str, @@ -2492,6 +3367,10 @@ impl RustackLambda { } /// Get reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn get_function_concurrency( &self, function_ref: &str, @@ -2504,6 +3383,10 @@ impl RustackLambda { } /// Delete reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references, missing functions and store validation failures. pub fn delete_function_concurrency( &self, function_ref: &str, @@ -2521,6 +3404,10 @@ impl RustackLambda { // ----------------------------------------------------------------- /// Put (create/replace) an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and store validation failures. pub fn put_function_event_invoke_config( &self, function_ref: &str, @@ -2555,6 +3442,11 @@ impl RustackLambda { } /// Get an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent event invoke + /// configurations. pub fn get_function_event_invoke_config( &self, function_ref: &str, @@ -2578,6 +3470,11 @@ impl RustackLambda { } /// Update (merge) an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions or resulting records + /// that fail store validation. An absent configuration is created before merging. pub fn update_function_event_invoke_config( &self, function_ref: &str, @@ -2623,6 +3520,11 @@ impl RustackLambda { } /// Delete an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and store validation failures. + /// Deleting an absent event invoke configuration succeeds. pub fn delete_function_event_invoke_config( &self, function_ref: &str, @@ -2640,6 +3542,10 @@ impl RustackLambda { } /// List all event invoke configs for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_function_event_invoke_configs( &self, function_ref: &str, @@ -2684,73 +3590,108 @@ impl RustackLambda { /// S3 packages are downloaded through the configured [`S3CodeFetcher`] and /// stored via the same path as inline `ZipFile`, so downstream validation, /// checksum computation, and executor behavior are identical. - async fn process_code( + fn validate_docker_zip_settings( + &self, + runtime: Option<&str>, + handler: Option<&str>, + architecture: Architecture, + ) -> Result<(), LambdaServiceError> { + if self.config.executor != ExecutorBackend::Docker { + return Ok(()); + } + validate_provided_settings(runtime, handler)?; + self.executor + .zip_available(architecture) + .map_err(|error| match error { + ExecutorError::Unsupported(_) => { + invalid_parameter("Docker ZIP source or architecture is unavailable") + } + other => other.into(), + }) + } + + async fn validate_docker_zip_candidate( &self, + code: &ProcessedCode<'_>, + architecture: Architecture, + ) -> Result<(), LambdaServiceError> { + if self.config.executor != ExecutorBackend::Docker { + return Ok(()); + } + let ProcessedCode::Zip(staged) = code else { + return Ok(()); + }; + #[cfg(unix)] + { + DockerCodeArchive::prepare( + staged.artifact().clone(), + architecture, + Instant::now() + CODE_FETCH_TIMEOUT, + ) + .await + .map(drop) + .map_err(|error| match error { + DockerError::InvalidCode | DockerError::TooLarge | DockerError::Image(_) => { + LambdaServiceError::InvalidZipFile { + message: "Unsupported provided.al2023 ZIP artifact".into(), + } + } + DockerError::Busy => LambdaServiceError::TooManyRequests, + DockerError::Deadline => LambdaServiceError::ResourceNotReady { + message: "Docker ZIP validation deadline exceeded".into(), + }, + _ => LambdaServiceError::DockerNotAvailable, + }) + } + #[cfg(not(unix))] + { + let _ = (staged, architecture); + Err(LambdaServiceError::DockerNotAvailable) + } + } + + async fn process_code<'a>( + &'a self, function_name: &str, version: &str, source: CodeSource<'_>, - ) -> Result< - ( - String, - u64, - Option, - Option, - Option, - ), - LambdaServiceError, - > { - if let Some(b64) = source.zip_file_b64 { + ) -> Result, LambdaServiceError> { + let zip_bytes = if let Some(b64) = source.zip_file_b64 { use base64::Engine; - let zip_bytes = base64::engine::general_purpose::STANDARD - .decode(b64) - .map_err(|e| LambdaServiceError::InvalidZipFile { - message: format!("Invalid base64 encoding: {e}"), - })?; - // Fail fast on oversized packages before any storage or - // extraction happens. - check_code_size(zip_bytes.len())?; - - let (code_path, sha256, size) = self - .store - .store_zip_code(function_name, version, &zip_bytes) - .await?; - - Ok(( - sha256, - size, - Some(Bytes::from(zip_bytes)), - Some(code_path), - None, - )) + Bytes::from( + base64::engine::general_purpose::STANDARD + .decode(b64) + .map_err(|error| LambdaServiceError::InvalidZipFile { + message: format!("Invalid base64 encoding: {error}"), + })?, + ) } else if let Some(bucket) = source.s3_bucket { - let key = source - .s3_key - .ok_or_else(|| LambdaServiceError::InvalidParameter { - message: "Code.S3Key is required when Code.S3Bucket is provided".to_owned(), - })?; - let zip_bytes = self - .code_fetcher - .fetch_code(bucket, key, source.s3_object_version) - .await - .map_err(map_s3_code_fetch_error)?; - // Fail fast on oversized packages before any storage or - // extraction happens. - check_code_size(zip_bytes.len())?; - - let (code_path, sha256, size) = self - .store - .store_zip_code(function_name, version, &zip_bytes) - .await?; - - Ok((sha256, size, Some(zip_bytes), Some(code_path), None)) - } else if let Some(uri) = source.image_uri { - let sha256 = compute_sha256(uri.as_bytes()); - Ok((sha256, 0, None, None, Some(uri.to_owned()))) + let key = source.s3_key.ok_or_else(|| { + invalid_parameter("Code.S3Key is required when Code.S3Bucket is provided") + })?; + tokio::time::timeout( + CODE_FETCH_TIMEOUT, + self.code_fetcher + .fetch_code(bucket, key, source.s3_object_version), + ) + .await + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "Code source fetch deadline exceeded".into(), + })? + .map_err(map_s3_code_fetch_error)? + } else if let Some(reference) = source.image_uri { + return reference + .parse() + .map(ProcessedCode::Image) + .map_err(|_| invalid_parameter("Invalid ImageUri")); } else { - // No code provided - use empty hash. - let sha256 = compute_sha256(b""); - Ok((sha256, 0, None, None, None)) - } + return Err(invalid_parameter("Deployment code is required")); + }; + check_code_size(zip_bytes.len())?; + self.store + .stage_zip_code(function_name, version, &zip_bytes) + .await + .map(ProcessedCode::Zip) } /// Build a `FunctionConfiguration` from internal records. @@ -2819,6 +3760,7 @@ impl RustackLambda { optimization_status: Some("Off".to_owned()), }); + let state_failure = version.deployment.creation_failure(); FunctionConfiguration { function_name: Some(record.name.clone()), function_arn: Some(arn), @@ -2835,7 +3777,7 @@ impl RustackLambda { timeout: Some(version.timeout), memory_size: Some(version.memory_size), last_modified: Some(version.last_modified.clone()), - code_sha256: Some(version.code_sha256.clone()), + code_sha256: (!version.code_sha256.is_empty()).then(|| version.code_sha256.clone()), version: Some(version.version.clone()), environment: env_response, vpc_config: vpc_response, @@ -2843,9 +3785,9 @@ impl RustackLambda { tracing_config: tracing_response, revision_id: Some(version.revision_id.clone()), layers, - state: Some(version.state.clone()), - state_reason: None, - state_reason_code: None, + state: Some(version.deployment.state().into()), + state_reason: state_failure.map(|failure| failure.reason().into()), + state_reason_code: state_failure.map(|failure| failure.code().into()), package_type: Some(version.package_type.clone()), architectures: Some(version.architectures.clone()), ephemeral_storage: Some(EphemeralStorage { @@ -2854,9 +3796,15 @@ impl RustackLambda { logging_config: version.logging_config.clone(), snap_start: snap_start_response, image_config_response, - last_update_status: Some("Successful".to_owned()), - last_update_status_reason: None, - last_update_status_reason_code: None, + last_update_status: Some(version.deployment.update_status().into()), + last_update_status_reason: version + .deployment + .failure() + .map(|failure| failure.reason().into()), + last_update_status_reason_code: version + .deployment + .failure() + .map(|failure| failure.code().into()), } } @@ -2872,8 +3820,144 @@ impl RustackLambda { } } -/// Get current time in ISO 8601 format matching AWS Lambda conventions. -fn now_iso8601() -> String { +/// Reject publication before asynchronous image preparation can begin. +fn validate_code_update( + before: &FunctionRecord, + input: &UpdateFunctionCodeInput, + code_source: CodeSource<'_>, +) -> Result { + // Validate that some code source is provided. + if code_source.zip_file_b64.is_none() + && code_source.image_uri.is_none() + && code_source.s3_bucket.is_none() + { + return Err(LambdaServiceError::InvalidParameter { + message: "Provide at least one of ZipFile, S3Bucket, or ImageUri.".to_owned(), + }); + } + + check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; + before.latest.deployment.ensure_mutable()?; + validate_image_publication(&before.latest.package_type, input.publish.unwrap_or(false))?; + if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) + || (before.latest.package_type == "Image" && code_source.image_uri.is_none()) + { + return Err(invalid_parameter("Deployment package type cannot change")); + } + if input.architectures.as_ref().is_some_and(|architectures| { + architectures.len() != 1 + || architectures + .iter() + .any(|architecture| architecture != "arm64" && architecture != "x86_64") + }) { + return Err(invalid_parameter( + "Exactly one supported architecture is required", + )); + } + match input + .architectures + .as_ref() + .and_then(|values| values.first()) + { + Some(value) => value + .parse() + .map_err(|_| invalid_parameter("Invalid architecture")), + None => validate_execution_settings(&before.latest), + } +} + +fn mutable_function_name(reference: &str) -> Result { + let (name, qualifier) = resolve_function_ref(reference)?; + if qualifier.as_deref().is_some_and(|value| value != "$LATEST") { + return Err(invalid_parameter( + "Only the unpublished function configuration can be updated", + )); + } + Ok(name) +} + +fn check_staged_revision( + record: &FunctionRecord, + before: &FunctionRecord, +) -> Result<(), LambdaServiceError> { + if record.resource_id != before.resource_id + || record.latest.revision_id != before.latest.revision_id + { + return Err(LambdaServiceError::ResourceConflict { + message: "Function changed while deployment was staged".into(), + }); + } + Ok(()) +} + +fn validate_image_publication(package_type: &str, publish: bool) -> Result<(), LambdaServiceError> { + if package_type == "Image" && publish { + return Err(invalid_parameter( + "Image deployment requires a separate PublishVersion after successful preparation", + )); + } + Ok(()) +} + +fn validate_configuration_update( + input: &UpdateFunctionConfigurationInput, +) -> Result<(), LambdaServiceError> { + // Validate handler length (Appendix C: max 128 chars). + if let Some(ref handler) = input.handler { + if handler.len() > 128 { + return Err(LambdaServiceError::InvalidParameter { + message: "Handler must be at most 128 characters".to_owned(), + }); + } + } + // Validate description length (Appendix C: max 256 chars). + if let Some(ref desc) = input.description { + if desc.len() > 256 { + return Err(LambdaServiceError::InvalidParameter { + message: "Description must be at most 256 characters".to_owned(), + }); + } + } + // Validate timeout (Appendix C: 1-900 seconds). + if let Some(timeout) = input.timeout { + if !(1..=900).contains(&timeout) { + return Err(LambdaServiceError::InvalidParameter { + message: "Timeout must be between 1 and 900 seconds".to_owned(), + }); + } + } + // Validate memory size (Appendix C: 128-10240 MB). + if let Some(memory) = input.memory_size { + if !(128..=10_240).contains(&memory) { + return Err(LambdaServiceError::InvalidParameter { + message: "Memory size must be between 128 and 10240 MB".to_owned(), + }); + } + } + // Validate ephemeral storage (Appendix C: 512-10240 MB). + if let Some(ref ephemeral) = input.ephemeral_storage { + if !(512..=10_240).contains(&ephemeral.size) { + return Err(LambdaServiceError::InvalidParameter { + message: "Ephemeral storage must be between 512 and 10240 MB".to_owned(), + }); + } + } + // Validate environment variables total size (Appendix C: 4 KB). + if let Some(ref env) = input.environment { + if let Some(ref vars) = env.variables { + let total_size: usize = vars.iter().map(|(k, v)| k.len() + v.len()).sum(); + if total_size > 4096 { + return Err(LambdaServiceError::InvalidParameter { + message: "Environment variables total size exceeds 4 KB limit".to_owned(), + }); + } + } + } + Ok(()) +} + +/// Get current time as an ISO 8601 string in AWS Lambda format. +pub(crate) fn now_iso8601() -> String { chrono::Utc::now() .format("%Y-%m-%dT%H:%M:%S%.3f+0000") .to_string() @@ -2967,7 +4051,7 @@ fn validate_code_source(source: &CodeSource<'_>) -> Result<(), LambdaServiceErro /// storage or extraction happens, so oversized packages fail fast instead /// of materializing onto disk. fn check_code_size(size_bytes: usize) -> Result<(), LambdaServiceError> { - if size_bytes as u64 > MAX_ZIP_SIZE { + if size_bytes > MAX_ZIP_SIZE { return Err(LambdaServiceError::InvalidParameter { message: format!("Unzipped size must be smaller than {MAX_ZIP_SIZE} bytes"), }); @@ -2989,6 +4073,15 @@ fn validate_s3_field(name: &str, value: &str, max_bytes: usize) -> Result<(), La } /// Build an `InvalidParameter` service error. +fn validate_invoke_payload(payload: &[u8], limit: usize) -> Result<(), LambdaServiceError> { + if payload.len() > limit { + return Err(LambdaServiceError::RequestTooLarge { + message: format!("Invocation payload exceeds {limit} bytes"), + }); + } + Ok(()) +} + fn invalid_parameter(message: &str) -> LambdaServiceError { LambdaServiceError::InvalidParameter { message: message.to_owned(), @@ -3024,6 +4117,134 @@ fn map_s3_code_fetch_error(err: S3CodeFetchError) -> LambdaServiceError { mod tests { use super::*; + #[derive(Debug)] + struct DelayedFetcher { + started: tokio::sync::Notify, + release: tokio::sync::Notify, + } + + #[async_trait::async_trait] + impl S3CodeFetcher for DelayedFetcher { + async fn fetch_code( + &self, + _bucket: &str, + _key: &str, + _version: Option<&str>, + ) -> Result { + self.started.notify_one(); + self.release.notified().await; + Ok(Bytes::from(crate::test_zip::minimal_zip( + b"conflicting-update", + ))) + } + } + + #[tokio::test(start_paused = true)] + async fn test_should_bound_a_stalled_code_fetch_without_creating_an_artifact() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let fetcher = Arc::new(DelayedFetcher { + started: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher); + let started = Instant::now(); + assert!(matches!( + provider + .create_function(s3_code_input("fetch-timeout")) + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + assert!(started.elapsed() >= CODE_FETCH_TIMEOUT); + assert!(provider.store.is_empty()); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_staged_code_after_a_real_revision_compare_and_swap_conflict() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let fetcher = Arc::new(DelayedFetcher { + started: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let provider = Arc::new( + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher.clone()), + ); + provider + .create_function(sample_create_input("conflict")) + .await?; + let original = provider.get_record("conflict")?.latest; + let updating = Arc::clone(&provider); + let task = tokio::spawn(async move { + updating + .update_function_code( + "conflict", + UpdateFunctionCodeInput { + s3_bucket: Some("code-bucket".into()), + s3_key: Some("new.zip".into()), + ..UpdateFunctionCodeInput::default() + }, + ) + .await + }); + let _cancel = crate::work::CancelOnDrop(task.abort_handle()); + tokio::time::timeout(Duration::from_secs(5), fetcher.started.notified()).await?; + provider.update_function_configuration( + "conflict", + &UpdateFunctionConfigurationInput { + description: Some("newer configuration".into()), + ..UpdateFunctionConfigurationInput::default() + }, + )?; + fetcher.release.notify_one(); + let result = tokio::time::timeout(Duration::from_secs(5), task).await??; + assert!(matches!( + result, + Err(LambdaServiceError::ResourceConflict { .. }) + )); + let current = provider.get_record("conflict")?.latest; + assert_eq!(current.code_sha256, original.code_sha256); + assert_eq!(current.code_path, original.code_path); + assert_eq!(current.description, "newer configuration"); + let mut entries = tokio::fs::read_dir(root.path()).await?; + assert!(entries.next_entry().await?.is_some()); + assert!(entries.next_entry().await?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn test_should_validate_partial_invocation_account_before_local_resolution() { + let provider = test_provider(); + provider + .create_function(sample_create_input("scoped")) + .await + .unwrap(); + assert!( + provider + .resolve_invocation("000000000000:function:scoped", None) + .is_ok() + ); + assert!(matches!( + provider.resolve_invocation("999999999999:function:scoped", None), + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(matches!( + provider.resolve_invocation("999999999999:function:scoped:1", None), + Err(LambdaServiceError::InvalidParameter { .. }) + )); + } + fn test_provider() -> RustackLambda { let tmp = tempfile::tempdir().unwrap(); let store = FunctionStore::new(tmp.path()); @@ -3048,6 +4269,443 @@ mod tests { } } + async fn artifact_count(path: &std::path::Path) -> usize { + let mut entries = tokio::fs::read_dir(path).await.unwrap(); + let mut count = 0usize; + while entries.next_entry().await.unwrap().is_some() { + count = count.saturating_add(1); + } + count + } + + #[derive(Debug, Default)] + struct CountingFetcher(std::sync::atomic::AtomicUsize); + #[async_trait::async_trait] + impl S3CodeFetcher for CountingFetcher { + async fn fetch_code( + &self, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result { + self.0.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Err(S3CodeFetchError::Internal(anyhow::anyhow!( + "unexpected fetch" + ))) + } + } + + #[tokio::test] + async fn test_should_validate_code_dry_run_without_staging_fetching_or_publishing() { + use base64::{Engine, engine::general_purpose::STANDARD}; + let root = tempfile::tempdir().unwrap(); + let fetcher = Arc::new(CountingFetcher::default()); + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher.clone()); + provider + .create_function(sample_create_input("dry-update")) + .await + .unwrap(); + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + let count = artifact_count(root.path()).await; + for input in [ + UpdateFunctionCodeInput { + zip_file: Some(STANDARD.encode(b"not an archive")), + dry_run: Some(true), + publish: Some(true), + ..Default::default() + }, + UpdateFunctionCodeInput { + s3_bucket: Some("bucket".into()), + s3_key: Some("missing.zip".into()), + dry_run: Some(true), + ..Default::default() + }, + ] { + let result = provider + .update_function_code("dry-update", input) + .await + .unwrap(); + assert_eq!(result.version.as_deref(), Some("$LATEST")); + } + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + assert_eq!(artifact_count(root.path()).await, count); + assert_eq!(fetcher.0.load(std::sync::atomic::Ordering::Acquire), 0); + } + + #[tokio::test] + async fn test_should_reject_invalid_or_qualified_code_dry_run_without_mutation() { + let provider = test_provider(); + provider + .create_function(sample_create_input("dry-invalid")) + .await + .unwrap(); + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + for input in [ + UpdateFunctionCodeInput { + zip_file: Some("AB==".into()), + dry_run: Some(true), + ..Default::default() + }, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + revision_id: Some("stale".into()), + ..Default::default() + }, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + architectures: Some(Vec::new()), + ..Default::default() + }, + ] { + assert!( + provider + .update_function_code("dry-invalid", input) + .await + .is_err() + ); + } + for reference in ["dry-invalid:1", "dry-invalid:prod"] { + assert!( + provider + .update_function_code( + reference, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + ..Default::default() + } + ) + .await + .is_err() + ); + assert!( + provider + .update_function_configuration( + reference, + &UpdateFunctionConfigurationInput::default() + ) + .is_err() + ); + } + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + } + + #[tokio::test] + async fn test_should_reject_invalid_settings_before_fetching_or_staging_code() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()); + let mut input = sample_create_input("invalid-env"); + input.environment = Some(rustack_lambda_model::types::Environment { + variables: Some(HashMap::from([("AWS_REGION".into(), "us-east-1".into())])), + }); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut input = sample_create_input("invalid-image-config"); + input.image_config = Some(rustack_lambda_model::types::ImageConfig::default()); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut input = s3_code_input("invalid-s3-settings"); + input.timeout = Some(901); + // The default fetcher is unavailable; InvalidParameter proves it was not called. + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(provider.store.is_empty()); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_cross_scope_arns_and_conflicting_invocation_qualifiers() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("scoped")) + .await?; + let arn = provider.get_record("scoped")?.arn; + for reference in [ + arn.replace("us-east-1", "us-west-2"), + arn.replace("000000000000", "111111111111"), + arn.replace("arn:aws:", "arn:aws-cn:"), + ] { + assert!(matches!( + provider + .invoke(&reference, None, b"{}", InvokeKind::DryRun) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + } + assert!(matches!( + provider + .invoke( + &format!("{arn}:$LATEST"), + Some("1"), + b"{}", + InvokeKind::DryRun + ) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(matches!( + provider + .invoke( + &format!("{arn}:$LATEST"), + Some("$LATEST"), + b"{}", + InvokeKind::DryRun + ) + .await?, + InvokeOutcome::DryRun + )); + Ok(()) + } + + #[tokio::test] + async fn test_should_preserve_invoked_alias_separately_from_resolved_version() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("qualified")) + .await?; + provider.publish_version("qualified", &PublishVersionInput::default())?; + provider.create_alias( + "qualified", + CreateAliasInput { + name: "prod".into(), + function_version: "1".into(), + ..CreateAliasInput::default() + }, + )?; + let record = provider.get_record("qualified")?; + for (qualifier, resolved) in [ + (None, "$LATEST"), + (Some("$LATEST"), "$LATEST"), + (Some("prod"), "1"), + (Some("1"), "1"), + ] { + let version = resolve_version(&record, qualifier)?; + let request = provider + .build_invoke_request(&record, version, qualifier, b"{}") + .await?; + let expected_arn = + qualifier.map_or_else(|| record.arn.clone(), |q| format!("{}:{q}", record.arn)); + assert_eq!(request.function_arn, expected_arn); + assert_eq!(request.revision.qualifier, resolved); + assert_eq!(request.revision.resource_id, record.resource_id); + assert_eq!(request.revision.revision_id, version.revision_id); + } + Ok(()) + } + + #[tokio::test] + async fn test_should_capture_image_overrides_and_ephemeral_storage_by_value() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("captured")) + .await?; + let record = provider.get_record("captured")?; + let mut version = record.latest.clone(); + version.ephemeral_storage_size = 2048; + version.package_type = "Image".into(); + version.runtime = None; + version.handler = None; + version.code_path = None; + version.zip_bytes = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + version.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + version.code_sha256 = resolved + .code_sha256() + .map_err(|_| invalid_parameter("Invalid unit image hash"))?; + version.code_size = resolved.metadata().size_bytes; + version.image = ImageArtifact::Pinned(crate::artifact::PinnedImage::fixture(resolved)); + version.image_config = Some(rustack_lambda_model::types::ImageConfig { + entry_point: Some(vec!["/custom/entrypoint".into()]), + command: Some(vec![]), + working_directory: Some("/function".into()), + }); + let request = provider + .build_invoke_request(&record, &version, None, b"{\"value\":1}") + .await?; + version.ephemeral_storage_size = 4096; + version.image_config = None; + assert_eq!(request.revision.ephemeral_storage_mb, 2048); + let image = request + .revision + .image_config + .as_ref() + .ok_or_else(|| invalid_parameter("Image configuration was lost"))?; + assert_eq!( + image.entry_point.as_deref(), + Some(["/custom/entrypoint".to_owned()].as_slice()) + ); + assert_eq!(image.command.as_deref(), Some([].as_slice())); + assert_eq!(image.working_directory.as_deref(), Some("/function")); + assert_eq!(request.payload, Bytes::from_static(b"{\"value\":1}")); + assert_ne!( + request.revision.ephemeral_storage_mb, + version.ephemeral_storage_size + ); + assert!(version.image_config.is_none()); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_image_create_publication_before_inserting_a_resource() { + let provider = test_provider(); + let mut input = sample_create_input("image-publish"); + input.package_type = Some("Image".into()); + input.runtime = None; + input.handler = None; + input.code = rustack_lambda_model::types::FunctionCode { + image_uri: Some("worker:v1".into()), + ..Default::default() + }; + input.publish = Some(true); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(!provider.store.contains("image-publish").unwrap()); + } + + #[tokio::test] + async fn test_should_capture_and_retain_only_live_image_availability() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("image-lease")) + .await?; + let record = provider.get_record("image-lease")?; + let mut version = record.latest.clone(); + version.package_type = "Image".into(); + version.runtime = None; + version.handler = None; + version.code_path = None; + version.zip_bytes = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + version.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + version.code_sha256 = resolved + .code_sha256() + .map_err(|_| invalid_parameter("Unit image hash"))?; + version.code_size = resolved.metadata().size_bytes; + version.image = ImageArtifact::Restored(resolved.clone()); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + let (release, released) = tokio::sync::watch::channel(()); + let (stop, stopping) = tokio::sync::watch::channel(false); + version.image = ImageArtifact::Pinned(crate::artifact::PinnedImage::new( + resolved, + version + .code_size + .try_into() + .map_err(|_| invalid_parameter("Unit image size"))?, + release, + Arc::new(std::sync::atomic::AtomicBool::new(true)), + stopping, + )); + let request = provider + .build_invoke_request(&record, &version, None, b"{}") + .await?; + let hash = version.code_sha256.clone(); + version.code_sha256 = "tampered".into(); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + version.code_sha256 = hash; + stop.send_replace(true); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + drop(version); + assert!( + !released + .has_changed() + .map_err(|_| invalid_parameter("Lease lost before request"))? + ); + assert!(matches!( + request.revision.artifact, + DeploymentArtifact::Image(_) + )); + drop(request); + assert!(released.has_changed().is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_unresolved_or_inconsistent_artifacts_before_work_admission() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("artifact-capture")) + .await?; + let record = provider.get_record("artifact-capture")?; + let mut invalid_zip = record.latest.clone(); + invalid_zip.code_sha256 = "different".into(); + assert!(matches!( + provider + .build_invoke_request(&record, &invalid_zip, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut image = record.latest.clone(); + image.package_type = "Image".into(); + image.runtime = None; + image.handler = None; + image.zip_bytes = None; + image.code_path = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + image.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + assert!(matches!( + provider + .build_invoke_request(&record, &image, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + image.image = ImageArtifact::Restored(resolved); + image.architectures = vec!["arm64".into()]; + assert!(matches!( + provider + .build_invoke_request(&record, &image, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + Ok(()) + } + #[tokio::test] async fn test_should_create_and_get_function() { let provider = test_provider(); @@ -4208,3 +5866,7 @@ mod tests { .unwrap(); } } + +#[cfg(all(test, unix))] +#[path = "provider/zip_tests.rs"] +pub(crate) mod provided_zip_tests; diff --git a/crates/rustack-lambda-core/src/provider/snapshot.rs b/crates/rustack-lambda-core/src/provider/snapshot.rs new file mode 100644 index 00000000..d60ffcc1 --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/snapshot.rs @@ -0,0 +1,644 @@ +//! Bounded Docker archive profiles and validation before candidate preparation. + +use crate::artifact::{Architecture, MAX_ZIP_BYTES as MAX_ORIGINAL_ZIP_BYTES}; + +/// Configured Docker archive allowance, independent of execution readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum DockerArchiveProfile { + /// Immutable image metadata only. + ImageOnly, + /// Original provided-runtime ZIP code and image metadata. + ProvidedZip(Architecture), +} +impl DockerArchiveProfile { + /// Compact metadata allowance, excluding original ZIP bodies. + pub const METADATA_BYTES: usize = 16 * 1024 * 1024; + /// Maximum number of functions in either Docker profile. + pub const MAX_FUNCTIONS: usize = 256; + /// Maximum aggregate latest and published revisions. + pub const MAX_REVISIONS: usize = 1024; + /// Maximum original ZIP bytes per revision. + pub const MAX_ZIP_BYTES: usize = MAX_ORIGINAL_ZIP_BYTES; + /// Aggregate original ZIP bytes, counting every saved revision. + pub const MAX_ZIP_DATA_BYTES: usize = 64 * 1024 * 1024; + + /// Complete service payload/work allowance for this configured profile. + /// + /// ``` + /// use rustack_lambda_core::provider::DockerArchiveProfile; + /// assert_eq!(DockerArchiveProfile::ImageOnly.payload_limit(), 16 * 1024 * 1024); + /// ``` + #[must_use] + pub const fn payload_limit(self) -> usize { + match self { + Self::ImageOnly => Self::METADATA_BYTES, + Self::ProvidedZip(_) => 128 * 1024 * 1024, + } + } + + /// Configured ZIP platform; this is not a health or execution grant. + /// + /// ``` + /// use rustack_lambda_core::provider::DockerArchiveProfile; + /// assert_eq!(DockerArchiveProfile::ImageOnly.zip_architecture(), None); + /// ``` + #[must_use] + pub const fn zip_architecture(self) -> Option { + match self { + Self::ImageOnly => None, + Self::ProvidedZip(architecture) => Some(architecture), + } + } +} + +pub(super) const IMAGE_METADATA_BYTES: usize = DockerArchiveProfile::METADATA_BYTES; +pub(super) const MAX_IMAGE_FUNCTIONS: usize = DockerArchiveProfile::MAX_FUNCTIONS; +pub(super) const MAX_IMAGE_REVISIONS: usize = DockerArchiveProfile::MAX_REVISIONS; + +#[cfg(unix)] +mod projection; +#[cfg(unix)] +mod zip; + +#[cfg(unix)] +mod validation { + use std::{ + fmt::{Debug, Formatter, Result as FmtResult}, + io::{Error as IoError, ErrorKind, Result as IoResult, Write}, + iter::once, + }; + + use rustack_core::json::{JsonLimits, validate}; + use serde::Serialize; + use serde_json::Value; + use tokio::{ + sync::Semaphore, + task::spawn_blocking, + time::{Instant, timeout_at}, + }; + + use super::{ + DockerArchiveProfile, IMAGE_METADATA_BYTES, MAX_IMAGE_FUNCTIONS, MAX_IMAGE_REVISIONS, + projection::Metadata, zip, + }; + use crate::{ + docker::DockerError, + error::LambdaServiceError, + provider::{LambdaSnapshot, RustackLambda}, + resolver::{FunctionName, function_arn}, + storage::{ + FunctionRecord, FunctionSnapshotRef, LayerRecord, LayerSnapshotRef, MappingSnapshotRef, + PolicyDocument, check_restore_deadline, image_error, + }, + }; + + static VALIDATORS: Semaphore = Semaphore::const_new(4); + + pub(super) fn invalid() -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: "Invalid or oversized Docker archive metadata".into(), + } + } + struct BoundedBytes(Vec, usize); + impl Debug for BoundedBytes { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("BoundedBytes") + .field("bytes", &self.0.len()) + .field("limit", &self.1) + .finish() + } + } + impl Default for BoundedBytes { + fn default() -> Self { + Self(Vec::new(), IMAGE_METADATA_BYTES) + } + } + impl Write for BoundedBytes { + fn write(&mut self, bytes: &[u8]) -> IoResult { + if bytes.len() > self.1.saturating_sub(self.0.len()) { + return Err(IoError::from(ErrorKind::FileTooLarge)); + } + self.0 + .try_reserve(bytes.len()) + .map_err(|_| IoError::from(ErrorKind::OutOfMemory))?; + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } + } + #[derive(Debug)] + struct GuardWork { + remaining: usize, + #[cfg(test)] + visited_nodes: usize, + } + impl GuardWork { + fn new() -> Self { + Self { + remaining: 65_536, + #[cfg(test)] + visited_nodes: 0, + } + } + fn charge(&mut self, units: usize) -> Result<(), LambdaServiceError> { + self.remaining = self.remaining.checked_sub(units).ok_or_else(invalid)?; + Ok(()) + } + } + fn condition(value: &Value, work: &mut GuardWork) -> Result<(), LambdaServiceError> { + let mut pending = vec![(value, 1usize)]; + while let Some((value, depth)) = pending.pop() { + work.charge(1)?; + #[cfg(test)] + { + work.visited_nodes = work.visited_nodes.checked_add(1).ok_or_else(invalid)?; + } + if depth > 64 { + return Err(invalid()); + } + let children = match value { + Value::Array(values) => values.len(), + Value::Object(values) => values.len(), + Value::String(value) if value.len() > 128 * 1024 => return Err(invalid()), + _ => 0, + }; + if children > work.remaining.saturating_sub(pending.len()) { + return Err(invalid()); + } + let next_depth = depth.checked_add(1).ok_or_else(invalid)?; + match value { + Value::Array(values) => { + pending.extend(values.iter().map(|value| (value, next_depth))); + } + Value::Object(values) => { + for (key, value) in values { + if key.len() > 1024 { + return Err(invalid()); + } + pending.push((value, next_depth)); + } + } + _ => {} + } + } + Ok(()) + } + fn policy(policy: &PolicyDocument, work: &mut GuardWork) -> Result<(), LambdaServiceError> { + if policy.statements.len() > 128 { + return Err(invalid()); + } + work.charge(policy.statements.len())?; + for statement in &policy.statements { + if let Some(value) = &statement.condition { + condition(value, work)?; + } + } + Ok(()) + } + fn function_record(record: &FunctionRecord) -> Result<(), LambdaServiceError> { + let mut work = GuardWork::new(); + work.charge(record.versions.len().checked_add(1).ok_or_else(invalid)?)?; + policy(&record.policy, &mut work)?; + for version in once(&record.latest).chain(record.versions.values()) { + if version.package_type != "Image" || version.zip_bytes.is_some() { + return Err(invalid()); + } + } + Ok(()) + } + fn managed_record(record: &FunctionRecord) -> Result<(), LambdaServiceError> { + zip::record(record)?; + let mut work = GuardWork::new(); + work.charge(record.versions.len().checked_add(1).ok_or_else(invalid)?)?; + policy(&record.policy, &mut work) + } + fn layer_record(record: &LayerRecord) -> Result<(), LambdaServiceError> { + layer_record_guarded(record, &mut GuardWork::new()) + } + fn layer_record_guarded( + record: &LayerRecord, + work: &mut GuardWork, + ) -> Result<(), LambdaServiceError> { + work.charge(record.versions.len())?; + for version in record.versions.values() { + policy(&version.policy, work)?; + } + Ok(()) + } + #[derive(Debug, Serialize)] + #[serde(rename_all = "camelCase")] + struct BorrowedSnapshot<'a> { + functions: FunctionSnapshotRef<'a>, + layers: LayerSnapshotRef<'a>, + event_source_mappings: MappingSnapshotRef<'a>, + } + pub(crate) async fn export_docker( + provider: &RustackLambda, + deadline: Instant, + profile: DockerArchiveProfile, + ) -> Result { + check_restore_deadline(deadline)?; + let permit = timeout_at(deadline, VALIDATORS.acquire()) + .await + .map_err(|_| image_error(DockerError::Deadline))? + .map_err(|_| image_error(DockerError::Stopped))?; + let stores = ( + provider.store.clone(), + provider.layer_store.clone(), + provider.esm_store.clone(), + ); + let scope = ( + provider.config.default_region.clone(), + provider.config.account_id.clone(), + ); + let mut job = spawn_blocking(move || { + let retained = (stores, permit); + check_restore_deadline(deadline)?; + retained + .0 + .0 + .check_snapshot_counts(MAX_IMAGE_FUNCTIONS, MAX_IMAGE_REVISIONS)?; + if profile.zip_architecture().is_some() { + retained.0.0.check_snapshot_zip_profile(deadline)?; + } + let guard = if profile.zip_architecture().is_some() { + managed_record + } else { + function_record + }; + let borrowed = BorrowedSnapshot { + functions: retained.0.0.borrow_snapshot(guard), + layers: retained.0.1.borrow_snapshot(layer_record), + event_source_mappings: retained.0.2.borrow_snapshot(), + }; + if profile.zip_architecture().is_some() { + let mut metadata = BoundedBytes::default(); + serde_json::to_writer(&mut metadata, &Metadata::new(&borrowed)) + .map_err(|_| invalid())?; + let limits = JsonLimits::new(IMAGE_METADATA_BYTES, 64, 65_536, 128 * 1024, 1024) + .map_err(|_| invalid())?; + validate(&metadata.0, limits).map_err(|_| invalid())?; + } + let mut bytes = BoundedBytes(Vec::new(), profile.payload_limit()); + serde_json::to_writer(&mut bytes, &borrowed).map_err(|_| invalid())?; + if profile.zip_architecture().is_none() { + let limits = JsonLimits::new(IMAGE_METADATA_BYTES, 64, 65_536, 128 * 1024, 1024) + .map_err(|_| invalid())?; + validate(&bytes.0, limits).map_err(|_| invalid())?; + } + check_restore_deadline(deadline)?; + let mut snapshot: LambdaSnapshot = + serde_json::from_slice(&bytes.0).map_err(|_| invalid())?; + input_profile(&mut snapshot, &scope.0, &scope.1, profile)?; + snapshot + .functions + .functions + .sort_by(|left, right| left.name.cmp(&right.name)); + snapshot + .layers + .layers + .sort_by(|left, right| left.name.cmp(&right.name)); + snapshot + .event_source_mappings + .mappings + .sort_by(|left, right| left.uuid.cmp(&right.uuid)); + check_restore_deadline(deadline)?; + Ok(snapshot) + }); + let result = timeout_at(deadline, &mut job) + .await + .map_err(|_| image_error(DockerError::Deadline))? + .map_err(|source| LambdaServiceError::ImageRestoreWorker { + source: source.into(), + })?; + check_restore_deadline(deadline)?; + result + } + + fn input( + snapshot: &LambdaSnapshot, + region: &str, + account: &str, + ) -> Result<(), LambdaServiceError> { + if snapshot.functions.functions.len() > MAX_IMAGE_FUNCTIONS { + return Err(invalid()); + } + let mut work = GuardWork::new(); + work.charge(snapshot.functions.functions.len())?; + work.charge(snapshot.layers.layers.len())?; + work.charge(snapshot.event_source_mappings.mappings.len())?; + let mut versions = 0usize; + for function in &snapshot.functions.functions { + FunctionName::parse(&function.name).map_err(|_| invalid())?; + if function.arn != function_arn(region, account, &function.name) { + return Err(invalid()); + } + versions = versions + .checked_add(1) + .and_then(|n| n.checked_add(function.versions.len())) + .ok_or_else(invalid)?; + if versions > MAX_IMAGE_REVISIONS || function.policy.statements.len() > 128 { + return Err(invalid()); + } + work.charge(function.versions.len().checked_add(1).ok_or_else(invalid)?)?; + for version in once(&function.latest).chain(function.versions.values()) { + if version.zip_bytes_base64.is_some() { + return Err(invalid()); + } + } + policy(&function.policy, &mut work)?; + } + for layer in &snapshot.layers.layers { + layer_record_guarded(layer, &mut work)?; + } + let mut bytes = BoundedBytes::default(); + serde_json::to_writer(&mut bytes, snapshot).map_err(|_| invalid())?; + let limits = JsonLimits::new(IMAGE_METADATA_BYTES, 64, 65_536, 128 * 1024, 1024) + .map_err(|_| invalid())?; + validate(&bytes.0, limits).map_err(|_| invalid()) + } + fn input_profile( + snapshot: &mut LambdaSnapshot, + region: &str, + account: &str, + profile: DockerArchiveProfile, + ) -> Result<(), LambdaServiceError> { + if snapshot.functions.functions.len() > MAX_IMAGE_FUNCTIONS { + return Err(invalid()); + } + let revisions = + snapshot + .functions + .functions + .iter() + .try_fold(0usize, |count, function| { + count + .checked_add(1) + .and_then(|n| n.checked_add(function.versions.len())) + .ok_or_else(invalid) + })?; + if revisions > MAX_IMAGE_REVISIONS { + return Err(invalid()); + } + let bodies = zip::detach(snapshot, profile)?; + input(snapshot, region, account)?; + zip::reattach(snapshot, bodies) + } + pub(crate) async fn validate_docker( + snapshot: LambdaSnapshot, + deadline: Instant, + region: &str, + account: &str, + profile: DockerArchiveProfile, + ) -> Result { + validate_with( + snapshot, + deadline, + (region.into(), account.into()), + move |snapshot, region, account| input_profile(snapshot, region, account, profile), + ) + .await + } + #[cfg(test)] + pub(crate) async fn validate_image( + snapshot: LambdaSnapshot, + deadline: Instant, + region: &str, + account: &str, + ) -> Result { + validate_docker( + snapshot, + deadline, + region, + account, + DockerArchiveProfile::ImageOnly, + ) + .await + } + async fn validate_with( + snapshot: LambdaSnapshot, + deadline: Instant, + scope: (String, String), + worker: F, + ) -> Result + where + F: FnOnce(&mut LambdaSnapshot, &str, &str) -> Result<(), LambdaServiceError> + + Send + + 'static, + { + check_restore_deadline(deadline)?; + let permit = timeout_at(deadline, VALIDATORS.acquire()) + .await + .map_err(|_| image_error(DockerError::Deadline))? + .map_err(|_| image_error(DockerError::Stopped))?; + // The read-only job owns input before its slot in drop order, retaining + // admission through iterative policy destruction even on worker unwind. + let mut job = spawn_blocking(move || { + let mut retained = (snapshot, permit); + let result = (|| { + check_restore_deadline(deadline)?; + worker(&mut retained.0, &scope.0, &scope.1)?; + check_restore_deadline(deadline) + })(); + match result { + Ok(()) => Ok(retained.0), + Err(error) => { + drop(retained); + Err(error) + } + } + }); + // Dropped observations detach only bounded read-only owned CPU work. + // No late result has a store or publication capability. + let result = timeout_at(deadline, &mut job) + .await + .map_err(|_| image_error(DockerError::Deadline))? + .map_err(|source| LambdaServiceError::ImageRestoreWorker { + source: source.into(), + })?; + check_restore_deadline(deadline)?; + result + } + + #[cfg(test)] + mod tests { + use std::{error::Error as _, time::Duration}; + + use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; + + use super::*; + use crate::{ + config::LambdaConfig, + deployment::image_record_fixture as record, + executor::ExecutorBackend, + provider::RustackLambda, + storage::{FunctionStore, PolicyStatement}, + }; + + fn deep_snapshot() -> LambdaSnapshot { + let store = FunctionStore::new("unused-pure-snapshot-fixture"); + store.insert(record()).unwrap(); + let mut snapshot = LambdaSnapshot { + functions: store.export_snapshot(), + ..LambdaSnapshot::default() + }; + let mut condition = Value::Null; + for _ in 0..50_000 { + condition = Value::Array(vec![condition]); + } + snapshot + .functions + .functions + .first_mut() + .unwrap() + .policy + .statements + .push(PolicyStatement { + sid: "deep".into(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:InvokeFunction".into(), + resource: "*".into(), + condition: Some(condition), + }); + snapshot + } + #[tokio::test] + async fn test_should_destroy_deep_rejected_expired_and_unpolled_policy_inputs_without_recursion() + { + let error = validate_image( + deep_snapshot(), + Instant::now() + Duration::from_secs(30), + "us-east-1", + "000000000000", + ) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidParameter { .. })); + let error = + validate_image(deep_snapshot(), Instant::now(), "us-east-1", "000000000000") + .await + .unwrap_err(); + assert!( + matches!(error, LambdaServiceError::ImageRestore { source } if matches!(*source, DockerError::Deadline)) + ); + let provider = RustackLambda::new(LambdaConfig { + executor: ExecutorBackend::Docker, + ..LambdaConfig::default() + }); + drop( + provider.import_snapshot_before( + deep_snapshot(), + Instant::now() + Duration::from_secs(30), + ), + ); + } + fn panic_worker( + _: &mut LambdaSnapshot, + _: &str, + _: &str, + ) -> Result<(), LambdaServiceError> { + panic!("private-image-validator-panic"); + } + #[tokio::test] + async fn test_should_preserve_typed_worker_panic_without_exposing_payload_or_parameter_error() + { + let error = validate_with( + LambdaSnapshot::default(), + Instant::now() + Duration::from_secs(30), + ("us-east-1".into(), "000000000000".into()), + panic_worker, + ) + .await + .unwrap_err(); + assert!( + matches!(&error, LambdaServiceError::ImageRestoreWorker { source } if source.is_panic() && !source.is_cancelled()) + ); + assert!(!format!("{error:?}").contains("private-image-validator-panic")); + let source = error.source().unwrap(); + assert!(!source.to_string().contains("private-image-validator-panic")); + assert!(source.source().is_none()); + let public: LambdaError = error.into(); + assert_eq!(public.code, LambdaErrorCode::ServiceException); + } + #[test] + fn test_should_share_guard_work_across_ancillary_versions_and_conditions() { + let mut layer: LayerRecord = serde_json::from_value(serde_json::json!({"name":"work","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:work","nextVersion":2, + "versions":{"1":{"version":1,"description":"","compatibleRuntimes":[],"compatibleArchitectures":[],"licenseInfo":null,"codeSha256":"unused","codeSize":0,"createdDate":"2026-10-27T00:00:00Z","layerArn":"arn:aws:lambda:us-east-1:000000000000:layer:work","layerVersionArn":"arn:aws:lambda:us-east-1:000000000000:layer:work:1","policy":{"statements":[]}}}})).unwrap(); + let version = layer.versions.get_mut(&1).unwrap(); + for index in 0..128 { + version.policy.statements.push(PolicyStatement { + sid: index.to_string(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:GetLayerVersion".into(), + resource: "*".into(), + condition: Some(Value::Array(vec![Value::Null; 512])), + }); + } + let mut work = GuardWork::new(); + assert!(layer_record_guarded(&layer, &mut work).is_err()); + assert!( + work.visited_nodes > 1 + && work.visited_nodes <= 65_536 + && work.visited_nodes < 128 * 513 + ); + layer.versions.get_mut(&1).unwrap().policy = PolicyDocument::default(); + let template = layer.versions.get(&1).unwrap().clone(); + for index in 2..=6 { + layer.versions.insert(index, template.clone()); + } + let mut work = GuardWork::new(); + work.remaining = 5; + assert!(layer_record_guarded(&layer, &mut work).is_err()); + assert_eq!(work.visited_nodes, 0); + assert_eq!(work.remaining, 5); + } + #[tokio::test] + async fn test_should_classify_canceled_validation_workers_without_raw_join_diagnostics() { + let task = tokio::spawn(std::future::pending::<()>()); + task.abort(); + let source = crate::error::ImageRestoreWorker::from(task.await.unwrap_err()); + assert!(source.is_cancelled() && !source.is_panic()); + assert!(source.source().is_none()); + let error = LambdaServiceError::ImageRestoreWorker { source }; + let public: LambdaError = error.into(); + assert_eq!(public.code, LambdaErrorCode::ServiceException); + } + #[test] + fn test_should_bound_profile_counts_strings_depth_and_writer_before_effects() { + let mut bytes = BoundedBytes::default(); + assert!(bytes.write(&vec![0; IMAGE_METADATA_BYTES + 1]).is_err()); + assert_eq!(bytes.0, [] as [u8; 0]); + let mut snapshot = LambdaSnapshot::default(); + let store = FunctionStore::new("unused-count-fixture"); + store.insert(record()).unwrap(); + let one = store.export_snapshot().functions.pop().unwrap(); + snapshot.functions.functions = vec![one; 257]; + assert!(input(&snapshot, "us-east-1", "000000000000").is_err()); + assert!( + condition( + &Value::String("x".repeat(128 * 1024 + 1)), + &mut GuardWork::new() + ) + .is_err() + ); + assert!( + condition( + &Value::Array(vec![Value::Null; 65_536]), + &mut GuardWork::new() + ) + .is_err() + ); + } + } +} +#[cfg(unix)] +pub(super) use validation::{export_docker, validate_docker}; + +#[cfg(all(test, unix))] +mod managed_tests; diff --git a/crates/rustack-lambda-core/src/provider/snapshot/managed_tests.rs b/crates/rustack-lambda-core/src/provider/snapshot/managed_tests.rs new file mode 100644 index 00000000..3ee459cf --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/snapshot/managed_tests.rs @@ -0,0 +1,233 @@ +//! Managed source bodies remain separate from bounded ordinary metadata. + +use std::{ + io::{Cursor, Write}, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::time::Instant; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +use super::{ + DockerArchiveProfile, + validation::{export_docker, validate_docker}, + zip as zip_source, +}; +use crate::{ + artifact::Architecture, + error::LambdaServiceError, + identity::FunctionResourceId, + provider::{ + LambdaSnapshot, RustackLambda, + provided_zip_tests::{fixture, input, package}, + }, + storage::PolicyStatement, +}; + +fn profile() -> DockerArchiveProfile { + DockerArchiveProfile::ProvidedZip(Architecture::Arm64) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn source(large: bool) -> (tempfile::TempDir, RustackLambda, LambdaSnapshot) { + let (root, provider) = fixture(); + let mut input = input(); + input.publish = Some(true); + if large { + let bytes = STANDARD.decode(package(true)).unwrap(); + let mut writer = ZipWriter::new_append(Cursor::new(bytes)).unwrap(); + writer + .start_file( + "payload", + SimpleFileOptions::default().compression_method(CompressionMethod::Stored), + ) + .unwrap(); + writer.write_all(&vec![42; 200 * 1024]).unwrap(); + input.code.zip_file = Some(STANDARD.encode(writer.finish().unwrap().into_inner())); + } + provider.create_function(input).await.unwrap(); + let snapshot = provider.export_snapshot(); + (root, provider, snapshot) +} +async fn validated(snapshot: LambdaSnapshot) -> Result { + validate_docker(snapshot, deadline(), "us-east-1", "000000000000", profile()).await +} +#[tokio::test] +async fn test_should_round_trip_borrowed_large_original_bodies_without_relaxing_metadata() { + let (_root, provider, original) = source(true).await; + let expected = serde_json::to_value(&original).unwrap(); + assert!( + original + .functions + .functions + .first() + .unwrap() + .latest + .zip_bytes_base64 + .as_ref() + .unwrap() + .len() + > 128 * 1024 + ); + let saved = export_docker(&provider, deadline(), profile()) + .await + .unwrap(); + assert_eq!(serde_json::to_value(&saved).unwrap(), expected); + assert_eq!( + serde_json::to_value(validated(saved).await.unwrap()).unwrap(), + expected + ); + assert!( + export_docker(&provider, deadline(), DockerArchiveProfile::ImageOnly) + .await + .is_err() + ); + provider.shutdown_before(deadline()).await.unwrap(); +} +#[rstest] +#[case::description(false)] +#[case::policy_lookalike(true)] +#[tokio::test] +async fn test_should_reject_large_ancillary_text_while_zip_bodies_keep_their_own_allowance( + #[case] policy: bool, +) { + let (_root, provider, mut snapshot) = source(true).await; + let record = snapshot.functions.functions.first_mut().unwrap(); + if policy { + record.policy.statements.push(PolicyStatement { + sid: "lookalike".into(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:InvokeFunction".into(), + resource: "*".into(), + condition: Some(json!({"zipBytesBase64":"x".repeat(128 * 1024 + 1)})), + }); + } else { + record.latest.description = "x".repeat(128 * 1024 + 1); + } + assert!(matches!( + validated(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + provider.shutdown_before(deadline()).await.unwrap(); +} +#[rstest] +#[case::name(false)] +#[case::resource(true)] +#[tokio::test] +async fn test_should_reject_duplicate_function_scope_before_any_staging(#[case] identity: bool) { + let (_root, provider, mut snapshot) = source(false).await; + let mut duplicate = snapshot.functions.functions.first().unwrap().clone(); + if identity { + duplicate.name = "another".into(); + duplicate.arn = "arn:aws:lambda:us-east-1:000000000000:function:another".into(); + } else { + duplicate.resource_id = FunctionResourceId::new(); + } + snapshot.functions.functions.push(duplicate); + assert!(matches!( + validated(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + provider.shutdown_before(deadline()).await.unwrap(); +} +#[rstest] +#[case("runtime")] +#[case("handler")] +#[case("architecture")] +#[case("size")] +#[case("hash")] +#[case("inactive")] +#[case("mixed")] +#[case("missing")] +#[tokio::test] +async fn test_should_reject_invalid_saved_zip_configuration_and_identity_without_effects( + #[case] changed: &str, +) { + let (_root, provider, mut snapshot) = source(false).await; + let row = &mut snapshot.functions.functions.first_mut().unwrap().latest; + match changed { + "runtime" => row.runtime = Some("provided.al2".into()), + "handler" => row.handler = Some("bad handler".into()), + "architecture" => row.architectures = vec!["x86_64".into()], + "size" => row.code_size += 1, + "hash" => row.code_sha256 = "AA==".into(), + "inactive" => row.state = "Failed".into(), + "mixed" => row.package_type = "Image".into(), + _ => row.zip_bytes_base64 = None, + } + let old = serde_json::to_value(provider.export_snapshot()).unwrap(); + assert!(matches!( + validated(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + old + ); + provider.shutdown_before(deadline()).await.unwrap(); +} +#[tokio::test] +async fn test_should_count_repeated_revision_source_bytes_at_exact_aggregate_boundary() { + let (_root, provider, mut snapshot) = source(false).await; + let expected: Value = serde_json::to_value(&snapshot).unwrap(); + let row = snapshot.functions.functions.first().unwrap(); + let bytes = usize::try_from(row.latest.code_size) + .unwrap() + .checked_mul(2) + .unwrap(); + provider.store.check_snapshot_zip_bytes(bytes).unwrap(); + assert!(provider.store.check_snapshot_zip_bytes(bytes - 1).is_err()); + assert!(zip_source::detach_with_limit(&mut snapshot.clone(), profile(), bytes - 1).is_err()); + let bodies = zip_source::detach_with_limit(&mut snapshot, profile(), bytes).unwrap(); + assert_eq!(bodies.len(), 2); + zip_source::reattach(&mut snapshot, bodies).unwrap(); + assert_eq!(serde_json::to_value(snapshot).unwrap(), expected); + provider.shutdown_before(deadline()).await.unwrap(); +} + +#[tokio::test] +async fn test_should_preflight_global_metadata_nodes_across_individually_legal_live_records() { + let (_root, provider, _) = source(false).await; + let original = provider.store.get("provided-control").unwrap().unwrap(); + for index in 0..16 { + let mut row = original.clone(); + row.name = format!("metadata-{index}"); + row.arn = format!("arn:aws:lambda:us-east-1:000000000000:function:metadata-{index}"); + row.resource_id = FunctionResourceId::new(); + row.policy.statements.push(PolicyStatement { + sid: "nodes".into(), + effect: "Allow".into(), + principal: "*".into(), + action: "lambda:InvokeFunction".into(), + resource: "*".into(), + condition: Some(Value::Array(vec![Value::Null; 5000])), + }); + provider.store.insert(row).unwrap(); + } + assert!(matches!( + export_docker(&provider, deadline(), profile()).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + provider.shutdown_before(deadline()).await.unwrap(); +} +#[tokio::test] +async fn test_should_reject_oversized_live_ancillary_text_before_full_dto_construction() { + let (_root, provider, _) = source(false).await; + provider + .store + .try_update_if_revision("provided-control", None, |record| { + record.latest.description = "x".repeat(20 * 1024 * 1024); + Ok(()) + }) + .unwrap(); + assert!(matches!( + export_docker(&provider, deadline(), profile()).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + provider.shutdown_before(deadline()).await.unwrap(); +} diff --git a/crates/rustack-lambda-core/src/provider/snapshot/projection.rs b/crates/rustack-lambda-core/src/provider/snapshot/projection.rs new file mode 100644 index 00000000..39c91f4a --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/snapshot/projection.rs @@ -0,0 +1,391 @@ +//! Borrowed metadata serializer; original body serializers are never invoked. +//! +//! The source is the fixed live-store schema, not arbitrary input JSON. Field +//! paths are tracked through both derives and flattened maps. The projected JSON +//! is validated under the ordinary limits before a full owned DTO can exist. + +use std::{ + fmt::{Debug, Display, Formatter, Result as FmtResult}, + io::{Error as IoError, ErrorKind, Result as IoResult, Write}, +}; + +use serde::{ + Serialize, Serializer, + ser::{ + Error as SerializeError, SerializeMap, SerializeSeq, SerializeStruct, + SerializeStructVariant, SerializeTuple, SerializeTupleStruct, SerializeTupleVariant, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Path { + Root, + Functions, + Rows, + Function, + Versions, + Version, + Zip, + Other, +} +impl Path { + fn named(self, key: &str) -> Self { + match (self, key) { + (Self::Root, "functions") => Self::Functions, + (Self::Functions, "functions") => Self::Rows, + (Self::Function, "latest") => Self::Version, + (Self::Function, "versions") => Self::Versions, + (Self::Version, "zipBytesBase64") => Self::Zip, + _ => Self::Other, + } + } + fn element(self) -> Self { + if self == Self::Rows { + Self::Function + } else { + Self::Other + } + } + fn key(self, key: &K) -> Result { + if self == Self::Other { + return Ok(Self::Other); + } + let mut bytes = KeyBytes(Vec::new()); + serde_json::to_writer(&mut bytes, key) + .map_err(|_| E::custom("invalid archive metadata key"))?; + Ok(match bytes.0.as_slice() { + b"\"functions\"" => self.named("functions"), + b"\"latest\"" => self.named("latest"), + b"\"versions\"" => self.named("versions"), + b"\"zipBytesBase64\"" => self.named("zipBytesBase64"), + value + if self == Self::Versions + && !value.is_empty() + && value.iter().all(u8::is_ascii_digit) => + { + Self::Version + } + _ => Self::Other, + }) + } +} +struct KeyBytes(Vec); +impl Debug for KeyBytes { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter.write_str("KeyBytes()") + } +} +impl Write for KeyBytes { + fn write(&mut self, bytes: &[u8]) -> IoResult { + let end = self + .0 + .len() + .checked_add(bytes.len()) + .ok_or(ErrorKind::FileTooLarge)?; + // JSON escapes can encode one bounded key byte in six bytes, plus quotes. + if end > 6 * 1024 + 2 { + return Err(IoError::from(ErrorKind::FileTooLarge)); + } + self.0 + .try_reserve(bytes.len()) + .map_err(|_| IoError::from(ErrorKind::OutOfMemory))?; + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +pub(super) struct Metadata<'a, T: ?Sized> { + value: &'a T, + path: Path, +} +impl Debug for Metadata<'_, T> { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter.write_str("Metadata()") + } +} +impl<'a, T: ?Sized> Metadata<'a, T> { + pub(super) fn new(value: &'a T) -> Self { + Self { + value, + path: Path::Root, + } + } +} +impl Serialize for Metadata<'_, T> { + fn serialize(&self, serializer: S) -> Result { + self.value.serialize(Project { + inner: serializer, + path: self.path, + }) + } +} +struct Project { + inner: S, + path: Path, +} +struct List { + inner: S, + path: Path, +} +struct Fields { + inner: S, + path: Path, +} +struct Map { + inner: S, + path: Path, + pending: Option, +} +macro_rules! private_debug { + ($name:ident) => { + impl Debug for $name { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct(stringify!($name)) + .field("path", &self.path) + .finish_non_exhaustive() + } + } + }; +} +private_debug!(Project); +private_debug!(List); +private_debug!(Fields); +private_debug!(Map); +macro_rules! scalar { + ($name:ident, $type:ty) => { + fn $name(self, value: $type) -> Result { + self.inner.$name(value) + } + }; +} +impl Serializer for Project { + type Ok = S::Ok; + type Error = S::Error; + type SerializeSeq = List; + type SerializeTuple = List; + type SerializeTupleStruct = List; + type SerializeTupleVariant = List; + type SerializeMap = Map; + type SerializeStruct = Fields; + type SerializeStructVariant = Fields; + scalar!(serialize_bool, bool); + scalar!(serialize_i8, i8); + scalar!(serialize_i16, i16); + scalar!(serialize_i32, i32); + scalar!(serialize_i64, i64); + scalar!(serialize_i128, i128); + scalar!(serialize_u8, u8); + scalar!(serialize_u16, u16); + scalar!(serialize_u32, u32); + scalar!(serialize_u64, u64); + scalar!(serialize_u128, u128); + scalar!(serialize_f32, f32); + scalar!(serialize_f64, f64); + scalar!(serialize_char, char); + scalar!(serialize_str, &str); + scalar!(serialize_bytes, &[u8]); + fn serialize_none(self) -> Result { + self.inner.serialize_none() + } + fn serialize_unit(self) -> Result { + self.inner.serialize_unit() + } + fn serialize_unit_struct(self, name: &'static str) -> Result { + self.inner.serialize_unit_struct(name) + } + fn serialize_unit_variant( + self, + name: &'static str, + index: u32, + variant: &'static str, + ) -> Result { + self.inner.serialize_unit_variant(name, index, variant) + } + fn serialize_some(self, value: &T) -> Result { + self.inner.serialize_some(&Metadata { + value, + path: self.path, + }) + } + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result { + self.inner.serialize_newtype_struct( + name, + &Metadata { + value, + path: self.path, + }, + ) + } + fn serialize_newtype_variant( + self, + name: &'static str, + index: u32, + variant: &'static str, + value: &T, + ) -> Result { + self.inner.serialize_newtype_variant( + name, + index, + variant, + &Metadata { + value, + path: Path::Other, + }, + ) + } + fn serialize_seq(self, len: Option) -> Result { + Ok(List { + inner: self.inner.serialize_seq(len)?, + path: self.path, + }) + } + fn serialize_tuple(self, len: usize) -> Result { + Ok(List { + inner: self.inner.serialize_tuple(len)?, + path: self.path, + }) + } + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + Ok(List { + inner: self.inner.serialize_tuple_struct(name, len)?, + path: self.path, + }) + } + fn serialize_tuple_variant( + self, + name: &'static str, + index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(List { + inner: self + .inner + .serialize_tuple_variant(name, index, variant, len)?, + path: Path::Other, + }) + } + fn serialize_map(self, len: Option) -> Result { + Ok(Map { + inner: self.inner.serialize_map(len)?, + path: self.path, + pending: None, + }) + } + fn serialize_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + Ok(Fields { + inner: self.inner.serialize_struct(name, len)?, + path: self.path, + }) + } + fn serialize_struct_variant( + self, + name: &'static str, + index: u32, + variant: &'static str, + len: usize, + ) -> Result { + Ok(Fields { + inner: self + .inner + .serialize_struct_variant(name, index, variant, len)?, + path: Path::Other, + }) + } + fn collect_str(self, value: &T) -> Result { + self.inner.collect_str(value) + } + fn is_human_readable(&self) -> bool { + self.inner.is_human_readable() + } +} +macro_rules! list { + ($trait:ident, $method:ident) => { + impl $trait for List { + type Ok = S::Ok; + type Error = S::Error; + fn $method(&mut self, value: &T) -> Result<(), Self::Error> { + self.inner.$method(&Metadata { + value, + path: self.path.element(), + }) + } + fn end(self) -> Result { + self.inner.end() + } + } + }; +} +list!(SerializeSeq, serialize_element); +list!(SerializeTuple, serialize_element); +list!(SerializeTupleStruct, serialize_field); +list!(SerializeTupleVariant, serialize_field); +macro_rules! fields { + ($trait:ident) => { + impl $trait for Fields { + type Ok = S::Ok; + type Error = S::Error; + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> { + let path = self.path.named(key); + if path == Path::Zip { + self.inner.serialize_field(key, &Option::::None) + } else { + self.inner.serialize_field(key, &Metadata { value, path }) + } + } + fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> { + self.inner.skip_field(key) + } + fn end(self) -> Result { + self.inner.end() + } + } + }; +} +fields!(SerializeStruct); +fields!(SerializeStructVariant); +impl SerializeMap for Map { + type Ok = S::Ok; + type Error = S::Error; + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> { + self.pending = Some(self.path.key::<_, Self::Error>(key)?); + self.inner.serialize_key(key) + } + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> { + let path = self + .pending + .take() + .ok_or_else(|| Self::Error::custom("archive metadata value without key"))?; + if path == Path::Zip { + self.inner.serialize_value(&Option::::None) + } else { + self.inner.serialize_value(&Metadata { value, path }) + } + } + fn end(self) -> Result { + self.inner.end() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/provider/snapshot/projection/tests.rs b/crates/rustack-lambda-core/src/provider/snapshot/projection/tests.rs new file mode 100644 index 00000000..a4e9f440 --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/snapshot/projection/tests.rs @@ -0,0 +1,102 @@ +//! Before-copy projection controls, including the flattened live revision shape. + +use std::{cell::Cell, collections::BTreeMap}; + +use rstest::rstest; +use serde::{Serialize, Serializer, ser::Error as _}; +use serde_json::{Value, json}; + +use super::Metadata; + +#[derive(Debug, Serialize)] +struct Root { + functions: Rows, +} +#[derive(Debug, Serialize)] +struct Rows { + functions: Vec>, +} +#[derive(Debug, Serialize)] +struct Function { + latest: V, + versions: BTreeMap, +} +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Plain { + zip_bytes_base64: V, + note: &'static str, +} +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Flat { + zip_bytes_base64: V, + #[serde(flatten)] + extra: BTreeMap, +} +#[derive(Debug)] +struct Body<'a>(&'a Cell); +impl Serialize for Body<'_> { + fn serialize(&self, _: S) -> Result { + self.0.set(self.0.get().saturating_add(1)); + Err(S::Error::custom("ZIP body formatter was invoked")) + } +} +fn project(latest: V, published: V) -> Value { + let source = Root { + functions: Rows { + functions: vec![Function { + latest, + versions: [(1, published)].into_iter().collect(), + }], + }, + }; + serde_json::to_value(Metadata::new(&source)).unwrap() +} +#[rstest] +#[case::plain(false)] +#[case::flattened(true)] +fn test_should_never_invoke_original_body_serializers_in_known_struct_and_flattened_slots( + #[case] flat: bool, +) { + let calls = Cell::new(0); + let output = if flat { + let make = || Flat { + zip_bytes_base64: Body(&calls), + extra: [("note".into(), json!("retained"))].into_iter().collect(), + }; + project(make(), make()) + } else { + let make = || Plain { + zip_bytes_base64: Body(&calls), + note: "retained", + }; + project(make(), make()) + }; + assert_eq!(calls.get(), 0); + for row in [ + &output["functions"]["functions"][0]["latest"], + &output["functions"]["functions"][0]["versions"]["1"], + ] { + assert!(row["zipBytesBase64"].is_null()); + assert_eq!(row["note"], "retained"); + } +} +#[rstest] +#[case(json!({"unknown":{"functions":{"functions":[{"latest":{"zipBytesBase64":"retained"}}]}}}))] +#[case(json!({"layers":{"latest":{"zipBytesBase64":"retained"}}}))] +#[case(json!({"functions":{"functions":[{"policy":{"latest":{"zipBytesBase64":"retained"}}}]}}))] +#[case(json!({"functions":{"functions":[{"versions":{"1":{"zipBytesBase64":"retained"}}}]}}))] +#[case(json!({"functions":{"functions":{"latest":{"zipBytesBase64":"retained"}}}}))] +#[case(json!({"functions":{"functions":[{"latest":[{"zipBytesBase64":"retained"}]}]}}))] +fn test_should_keep_nonmatching_paths_and_text_version_key_lookalikes(#[case] source: Value) { + let encoded = serde_json::to_vec(&Metadata::new(&source)).unwrap(); + assert_eq!(serde_json::from_slice::(&encoded).unwrap(), source); +} +#[test] +fn test_should_bound_map_key_formatting_before_value_serialization() { + let source = json!({"x".repeat(7 * 1024):{"zipBytesBase64":"retained"}}); + let error = serde_json::to_vec(&Metadata::new(&source)).unwrap_err(); + assert!(error.to_string().contains("invalid archive metadata key")); + assert!(!format!("{:?}", Metadata::new(&source)).contains("retained")); +} diff --git a/crates/rustack-lambda-core/src/provider/snapshot/zip.rs b/crates/rustack-lambda-core/src/provider/snapshot/zip.rs new file mode 100644 index 00000000..53008fb1 --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/snapshot/zip.rs @@ -0,0 +1,209 @@ +//! Original body ownership and canonical budgets, separate from compact metadata. + +use std::{collections::HashSet, iter::once}; + +use base64::{Engine, engine::general_purpose::STANDARD}; + +use super::{DockerArchiveProfile, validation::invalid}; +use crate::{ + artifact::Architecture, + error::LambdaServiceError, + function_config::{validate_provided_settings, validate_snapshot_settings}, + provider::LambdaSnapshot, + resolver::FunctionName, + storage::{FunctionRecord, VersionRecordSnapshot}, +}; + +fn digit(byte: u8) -> Result { + let (base, offset) = match byte { + b'A'..=b'Z' => (b'A', 0), + b'a'..=b'z' => (b'a', 26), + b'0'..=b'9' => (b'0', 52), + b'+' => return Ok(62), + b'/' => return Ok(63), + _ => return Err(invalid()), + }; + byte.checked_sub(base) + .and_then(|value| value.checked_add(offset)) + .ok_or_else(invalid) +} +/// Validate without decoding or producing a second owned body. +fn decoded_len(encoded: &str, maximum: usize) -> Result { + let bytes = encoded.as_bytes(); + let maximum_encoded = maximum.div_ceil(3).checked_mul(4).ok_or_else(invalid)?; + if bytes.is_empty() || bytes.len() > maximum_encoded || !bytes.len().is_multiple_of(4) { + return Err(invalid()); + } + let padding = if bytes.ends_with(b"==") { + 2 + } else { + usize::from(bytes.ends_with(b"=")) + }; + let end = bytes.len().checked_sub(padding).ok_or_else(invalid)?; + let mut last = 0; + for byte in bytes.get(..end).ok_or_else(invalid)? { + last = digit(*byte)?; + } + if (padding == 1 && last & 3 != 0) || (padding == 2 && last & 15 != 0) { + return Err(invalid()); + } + bytes + .len() + .checked_div(4) + .and_then(|n| n.checked_mul(3)) + .and_then(|n| n.checked_sub(padding)) + .filter(|n| *n <= maximum) + .ok_or_else(invalid) +} +fn version( + version: &VersionRecordSnapshot, + architecture: Architecture, +) -> Result { + validate_snapshot_settings(version).map_err(|_| invalid())?; + validate_provided_settings(version.runtime.as_deref(), version.handler.as_deref()) + .map_err(|_| invalid())?; + if version.state != "Active" + || version + .deployment + .is_some_and(|state| state.state() != "Active") + || version.architectures.as_slice() != [architecture.as_str()] + || version.image_uri.is_some() + || version.resolved_image.is_some() + || version.image_config.is_some() + { + return Err(invalid()); + } + let bytes = decoded_len( + version.zip_bytes_base64.as_deref().ok_or_else(invalid)?, + DockerArchiveProfile::MAX_ZIP_BYTES, + )?; + if u64::try_from(bytes).ok() != Some(version.code_size) || version.code_sha256.len() != 44 { + return Err(invalid()); + } + let digest = STANDARD + .decode(&version.code_sha256) + .map_err(|_| invalid())?; + if digest.len() != 32 || STANDARD.encode(digest) != version.code_sha256 { + return Err(invalid()); + } + Ok(bytes) +} + +pub(super) fn detach( + snapshot: &mut LambdaSnapshot, + profile: DockerArchiveProfile, +) -> Result, LambdaServiceError> { + detach_with_limit(snapshot, profile, DockerArchiveProfile::MAX_ZIP_DATA_BYTES) +} +pub(super) fn detach_with_limit( + snapshot: &mut LambdaSnapshot, + profile: DockerArchiveProfile, + maximum: usize, +) -> Result, LambdaServiceError> { + let mut remaining = maximum.min(DockerArchiveProfile::MAX_ZIP_DATA_BYTES); + let mut names = HashSet::new(); + let mut identities = HashSet::new(); + for function in &snapshot.functions.functions { + FunctionName::parse(&function.name).map_err(|_| invalid())?; + if !names.insert(function.name.as_str()) || !identities.insert(function.resource_id) { + return Err(invalid()); + } + } + let mut bodies = Vec::new(); + for function in &mut snapshot.functions.functions { + let package = match function.latest.package_type.as_str() { + "Image" => "Image", + "Zip" => "Zip", + _ => return Err(invalid()), + }; + for row in once(&mut function.latest).chain(function.versions.values_mut()) { + if row.package_type != package { + return Err(invalid()); + } + match row.package_type.as_str() { + "Image" if row.zip_bytes_base64.is_none() => {} + "Zip" => { + let len = version(row, profile.zip_architecture().ok_or_else(invalid)?)?; + remaining = remaining.checked_sub(len).ok_or_else(invalid)?; + bodies.push(row.zip_bytes_base64.take().ok_or_else(invalid)?); + } + _ => return Err(invalid()), + } + } + } + Ok(bodies) +} +pub(super) fn reattach( + snapshot: &mut LambdaSnapshot, + bodies: Vec, +) -> Result<(), LambdaServiceError> { + let mut bodies = bodies.into_iter(); + for function in &mut snapshot.functions.functions { + for row in once(&mut function.latest).chain(function.versions.values_mut()) { + if row.package_type == "Zip" { + if row.zip_bytes_base64.is_some() { + return Err(invalid()); + } + row.zip_bytes_base64 = Some(bodies.next().ok_or_else(invalid)?); + } + } + } + if bodies.next().is_some() { + return Err(invalid()); + } + Ok(()) +} + +pub(super) fn record(record: &FunctionRecord) -> Result<(), LambdaServiceError> { + for row in once(&record.latest).chain(record.versions.values()) { + if row.package_type != record.latest.package_type { + return Err(invalid()); + } + if row.package_type == "Zip" { + validate_provided_settings(row.runtime.as_deref(), row.handler.as_deref()) + .map_err(|_| invalid())?; + if row.deployment.state() != "Active" + || row + .zip_bytes + .as_ref() + .is_none_or(|bytes| bytes.len() > DockerArchiveProfile::MAX_ZIP_BYTES) + { + return Err(invalid()); + } + } else if row.package_type != "Image" || row.zip_bytes.is_some() { + return Err(invalid()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + #[rstest] + #[case("AA==", 1)] + #[case("AAA=", 2)] + #[case("AAAA", 3)] + fn test_should_measure_canonical_bodies_without_decoding( + #[case] body: &str, + #[case] bytes: usize, + ) { + assert_eq!(decoded_len(body, bytes).unwrap(), bytes); + assert!(decoded_len(body, bytes.saturating_sub(1)).is_err()); + } + #[rstest] + #[case("")] + #[case("AA")] + #[case("AB==")] + #[case("AAB=")] + #[case("A===")] + #[case("AA-A")] + #[case("AA_A")] + #[case("AA A")] + #[case("====")] + fn test_should_reject_noncanonical_source_without_copying(#[case] body: &str) { + assert!(decoded_len(body, 64).is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/provider/zip_tests.rs b/crates/rustack-lambda-core/src/provider/zip_tests.rs new file mode 100644 index 00000000..1d3b721d --- /dev/null +++ b/crates/rustack-lambda-core/src/provider/zip_tests.rs @@ -0,0 +1,360 @@ +//! Control-plane validation only; this fixture never executes a handler. + +use std::{ + io::{Cursor, Write}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use rustack_lambda_model::types::FunctionCode; +use tempfile::TempDir; +use tokio::{spawn, sync::Notify, time::timeout}; +use zip::{ZipWriter, write::SimpleFileOptions}; + +use super::*; +use crate::{ + docker::DockerExecutorError, + executor::{ExecutorError, InvokeResponse}, + invocation::InvocationContext, +}; + +#[derive(Debug)] +struct ZipControl(Arc); +#[async_trait] +impl Executor for ZipControl { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn zip_available(&self, architecture: Architecture) -> Result<(), ExecutorError> { + if !self.0.load(Ordering::Acquire) { + return Err(ExecutorError::Docker(DockerExecutorError::Unavailable)); + } + if architecture != Architecture::Arm64 { + return Err(ExecutorError::Unsupported("control platform".into())); + } + Ok(()) + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported( + "control-plane-only fixture".into(), + )) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +pub(super) fn fixture() -> (TempDir, RustackLambda) { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ZipControl(Arc::new(AtomicBool::new(true)))), + ); + (root, provider) +} +pub(crate) fn package(elf: bool) -> String { + let mut body = vec![0_u8; 64]; + if elf { + body.get_mut(..7) + .unwrap() + .copy_from_slice(b"\x7fELF\x02\x01\x01"); + body.get_mut(16..18) + .unwrap() + .copy_from_slice(&2_u16.to_le_bytes()); + body.get_mut(18..20) + .unwrap() + .copy_from_slice(&183_u16.to_le_bytes()); + body.get_mut(20..24) + .unwrap() + .copy_from_slice(&1_u32.to_le_bytes()); + body.get_mut(52..54) + .unwrap() + .copy_from_slice(&64_u16.to_le_bytes()); + } + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + ) + .unwrap(); + writer.write_all(&body).unwrap(); + STANDARD.encode(writer.finish().unwrap().into_inner()) +} +pub(crate) fn input() -> CreateFunctionInput { + CreateFunctionInput { + function_name: "provided-control".into(), + runtime: Some("provided.al2023".into()), + handler: Some("handler.entry".into()), + role: "arn:aws:iam::000000000000:role/test-role".into(), + architectures: Some(vec!["arm64".into()]), + code: FunctionCode { + zip_file: Some(package(true)), + ..Default::default() + }, + ..Default::default() + } +} + +#[tokio::test] +async fn test_should_preserve_latest_and_publications_after_bad_zip_or_configuration_updates() { + let (_root, provider) = fixture(); + let mut input = input(); + input.publish = Some(true); + provider.create_function(input).await.unwrap(); + let before = provider.get_record("provided-control").unwrap(); + assert!(matches!( + provider + .update_function_code( + "provided-control", + UpdateFunctionCodeInput { + zip_file: Some(package(false)), + ..Default::default() + } + ) + .await, + Err(LambdaServiceError::InvalidZipFile { .. }) + )); + assert!(matches!( + provider.update_function_configuration( + "provided-control", + &UpdateFunctionConfigurationInput { + runtime: Some("python3.13".into()), + ..Default::default() + } + ), + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let after = provider.get_record("provided-control").unwrap(); + assert_eq!(before.latest.revision_id, after.latest.revision_id); + assert_eq!(before.latest.code_sha256, after.latest.code_sha256); + assert_eq!(before.latest.code_path, after.latest.code_path); + assert_eq!(before.versions.len(), after.versions.len()); + assert_eq!(before.latest.runtime, after.latest.runtime); + let loaded = provider + .store + .capture_artifact(after.latest.code_path.as_ref().unwrap()) + .await + .unwrap(); + assert_eq!(loaded.sha256(), before.latest.code_sha256); + assert!( + provider + .export_snapshot_before(Instant::now() + Duration::from_secs(5)) + .await + .is_err() + ); + provider + .shutdown_before(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} + +#[rstest] +#[case("python3.13", "handler.entry", "arm64")] +#[case("provided.al2", "handler.entry", "arm64")] +#[case("provided.al2023", "bad handler", "arm64")] +#[case("provided.al2023", "handler.entry", "x86_64")] +#[tokio::test] +async fn test_should_reject_uninstalled_runtime_handler_or_platform_before_publication( + #[case] runtime: &str, + #[case] handler: &str, + #[case] architecture: &str, +) { + let (_root, provider) = fixture(); + let mut input = input(); + input.runtime = Some(runtime.into()); + input.handler = Some(handler.into()); + input.architectures = Some(vec![architecture.into()]); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(provider.get_record("provided-control").is_err()); + provider + .shutdown_before(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_should_reject_bootstrap_before_initial_commit_and_wrong_architecture_before_update_staging() + { + let (_root, provider) = fixture(); + let mut bad = input(); + bad.code.zip_file = Some(package(false)); + assert!(matches!( + provider.create_function(bad).await, + Err(LambdaServiceError::InvalidZipFile { .. }) + )); + assert!(provider.get_record("provided-control").is_err()); + provider.create_function(input()).await.unwrap(); + let before = provider.get_record("provided-control").unwrap(); + assert!(matches!( + provider + .update_function_code( + "provided-control", + UpdateFunctionCodeInput { + zip_file: Some(package(true)), + architectures: Some(vec!["x86_64".into()]), + ..Default::default() + } + ) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert_eq!( + provider + .get_record("provided-control") + .unwrap() + .latest + .revision_id, + before.latest.revision_id + ); + provider + .shutdown_before(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); +} + +#[derive(Debug)] +struct HeldZip { + entered: Notify, + release: Notify, +} +#[async_trait] +impl S3CodeFetcher for HeldZip { + async fn fetch_code( + &self, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result { + self.entered.notify_one(); + self.release.notified().await; + Ok(Bytes::from(STANDARD.decode(package(true)).unwrap())) + } +} + +#[rstest] +#[case::create(false)] +#[case::update(true)] +#[tokio::test] +async fn test_should_recheck_zip_capability_at_commit_after_held_fetch(#[case] update: bool) { + let root = tempfile::tempdir().unwrap(); + let live = Arc::new(AtomicBool::new(true)); + let fetch = Arc::new(HeldZip { + entered: Notify::new(), + release: Notify::new(), + }); + let provider = Arc::new( + RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ZipControl(live.clone())), + ) + .with_code_fetcher(fetch.clone()), + ); + if update { + let mut request = input(); + request.publish = Some(true); + provider.create_function(request).await.unwrap(); + } + let before = provider.get_record("provided-control").ok(); + let call = provider.clone(); + let waiting = spawn(async move { + if update { + call.update_function_code( + "provided-control", + UpdateFunctionCodeInput { + s3_bucket: Some("test-bucket".into()), + s3_key: Some("test.zip".into()), + publish: Some(true), + ..Default::default() + }, + ) + .await + } else { + let mut request = input(); + request.code = FunctionCode { + s3_bucket: Some("test-bucket".into()), + s3_key: Some("test.zip".into()), + ..Default::default() + }; + call.create_function(request).await + } + }); + timeout(Duration::from_secs(5), fetch.entered.notified()) + .await + .unwrap(); + live.store(false, Ordering::Release); + fetch.release.notify_one(); + let result = timeout(Duration::from_secs(5), waiting) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result, + Err(LambdaServiceError::DockerNotAvailable) + )); + if let Some(before) = before { + let after = provider.get_record("provided-control").unwrap(); + assert_eq!(after.latest.revision_id, before.latest.revision_id); + assert_eq!(after.latest.code_sha256, before.latest.code_sha256); + assert_eq!(after.latest.code_path, before.latest.code_path); + assert_eq!(after.versions.len(), before.versions.len()); + } else { + assert!(provider.get_record("provided-control").is_err()); + } +} + +#[rstest] +#[case::maximum(128, true)] +#[case::over_limit(129, false)] +#[tokio::test] +async fn test_should_apply_same_handler_byte_bound_to_create_and_configuration( + #[case] length: usize, + #[case] accepted: bool, +) { + let (_root, provider) = fixture(); + let mut request = input(); + request.handler = Some("x".repeat(length)); + let result = provider.create_function(request).await; + assert_eq!(result.is_ok(), accepted); + if !accepted { + provider.create_function(input()).await.unwrap(); + } + let before = provider.get_record("provided-control").unwrap(); + let result = provider.update_function_configuration( + "provided-control", + &UpdateFunctionConfigurationInput { + handler: Some("x".repeat(length)), + ..Default::default() + }, + ); + assert_eq!(result.is_ok(), accepted); + if !accepted { + assert_eq!( + provider + .get_record("provided-control") + .unwrap() + .latest + .revision_id, + before.latest.revision_id + ); + } +} diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index 01afab34..4f9625dc 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -38,6 +38,10 @@ pub fn resolve_function_ref( pub struct FunctionName(String); impl FunctionName { /// Validate an AWS function name. + /// + /// # Errors + /// Rejects empty names, names above64 bytes or characters outside ASCII letters/digits, hyphen + /// and underscore. pub fn parse(value: &str) -> Result { if value.is_empty() || value.len() > 64 @@ -83,6 +87,9 @@ impl FunctionArn { pub struct Qualifier(String); impl Qualifier { /// Validate a version or alias. + /// + /// # Errors + /// Rejects invalid/oversized aliases and zero or overflowing numeric versions. pub fn parse(value: &str) -> Result { if value != "$LATEST" && (value.is_empty() @@ -268,14 +275,14 @@ pub fn layer_version_arn(region: &str, account_id: &str, layer_name: &str, versi pub fn parse_layer_version_arn(arn: &str) -> Result<(String, u64), LambdaServiceError> { let parts: Vec<&str> = arn.split(':').collect(); // Expected: arn:aws:lambda:region:account:layer:name:version = 8 parts - if parts.len() < 8 || parts[0] != "arn" || parts[2] != "lambda" || parts[5] != "layer" { + let ["arn", _, "lambda", _, _, "layer", name, version, ..] = parts.as_slice() else { return Err(LambdaServiceError::InvalidArn { arn: arn.to_owned(), }); - } + }; - let name = parts[6].to_owned(); - let version: u64 = parts[7] + let name = (*name).to_owned(); + let version: u64 = version .parse() .map_err(|_| LambdaServiceError::InvalidArn { arn: arn.to_owned(), @@ -304,8 +311,9 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, + image: crate::artifact::ImageArtifact::Unresolved, zip_bytes: None, - state: "Active".to_owned(), + deployment: crate::deployment::DeploymentState::Ready, last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), architectures: vec!["x86_64".to_owned()], ephemeral_storage_size: 512, @@ -339,6 +347,7 @@ mod tests { ); FunctionRecord { + resource_id: crate::identity::FunctionResourceId::default(), name: "my-func".to_owned(), arn: "arn:aws:lambda:us-east-1:000000000000:function:my-func".to_owned(), latest: make_version("$LATEST"), diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index a9b392d8..5d5243f5 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -6,8 +6,11 @@ use std::{ collections::{BTreeMap, HashMap}, - io::ErrorKind, + fmt, + io::{Error as IoError, ErrorKind, Read}, + iter::once, path::{Path, PathBuf}, + str::from_utf8, }; use base64::{Engine, engine::general_purpose::STANDARD}; @@ -18,9 +21,28 @@ use rustack_lambda_model::types::{ TracingConfig, VpcConfig, }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use sha2::{Digest, Sha256}; +use tokio::{sync::watch, time::Instant}; +use zip::{System, read::ZipFile}; + +use crate::{ + artifact::{ImageArtifact, ResolvedImage, ValidatedArtifact}, + deployment::DeploymentState, + error::LambdaServiceError, + identity::FunctionResourceId, +}; -use crate::error::LambdaServiceError; +mod restore; +mod restore_zip; +mod snapshot_view; +mod zip_layout; +pub(crate) use restore::PreparedImport; +#[cfg(unix)] +pub(crate) use restore::{check_deadline as check_restore_deadline, image_error}; +use restore_zip::ZipRestoreBudget; +pub(crate) use snapshot_view::{FunctionSnapshotRef, LayerSnapshotRef, MappingSnapshotRef}; +pub(crate) use zip_layout::ZipLayout; /// In-memory store for Lambda functions. #[derive(Debug)] @@ -30,43 +52,58 @@ pub struct FunctionStore { /// Root directory for storing extracted code. code_dir: PathBuf, /// Own completed artifacts until the store is dropped; in-flight revisions remain readable. - artifacts: DashMap, + artifacts: DashMap, staging: std::sync::Arc, + changes: watch::Sender<()>, } -/// Owns a completed artifact and its resolved filesystem boundary. -#[derive(Debug)] -struct Artifact { - directory: Option, - root: PathBuf, - extracted: PathBuf, +/// Rolls back a newly registered artifact unless its record mutation commits. +pub(crate) struct StagedArtifact<'a> { + store: &'a FunctionStore, + artifact: ValidatedArtifact, + committed: bool, } -impl Drop for Artifact { - #[allow(clippy::disallowed_methods)] // Drop has no async context; cleanup is bounded to a store-owned directory and failures are reported. - fn drop(&mut self) { - let Some(directory) = self.directory.take() else { - return; - }; - let path = directory.keep(); - let safe = std::fs::canonicalize(&self.root).is_ok_and(|root| root == self.root) - && std::fs::symlink_metadata(&self.root) - .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()) - && std::fs::symlink_metadata(&path) - .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()); - if !safe { - tracing::error!("Refusing cleanup of replaced Lambda artifact root"); - return; +impl fmt::Debug for StagedArtifact<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StagedArtifact") + .field("artifact", &self.artifact) + .field("committed", &self.committed) + .finish() + } +} + +impl<'a> StagedArtifact<'a> { + fn new(store: &'a FunctionStore, artifact: ValidatedArtifact) -> Self { + Self { + store, + artifact, + committed: false, } - if let Err(error) = std::fs::remove_dir_all(path) { - tracing::error!(%error, "Lambda artifact cleanup failed"); + } + pub(crate) fn artifact(&self) -> &ValidatedArtifact { + &self.artifact + } + pub(crate) fn commit(mut self) { + self.committed = true; + } +} + +impl Drop for StagedArtifact<'_> { + fn drop(&mut self) { + if !self.committed { + self.store + .discard_staged_artifact(self.artifact.code_root()); } } } /// Complete record for a Lambda function. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] pub struct FunctionRecord { + /// Stable identity for this resource's lifetime, independent of its name. + pub resource_id: FunctionResourceId, /// Function name. pub name: String, /// Function ARN. @@ -114,7 +151,8 @@ pub struct EventInvokeConfigRecord { } /// A snapshot of function configuration at a specific version. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] pub struct VersionRecord { /// Version string (`$LATEST` or a numeric string like `"1"`). pub version: String, @@ -135,13 +173,19 @@ pub struct VersionRecord { /// Package type (`Zip` or `Image`). pub package_type: String, /// Path to extracted code on disk. + #[serde(skip_serializing)] pub code_path: Option, /// Container image URI. pub image_uri: Option, + /// Unresolved, restored metadata, or shared live image availability. + #[serde(rename = "resolvedImage", serialize_with = "snapshot_view::image")] + pub image: ImageArtifact, /// Raw zip bytes. + #[serde(rename = "zipBytesBase64", serialize_with = "snapshot_view::zip")] pub zip_bytes: Option, - /// Function state (`Active`, `Pending`, etc.). - pub state: String, + /// Authoritative readiness and update progress. + #[serde(flatten, serialize_with = "snapshot_view::deployment")] + pub deployment: DeploymentState, /// ISO 8601 last-modified timestamp. pub last_modified: String, /// Supported architectures. @@ -166,7 +210,7 @@ pub struct VersionRecord { pub image_config: Option, /// Logging configuration. pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. pub snap_start: Option, } @@ -212,6 +256,21 @@ pub struct PolicyStatement { pub condition: Option, } +impl Drop for PolicyStatement { + fn drop(&mut self) { + // A rejected or unpolled import still owns hostile dynamic conditions. + // Move children out before their containers drop; never recurse on depth. + let mut pending: Vec<_> = self.condition.take().into_iter().collect(); + while let Some(value) = pending.pop() { + match value { + Value::Array(mut children) => pending.append(&mut children), + Value::Object(children) => pending.extend(children.into_values()), + _ => {} + } + } + } +} + /// Function URL configuration record. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -438,7 +497,7 @@ pub struct LayerStoreSnapshot { pub struct EventSourceMappingRecord { /// Unique identifier for the mapping. pub uuid: String, - /// ARN of the event source (e.g., SQS queue, DynamoDB stream). + /// ARN of the event source (e.g., SQS queue, `DynamoDB` stream). pub event_source_arn: String, /// ARN of the target Lambda function. pub function_arn: String, @@ -601,6 +660,9 @@ pub struct FunctionStoreSnapshot { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FunctionRecordSnapshot { + /// Stable identity; older snapshots allocate one when first decoded. + #[serde(default)] + pub resource_id: FunctionResourceId, /// Function name. pub name: String, /// Function ARN. @@ -653,10 +715,16 @@ pub struct VersionRecordSnapshot { pub package_type: String, /// Container image URI. pub image_uri: Option, + /// Immutable image identity; older metadata-only snapshots have none. + #[serde(default)] + pub resolved_image: Option, /// Raw zip bytes encoded as standard base64. pub zip_bytes_base64: Option, - /// Function state. + /// Legacy function state, derived from deployment on new exports. pub state: String, + /// Authoritative deployment state in current snapshots. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deployment: Option, /// ISO 8601 last-modified timestamp. pub last_modified: String, /// Supported architectures. @@ -681,13 +749,14 @@ pub struct VersionRecordSnapshot { pub image_config: Option, /// Logging configuration. pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. pub snap_start: Option, } impl FunctionRecordSnapshot { fn from_record(record: FunctionRecord) -> Self { Self { + resource_id: record.resource_id, name: record.name, arn: record.arn, latest: VersionRecordSnapshot::from_record(record.latest), @@ -708,31 +777,42 @@ impl FunctionRecordSnapshot { } } - async fn into_record( + async fn into_record<'a>( self, - store: &FunctionStore, - ) -> Result { + store: &'a FunctionStore, + mut zip: Option<&mut ZipRestoreBudget>, + ) -> Result<(FunctionRecord, Vec>), LambdaServiceError> { let name = self.name; - let latest = self.latest.into_record(store, &name).await?; + let (latest, pending) = self + .latest + .into_record(store, &name, zip.as_deref_mut()) + .await?; + let mut artifacts: Vec<_> = pending.into_iter().collect(); let mut versions = BTreeMap::new(); for (version, record) in self.versions { - versions.insert(version, record.into_record(store, &name).await?); + let (record, artifact) = record.into_record(store, &name, zip.as_deref_mut()).await?; + artifacts.extend(artifact); + versions.insert(version, record); } - Ok(FunctionRecord { - name, - arn: self.arn, - latest, - versions, - next_version: self.next_version, - aliases: self.aliases, - policy: self.policy, - tags: self.tags, - code_signing_config_arn: self.code_signing_config_arn, - url_config: self.url_config, - reserved_concurrent_executions: self.reserved_concurrent_executions, - event_invoke_configs: self.event_invoke_configs, - created_at: self.created_at, - }) + Ok(( + FunctionRecord { + resource_id: self.resource_id, + name, + arn: self.arn, + latest, + versions, + next_version: self.next_version, + aliases: self.aliases, + policy: self.policy, + tags: self.tags, + code_signing_config_arn: self.code_signing_config_arn, + url_config: self.url_config, + reserved_concurrent_executions: self.reserved_concurrent_executions, + event_invoke_configs: self.event_invoke_configs, + created_at: self.created_at, + }, + artifacts, + )) } } @@ -749,11 +829,13 @@ impl VersionRecordSnapshot { environment: record.environment, package_type: record.package_type, image_uri: record.image_uri, + resolved_image: record.image.resolved().cloned(), zip_bytes_base64: record .zip_bytes .as_ref() .map(|bytes| STANDARD.encode(bytes.as_ref())), - state: record.state, + state: record.deployment.state().into(), + deployment: Some(record.deployment), last_modified: record.last_modified, architectures: record.architectures, ephemeral_storage_size: record.ephemeral_storage_size, @@ -770,63 +852,86 @@ impl VersionRecordSnapshot { } } - async fn into_record( + async fn into_record<'a>( self, - store: &FunctionStore, + store: &'a FunctionStore, function_name: &str, - ) -> Result { - let zip_bytes = match self.zip_bytes_base64 { - Some(encoded) => { - let decoded = STANDARD.decode(encoded.as_bytes()).map_err(|source| { + mut zip: Option<&mut ZipRestoreBudget>, + ) -> Result<(VersionRecord, Option>), LambdaServiceError> { + crate::function_config::validate_snapshot_settings(&self)?; + if let Some(budget) = zip.as_deref() { + budget.check()?; + } + let (zip_bytes, artifact) = match (self.zip_bytes_base64, zip.as_deref_mut()) { + (Some(encoded), Some(budget)) => { + let staged = budget.stage(store, encoded).await?; + (Some(staged.artifact().zip_bytes().clone()), Some(staged)) + } + (Some(encoded), None) => { + let bytes = Bytes::from(STANDARD.decode(encoded.as_bytes()).map_err(|source| { LambdaServiceError::Internal { message: format!("Failed to decode Lambda zip bytes: {source}"), } - })?; - Some(Bytes::from(decoded)) + })?); + let staged = store + .stage_zip_code(function_name, &self.version, &bytes) + .await?; + (Some(bytes), Some(staged)) } - None => None, + (None, _) => (None, None), }; - let code_path = if let Some(bytes) = zip_bytes.as_ref() { - let (path, _, _) = store - .store_zip_code(function_name, &self.version, bytes.as_ref()) - .await?; - Some(path) - } else { - None - }; - - Ok(VersionRecord { - version: self.version, - runtime: self.runtime, - handler: self.handler, - role: self.role, - description: self.description, - timeout: self.timeout, - memory_size: self.memory_size, - environment: self.environment, - package_type: self.package_type, - code_path, - image_uri: self.image_uri, - zip_bytes, - state: self.state, - last_modified: self.last_modified, - architectures: self.architectures, - ephemeral_storage_size: self.ephemeral_storage_size, - code_sha256: self.code_sha256, - code_size: self.code_size, - revision_id: self.revision_id, - layers: self.layers, - vpc_config: self.vpc_config, - dead_letter_config: self.dead_letter_config, - tracing_config: self.tracing_config, - image_config: self.image_config, - logging_config: self.logging_config, - snap_start: self.snap_start, - }) + if let Some(budget) = zip { + budget.check()?; + } + let code_path = artifact + .as_ref() + .map(|artifact| artifact.artifact().code_root().to_path_buf()); + + Ok(( + VersionRecord { + version: self.version, + runtime: self.runtime, + handler: self.handler, + role: self.role, + description: self.description, + timeout: self.timeout, + memory_size: self.memory_size, + environment: self.environment, + package_type: self.package_type, + code_path, + image_uri: self.image_uri, + image: self + .resolved_image + .map_or(ImageArtifact::Unresolved, ImageArtifact::Restored), + zip_bytes, + deployment: DeploymentState::restore(&self.state, self.deployment)?, + last_modified: self.last_modified, + architectures: self.architectures, + ephemeral_storage_size: self.ephemeral_storage_size, + code_sha256: self.code_sha256, + code_size: self.code_size, + revision_id: self.revision_id, + layers: self.layers, + vpc_config: self.vpc_config, + dead_letter_config: self.dead_letter_config, + tracing_config: self.tracing_config, + image_config: self.image_config, + logging_config: self.logging_config, + snap_start: self.snap_start, + }, + artifact, + )) } } impl FunctionStore { + /// Inspect image availability without cloning stored metadata or following tags. + pub(crate) fn has_unleased_images(&self) -> bool { + self.functions.iter().any(|record| { + once(&record.latest).chain(record.versions.values()) + .any(|version| version.package_type == "Image" && version.deployment.state() == "Active" && !matches!(&version.image, ImageArtifact::Pinned(image) if image.is_live())) + }) + } fn validate_record(&self, record: &FunctionRecord) -> Result<(), LambdaServiceError> { let invalid = |message: &str| LambdaServiceError::InvalidParameter { message: message.into(), @@ -851,7 +956,11 @@ impl FunctionStore { return Err(invalid("Invalid function version or concurrency")); } for (number, version) in &record.versions { - if *number == 0 || version.version != number.to_string() { + if *number == 0 + || version.version != number.to_string() + || version.deployment != DeploymentState::Ready + || (version.package_type == "Image" && version.image.resolved().is_none()) + { return Err(invalid("Published version number mismatch")); } } @@ -868,34 +977,27 @@ impl FunctionStore { return Err(invalid("Event qualifier mismatch")); } } - for version in std::iter::once(&record.latest).chain(record.versions.values()) { + for version in once(&record.latest).chain(record.versions.values()) { crate::resolver::Qualifier::parse(&version.version)?; - if !(1..=900).contains(&version.timeout) - || !(128..=10240).contains(&version.memory_size) - || version.architectures.len() != 1 - || version - .architectures - .iter() - .any(|architecture| architecture != "arm64" && architecture != "x86_64") - || version.environment.len() > 128 - || version - .environment - .iter() - .any(|(key, value)| key.len() > 256 || value.len() > 4096) - { - return Err(invalid("Invalid execution configuration")); - } + crate::function_config::validate_execution_settings(version)?; if let Some(path) = &version.code_path { - if !self.artifacts.contains_key(path) { - return Err(invalid("Code path is not an owned immutable artifact")); - } - } - if let Some(bytes) = &version.zip_bytes { - if version.code_sha256 != compute_sha256(bytes) - || version.code_size != bytes.len() as u64 + let artifact = self + .artifacts + .get(path) + .ok_or_else(|| invalid("Code path is not an owned immutable artifact"))?; + let zip_matches = version.zip_bytes.as_ref().is_some_and(|bytes| { + bytes.len() == artifact.zip_bytes().len() + && (bytes.as_ptr() == artifact.zip_bytes().as_ptr() + || bytes == artifact.zip_bytes()) + }); + if !zip_matches + || version.code_sha256 != artifact.sha256() + || version.code_size != artifact.zip_bytes().len() as u64 { return Err(invalid("Deployment package metadata mismatch")); } + } else if version.zip_bytes.is_some() { + return Err(invalid("ZIP bytes have no owned extracted artifact")); } } Ok(()) @@ -908,9 +1010,15 @@ impl FunctionStore { code_dir: code_dir.into(), artifacts: DashMap::new(), staging: std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + changes: watch::channel(()).0, } } + #[cfg(unix)] + pub(crate) fn changes(&self) -> watch::Receiver<()> { + self.changes.subscribe() + } + /// Returns a reference to the code directory path. #[must_use] pub fn code_dir(&self) -> &Path { @@ -931,18 +1039,27 @@ impl FunctionStore { }), Entry::Vacant(entry) => { entry.insert(record); + self.changes.send_replace(()); Ok(()) } } } /// Get a clone of a function record by name. + /// + /// # Errors + /// + /// Rejects an invalid logical function name; an absent valid name returns `None`. pub fn get(&self, name: &str) -> Result, LambdaServiceError> { crate::resolver::FunctionName::parse(name)?; Ok(self.functions.get(name).map(|r| r.value().clone())) } /// Check whether a function exists. + /// + /// # Errors + /// + /// Rejects an invalid logical function name. pub fn contains(&self, name: &str) -> Result { crate::resolver::FunctionName::parse(name)?; Ok(self.functions.contains_key(name)) @@ -974,6 +1091,32 @@ impl FunctionStore { ) -> Result where F: FnOnce(&mut FunctionRecord) -> R, + { + self.try_update_if_revision(name, expected, |record| Ok(f(record))) + } + + /// Apply a fallible mutation to a private candidate and commit only on success. + /// + /// # Errors + /// Rejects stale revisions, invalid records, missing functions or closure failures; + /// a closure error leaves all live metadata unchanged. + /// + /// ```no_run + /// # fn update(store: &rustack_lambda_core::storage::FunctionStore) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// store.try_update_if_revision("worker", None, |record| { + /// record.latest.description = "updated".into(); + /// Ok(()) + /// })?; + /// # Ok(()) } + /// ``` + pub fn try_update_if_revision( + &self, + name: &str, + expected: Option<&str>, + f: F, + ) -> Result + where + F: FnOnce(&mut FunctionRecord) -> Result, { crate::resolver::FunctionName::parse(name)?; match self.functions.get_mut(name) { @@ -984,14 +1127,15 @@ impl FunctionStore { }); } let mut candidate = entry.value().clone(); - let result = f(&mut candidate); - if candidate.name != name { + let result = f(&mut candidate)?; + if candidate.name != name || candidate.resource_id != entry.resource_id { return Err(LambdaServiceError::InvalidParameter { message: "Function identity is immutable".into(), }); } self.validate_record(&candidate)?; *entry = candidate; + self.changes.send_replace(()); Ok(result) } None => Err(LambdaServiceError::FunctionNotFound { @@ -1003,9 +1147,17 @@ impl FunctionStore { /// Remove a function from the store. /// /// Returns the removed record, or `None` if it did not exist. + /// + /// # Errors + /// + /// Rejects an invalid logical function name before changing the store. pub fn remove(&self, name: &str) -> Result, LambdaServiceError> { crate::resolver::FunctionName::parse(name)?; - Ok(self.functions.remove(name).map(|(_, v)| v)) + let removed = self.functions.remove(name).map(|(_, v)| v); + if removed.is_some() { + self.changes.send_replace(()); + } + Ok(removed) } /// List all function records. @@ -1052,32 +1204,7 @@ impl FunctionStore { &self, snapshot: FunctionStoreSnapshot, ) -> Result<(), LambdaServiceError> { - self.validate_root().await?; - let mut staged = BTreeMap::new(); - for function in snapshot.functions { - crate::resolver::FunctionName::parse(&function.name)?; - let (name, qualifier) = crate::resolver::resolve_function_ref(&function.arn)?; - if name != function.name || qualifier.is_some() { - return Err(LambdaServiceError::InvalidParameter { - message: "Snapshot function ARN mismatch".into(), - }); - } - for (alias, record) in &function.aliases { - crate::resolver::Qualifier::parse(alias)?; - crate::resolver::Qualifier::parse(&record.function_version)?; - } - let record = function.into_record(self).await?; - self.validate_record(&record)?; - if staged.insert(record.name.clone(), record).is_some() { - return Err(LambdaServiceError::InvalidParameter { - message: "Duplicate snapshot function".into(), - }); - } - } - self.functions.clear(); - for (name, record) in staged { - self.functions.insert(name, record); - } + self.prepare_snapshot(snapshot).await?.commit(); Ok(()) } @@ -1102,7 +1229,7 @@ impl FunctionStore { &self, function_name: &str, version: &str, - zip_bytes: &[u8], + zip_bytes: &Bytes, ) -> Result<(PathBuf, String, u64), LambdaServiceError> { crate::resolver::FunctionName::parse(function_name)?; crate::resolver::Qualifier::parse(version)?; @@ -1118,10 +1245,10 @@ impl FunctionStore { message: "ZIP staging capacity exhausted".into(), })?; let root = self.code_dir.clone(); - let bytes_owned = zip_bytes.to_vec(); + let bytes_owned = zip_bytes.clone(); let task = tokio::task::spawn_blocking(move || { let _permit = permit; - stage_zip(&root, &bytes_owned) + stage_zip(&root, bytes_owned) }); let artifact = tokio::time::timeout(std::time::Duration::from_secs(30), task) .await @@ -1131,31 +1258,62 @@ impl FunctionStore { .map_err(|e| LambdaServiceError::Internal { message: format!("ZIP extraction task failed: {e}"), })??; - let extracted = artifact.extracted.clone(); + let extracted = artifact.code_root().to_path_buf(); + let sha256 = artifact.sha256().to_owned(); + let code_size = artifact.zip_bytes().len() as u64; self.artifacts.insert(extracted.clone(), artifact); - let sha256 = compute_sha256(zip_bytes); - let code_size = zip_bytes.len() as u64; - Ok((extracted, sha256, code_size)) } + pub(crate) async fn stage_zip_code( + &self, + function_name: &str, + version: &str, + bytes: &Bytes, + ) -> Result, LambdaServiceError> { + let (path, _, _) = self.store_zip_code(function_name, version, bytes).await?; + let artifact = self + .artifacts + .get(&path) + .map(|artifact| artifact.value().clone()) + .ok_or_else(|| LambdaServiceError::Internal { + message: "Newly staged artifact is missing".into(), + })?; + Ok(StagedArtifact::new(self, artifact)) + } + + fn discard_staged_artifact(&self, path: &Path) { + self.artifacts.remove(path); + } + /// Validate logical retirement of a function's code. /// /// Immutable artifacts are retained until store drop so published/in-flight /// references remain valid. No name-derived directory is ever deleted. + /// + /// # Errors + /// + /// Rejects an invalid function name, a symlink/non-directory artifact root, or + /// filesystem inspection failures. pub async fn cleanup_code(&self, function_name: &str) -> Result<(), LambdaServiceError> { crate::resolver::FunctionName::parse(function_name)?; self.validate_root().await } /// Check an immutable artifact before handing it to an executor. + /// + /// # Errors + /// + /// Rejects unowned artifacts, replaced roots, symlinked artifact components/bootstrap, + /// paths outside the root, or filesystem inspection/canonicalization failures. + /// An absent bootstrap is accepted here; execution performs its own readiness check. pub async fn validate_artifact(&self, path: &Path) -> Result<(), LambdaServiceError> { self.validate_root().await?; let owned_root = self .artifacts .get(path) - .map(|artifact| artifact.root.clone()) + .map(|artifact| artifact.storage_root().to_path_buf()) .ok_or_else(|| LambdaServiceError::InvalidParameter { message: "Unowned Lambda artifact".into(), })?; @@ -1213,7 +1371,44 @@ impl FunctionStore { } } + /// Issue a shared artifact lease after bounded ownership/filesystem validation. + /// + /// # Errors + /// Rejects unowned or replaced paths, filesystem failures and validation timeout. + /// + /// ```no_run + /// # async fn capture(store: &rustack_lambda_core::storage::FunctionStore, path: &std::path::Path) + /// # -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let artifact = store.capture_artifact(path).await?; + /// assert_eq!(artifact.code_root(), path); + /// # Ok(()) } + /// ``` + pub async fn capture_artifact( + &self, + path: &Path, + ) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.validate_artifact(path), + ) + .await + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "Artifact validation deadline exceeded".into(), + })??; + self.artifacts + .get(path) + .map(|artifact| artifact.value().clone()) + .ok_or_else(|| LambdaServiceError::InvalidParameter { + message: "Unowned Lambda artifact".into(), + }) + } + /// Reject a symlink at the operator-selected artifact root. + /// + /// # Errors + /// + /// Rejects a symlink/non-directory root or filesystem inspection errors; an absent + /// root is valid before first staging. pub async fn validate_root(&self) -> Result<(), LambdaServiceError> { match tokio::fs::symlink_metadata(&self.code_dir).await { Ok(meta) if meta.file_type().is_symlink() || !meta.is_dir() => { @@ -1231,8 +1426,22 @@ impl FunctionStore { } /// Stage a package in a freshly created private directory on a blocking worker. -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] // Synchronous filesystem work is isolated to spawn_blocking. -fn stage_zip(root: &Path, bytes: &[u8]) -> Result { +pub(crate) fn stage_zip( + root: &Path, + bytes: Bytes, +) -> Result { + stage_zip_before(root, bytes, MAX_EXTRACTED_SIZE, None) +} + +/// Blocking staging retains only private artifact state, never store publication. +#[allow(clippy::disallowed_methods, clippy::disallowed_types)] // Called only in spawn_blocking, as the original staging boundary. +fn stage_zip_before( + root: &Path, + bytes: Bytes, + max_extracted: u64, + deadline: Option, +) -> Result { + restore_zip::check(deadline)?; let io_error = |source| LambdaServiceError::ArtifactIo { source }; std::fs::create_dir_all(root).map_err(io_error)?; if std::fs::symlink_metadata(root) @@ -1252,23 +1461,21 @@ fn stage_zip(root: &Path, bytes: &[u8]) -> Result let extracted = artifact.path().join("extracted"); let result = (|| { std::fs::create_dir(&extracted).map_err(io_error)?; - std::fs::write(artifact.path().join("code.zip"), bytes).map_err(io_error)?; - extract_zip(bytes, &extracted, MAX_EXTRACTED_SIZE) + std::fs::write(artifact.path().join("code.zip"), &bytes).map_err(io_error)?; + extract_zip(&bytes, &extracted, max_extracted, deadline) })(); if let Err(error) = result { artifact.close().map_err(io_error)?; return Err(error); } - Ok(Artifact { - directory: Some(artifact), - root, - extracted, - }) + Ok(ValidatedArtifact::from_staged( + artifact, root, extracted, bytes, + )) } /// Maximum extracted deployment package size (250 MB, mirroring the AWS /// unzipped-package limit). Bounds zip-bomb expansion during extraction. -const MAX_EXTRACTED_SIZE: u64 = 250 * 1024 * 1024; +pub(crate) const MAX_EXTRACTED_SIZE: u64 = 250 * 1024 * 1024; /// Extract a zip archive into `target`, preserving unix file modes. /// @@ -1276,7 +1483,7 @@ const MAX_EXTRACTED_SIZE: u64 = 250 * 1024 * 1024; /// and archives whose entries would expand beyond [`MAX_EXTRACTED_SIZE`] /// (zip bombs). ZIP and filesystem failures are always returned to the caller. /// -/// Synchronous std::fs is intentional: this runs inside `spawn_blocking` and +/// Synchronous `std::fs` is intentional: this runs inside `spawn_blocking` and /// the `zip` crate's reader API is itself blocking, so wrapping each I/O in /// tokio would only add overhead. #[allow(clippy::disallowed_methods, clippy::disallowed_types)] @@ -1284,6 +1491,7 @@ fn extract_zip( zip_bytes: &[u8], target: &Path, max_extracted: u64, + deadline: Option, ) -> Result<(), LambdaServiceError> { use std::{ fs::{self, File}, @@ -1296,44 +1504,36 @@ fn extract_zip( message: format!("not a valid zip archive: {e}"), })?; + if archive.has_duplicate_names() { + return Err(LambdaServiceError::InvalidZipFile { + message: "Duplicate ZIP entry".into(), + }); + } if archive.len() > 10_000 { return Err(LambdaServiceError::InvalidZipFile { message: "Too many ZIP entries".into(), }); } - let mut paths = std::collections::HashSet::new(); + let mut layout = ZipLayout::default(); let mut extracted_total: u64 = 0; for i in 0..archive.len() { + restore_zip::check(deadline)?; let mut entry = archive .by_index(i) .map_err(|e| LambdaServiceError::InvalidZipFile { message: format!("zip entry {i}: {e}"), })?; - let Some(rel) = entry.enclosed_name() else { - return Err(LambdaServiceError::InvalidZipFile { - message: format!("zip entry has invalid path: {}", entry.name()), - }); - }; - if entry.name().len() > 1024 - || entry.name().contains('\\') - || entry.name().split('/').any(|component| component == "..") - || entry - .unix_mode() - .is_some_and(|mode| mode & 0o170_000 == 0o120_000 || mode & 0o7000 != 0) - || !paths.insert(rel.clone()) - { - return Err(LambdaServiceError::InvalidZipFile { - message: "Unsafe or duplicate ZIP entry".into(), - }); - } + let rel = validated_zip_path(&entry)?; + layout.admit(&rel, entry.is_dir())?; let out_path = target.join(&rel); // Defense in depth: ensure the resolved path stays within target. if !out_path.starts_with(target) { return Err(LambdaServiceError::InvalidZipFile { - message: format!("zip entry escapes extraction root: {}", entry.name()), + message: "ZIP entry escapes extraction root".into(), }); } if entry.is_dir() { + validate_zip_directory(&mut entry)?; fs::create_dir_all(&out_path).map_err(|e| LambdaServiceError::Internal { message: format!("create dir {}: {e}", out_path.display()), })?; @@ -1344,10 +1544,7 @@ fn extract_zip( let remaining = max_extracted.saturating_sub(extracted_total); if entry.size() > remaining { return Err(LambdaServiceError::InvalidZipFile { - message: format!( - "zip archive expands beyond {max_extracted} bytes (entry: {})", - entry.name(), - ), + message: format!("ZIP archive expands beyond {max_extracted} bytes"), }); } if let Some(parent) = out_path.parent() { @@ -1357,19 +1554,9 @@ fn extract_zip( } let mut out = File::create_new(&out_path) .map_err(|source| LambdaServiceError::ArtifactIo { source })?; - let copied = io::copy( - &mut (&mut entry).take(remaining.saturating_add(1)), - &mut out, - ) - .map_err(|source| { - if source.kind() == ErrorKind::InvalidData { - LambdaServiceError::InvalidZipFile { - message: format!("ZIP CRC/decompression failed: {source}"), - } - } else { - LambdaServiceError::ArtifactIo { source } - } - })?; + let copied = + io::copy(&mut (&mut entry).take(remaining), &mut out).map_err(zip_read_error)?; + verify_zip_entry(&mut entry, copied)?; extracted_total = extracted_total.saturating_add(copied); if extracted_total > max_extracted { return Err(LambdaServiceError::InvalidZipFile { @@ -1392,6 +1579,78 @@ fn extract_zip( Ok(()) } +/// Probe EOF/CRC without writing beyond the charged expansion allowance. +fn verify_zip_entry( + entry: &mut ZipFile<'_, R>, + copied: u64, +) -> Result<(), LambdaServiceError> { + let mut overflow = [0_u8]; + if entry.read(&mut overflow).map_err(zip_read_error)? != 0 || copied != entry.size() { + return Err(LambdaServiceError::InvalidZipFile { + message: "ZIP size/decompression mismatch".into(), + }); + } + Ok(()) +} +fn zip_read_error(source: IoError) -> LambdaServiceError { + if source.kind() == ErrorKind::InvalidData { + LambdaServiceError::InvalidZipFile { + message: "ZIP CRC/decompression failed".into(), + } + } else { + LambdaServiceError::ArtifactIo { source } + } +} + +/// Require positive compressed-stream EOF for a metadata-validated directory. +pub(crate) fn validate_zip_directory( + entry: &mut ZipFile<'_, R>, +) -> Result<(), LambdaServiceError> { + let mut byte = [0]; + match entry.read(&mut byte) { + Ok(0) => Ok(()), + _ => Err(LambdaServiceError::InvalidZipFile { + message: "Invalid ZIP directory stream".into(), + }), + } +} + +/// Check original names and attributes before ZIP9 normalizes decoded metadata. +pub(crate) fn validated_zip_path( + entry: &ZipFile<'_, R>, +) -> Result { + let invalid = || LambdaServiceError::InvalidZipFile { + message: "Unsafe ZIP entry name or attributes".into(), + }; + let raw_name = entry.name_raw(); + if raw_name.len() > 1024 + || (entry.flags().as_u16() & (1 << 11) != 0 && from_utf8(raw_name).is_err()) + { + return Err(invalid()); + } + let name = entry.name().map_err(|_| invalid())?; + let mode = if entry.system() == System::Dos { + 0 + } else { + entry.external_attributes() >> 16 + }; + if name.len() > 1024 + || name.starts_with('/') + || name.as_bytes().get(1) == Some(&b':') + || name.contains('\0') + || name.contains('\\') + || name.split('/').any(|component| component == "..") + || !matches!(mode & 0o170_000, 0 | 0o100_000 | 0o040_000) + || (mode & 0o170_000 == 0o040_000 && !entry.is_dir()) + || (mode & 0o170_000 == 0o100_000 && entry.is_dir()) + || mode & 0o7000 != 0 + || (entry.is_dir() && (entry.size() != 0 || entry.crc32() != 0)) + { + return Err(invalid()); + } + entry.enclosed_name().ok_or_else(invalid) +} + /// Compute base64-encoded SHA-256 hash of the given data. #[must_use] pub fn compute_sha256(data: &[u8]) -> String { @@ -1401,9 +1660,141 @@ pub fn compute_sha256(data: &[u8]) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; + #[tokio::test] + async fn test_should_release_unpublished_artifacts_and_preserve_committed_code() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"code")); + let (committed, _, _) = store.store_zip_code("committed", "$LATEST", &bytes).await?; + let pending = store.stage_zip_code("pending", "$LATEST", &bytes).await?; + let pending_path = pending.artifact().code_root().to_path_buf(); + assert_eq!(store.artifacts.len(), 2); + drop(pending); + assert_eq!(store.artifacts.len(), 1); + assert!(tokio::fs::try_exists(committed).await?); + assert!(!tokio::fs::try_exists(pending_path).await?); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_unpublished_code_when_its_owner_future_is_aborted() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = std::sync::Arc::new(FunctionStore::new(root.path())); + let owner = std::sync::Arc::clone(&store); + let (started, receiver) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"pending")); + let staged = owner.stage_zip_code("cancelled", "$LATEST", &bytes).await?; + let _ = started.send(staged.artifact().code_root().to_path_buf()); + std::future::pending::<()>().await; + staged.commit(); + Ok::<(), LambdaServiceError>(()) + }); + let _cancel = crate::work::CancelOnDrop(task.abort_handle()); + let path = tokio::time::timeout(std::time::Duration::from_secs(5), receiver).await??; + task.abort(); + assert!(task.await.is_err_and(|error| error.is_cancelled())); + assert!(store.artifacts.is_empty()); + assert!(!tokio::fs::try_exists(path).await?); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_snapshot_artifacts_when_later_validation_rejects_restore() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + store.insert(sample_record("live"))?; + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"unpublished")); + let mut candidate = sample_record("candidate"); + candidate.latest.code_sha256 = "mismatched-snapshot-hash".into(); + candidate.latest.code_size = bytes.len() as u64; + candidate.latest.zip_bytes = Some(bytes); + let snapshot = FunctionStoreSnapshot { + functions: vec![FunctionRecordSnapshot::from_record(candidate)], + }; + assert!(matches!( + store.import_snapshot(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.contains("live")?); + assert_eq!(store.artifacts.len(), 0); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_validate_snapshot_settings_before_decoding_or_staging_its_archive() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let mut snapshot = FunctionRecordSnapshot::from_record(sample_record("invalid-settings")); + snapshot.latest.timeout = 0; + snapshot.latest.zip_bytes_base64 = Some("invalid-base64".into()); + let result = store + .import_snapshot(FunctionStoreSnapshot { + functions: vec![snapshot], + }) + .await; + assert!(matches!( + result, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.is_empty()); + assert!(store.artifacts.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn test_should_keep_code_alive_until_the_last_invocation_lease_releases_it() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"owned-code")); + let (path, sha256, size) = store.store_zip_code("leased", "$LATEST", &bytes).await?; + let lease = store.capture_artifact(&path).await?; + assert_eq!(lease.zip_bytes().as_ptr(), bytes.as_ptr()); + assert_eq!(lease.sha256(), sha256); + assert_eq!(lease.zip_bytes().len() as u64, size); + drop(store); + assert!(tokio::fs::metadata(&path).await?.is_dir()); + assert_eq!( + tokio::fs::read(path.join("fixture.txt")).await?, + b"owned-code" + ); + let retained = lease.clone(); + drop(lease); + assert!(tokio::fs::metadata(&path).await?.is_dir()); + drop(retained); + assert!( + matches!(tokio::fs::metadata(path).await, Err(error) if error.kind() == ErrorKind::NotFound) + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_unowned_code_when_issuing_an_artifact_lease() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + assert!(matches!( + store.capture_artifact(root.path()).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + Ok(()) + } + #[tokio::test] async fn test_should_reject_hostile_names_at_every_store_boundary() { let root = tempfile::tempdir().unwrap(); @@ -1424,7 +1815,12 @@ mod tests { assert!(store.contains(name).is_err()); assert!(store.remove(name).is_err()); assert!(store.update(name, |_| ()).is_err()); - assert!(store.store_zip_code(name, "$LATEST", &zip).await.is_err()); + assert!( + store + .store_zip_code(name, "$LATEST", &Bytes::copy_from_slice(&zip)) + .await + .is_err() + ); assert!(store.cleanup_code(name).await.is_err()); } assert!(store.is_empty()); @@ -1444,15 +1840,25 @@ mod tests { let root = tempfile::tempdir().unwrap(); let store = FunctionStore::new(root.path()); let (a, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); assert!(matches!( - store.store_zip_code("safe", "$LATEST", b"bad ZIP").await, + store + .store_zip_code("safe", "$LATEST", &Bytes::from_static(b"bad ZIP")) + .await, Err(LambdaServiceError::InvalidZipFile { .. }) )); let (b, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"B")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"B")), + ) .await .unwrap(); assert_ne!(a, b); @@ -1470,7 +1876,11 @@ mod tests { let root = tempfile::tempdir().unwrap(); let store = FunctionStore::new(root.path()); let (old, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"old")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"old")), + ) .await .unwrap(); let mut corrupt = crate::test_zip::minimal_zip(b"crc-marker"); @@ -1481,7 +1891,7 @@ mod tests { corrupt[position] ^= 1; assert!( store - .store_zip_code("safe", "$LATEST", &corrupt) + .store_zip_code("safe", "$LATEST", &Bytes::copy_from_slice(&corrupt)) .await .is_err() ); @@ -1497,7 +1907,7 @@ mod tests { let conflict = writer.finish().unwrap().into_inner(); assert!( store - .store_zip_code("safe", "$LATEST", &conflict) + .store_zip_code("safe", "$LATEST", &Bytes::copy_from_slice(&conflict)) .await .is_err() ); @@ -1547,7 +1957,11 @@ mod tests { let store = FunctionStore::new(&link); assert!( store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")) + ) .await .is_err() ); @@ -1560,7 +1974,11 @@ mod tests { ); let store = FunctionStore::new(root.path().join("owned")); let (artifact, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); tokio::fs::remove_dir_all(&artifact).await.unwrap(); @@ -1571,7 +1989,11 @@ mod tests { let managed = root.path().join("replaceable"); let store = FunctionStore::new(&managed); let (artifact, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); let artifact_id = artifact.parent().unwrap().file_name().unwrap(); @@ -1611,8 +2033,9 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, + image: ImageArtifact::Unresolved, zip_bytes: None, - state: "Active".to_owned(), + deployment: DeploymentState::Ready, last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), architectures: vec!["x86_64".to_owned()], ephemeral_storage_size: 512, @@ -1629,8 +2052,9 @@ mod tests { } } - fn sample_record(name: &str) -> FunctionRecord { + pub(crate) fn sample_record(name: &str) -> FunctionRecord { FunctionRecord { + resource_id: FunctionResourceId::default(), name: name.to_owned(), arn: format!("arn:aws:lambda:us-east-1:000000000000:function:{name}"), latest: sample_version(), @@ -1647,6 +2071,95 @@ mod tests { } } + #[tokio::test] + async fn test_should_preserve_function_resource_identity_through_snapshot_and_updates() + -> Result<(), Box> { + let store = test_store(); + let record = sample_record("identity"); + let identity = record.resource_id; + store.insert(record)?; + store.update("identity", |record| { + record.latest.description = "changed".into(); + })?; + let saved = serde_json::to_vec(&store.export_snapshot())?; + let snapshot: FunctionStoreSnapshot = serde_json::from_slice(&saved)?; + assert_eq!( + snapshot.functions.first().map(|record| record.resource_id), + Some(identity) + ); + let restored = test_store(); + restored.import_snapshot(snapshot).await?; + assert_eq!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + let result = restored.update("identity", |record| { + record.resource_id = FunctionResourceId::default(); + }); + assert!(matches!( + result, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert_eq!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + restored.remove("identity")?; + restored.insert(sample_record("identity"))?; + assert_ne!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_duplicate_snapshot_identities_without_replacing_live_store() + -> Result<(), LambdaServiceError> { + let store = test_store(); + store.insert(sample_record("live"))?; + let mut first = sample_record("first"); + let second = sample_record("second"); + first.resource_id = second.resource_id; + let snapshot = FunctionStoreSnapshot { + functions: vec![ + FunctionRecordSnapshot::from_record(first), + FunctionRecordSnapshot::from_record(second), + ], + }; + assert!(matches!( + store.import_snapshot(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.contains("live")?); + assert_eq!(store.len(), 1); + Ok(()) + } + + #[test] + fn test_should_assign_identity_only_when_legacy_snapshot_omits_it() + -> Result<(), Box> { + let snapshot = FunctionRecordSnapshot::from_record(sample_record("legacy")); + let mut value = serde_json::to_value(snapshot)?; + let fields = value.as_object_mut().ok_or("snapshot must be an object")?; + fields.remove("resourceId"); + let restored: FunctionRecordSnapshot = serde_json::from_value(value.clone())?; + let saved = serde_json::to_value(&restored)?; + assert_eq!( + serde_json::from_value::(saved)?.resource_id, + restored.resource_id + ); + value + .as_object_mut() + .ok_or("snapshot must be an object")? + .insert( + "resourceId".into(), + "00000000-0000-0000-0000-000000000000".into(), + ); + assert!(serde_json::from_value::(value).is_err()); + Ok(()) + } + #[test] fn test_should_insert_and_get_function() { let store = test_store(); @@ -1842,7 +2355,7 @@ mod tests { let zip_data = crate::test_zip::minimal_zip(b"fixture"); let (dir, sha256, size) = store - .store_zip_code("test-func", "$LATEST", &zip_data) + .store_zip_code("test-func", "$LATEST", &Bytes::copy_from_slice(&zip_data)) .await .unwrap(); @@ -1891,7 +2404,7 @@ mod tests { } let (dir, _sha, _size) = store - .store_zip_code("real-func", "$LATEST", &buf) + .store_zip_code("real-func", "$LATEST", &Bytes::copy_from_slice(&buf)) .await .unwrap(); @@ -1932,12 +2445,215 @@ mod tests { } let err = store - .store_zip_code("evil", "$LATEST", &buf) + .store_zip_code("evil", "$LATEST", &Bytes::copy_from_slice(&buf)) .await .unwrap_err(); assert!(matches!(err, LambdaServiceError::InvalidZipFile { .. })); } + #[rstest::rstest] + #[case::invalid_utf8(vec![0xff; 11], true)] + #[case::raw_limit(vec![b'a'; 1025], true)] + #[case::decoded_limit(vec![0x82; 513], false)] + #[tokio::test] + async fn test_should_reject_invalid_zip_names_without_publishing_artifacts( + #[case] raw: Vec, + #[case] utf8: bool, + ) { + use std::io::{Cursor, Write}; + + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "a".repeat(raw.len()), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.write_all(b"payload").unwrap(); + let mut bytes = writer.finish().unwrap().into_inner(); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + let flag = if utf8 { 1u16 << 11 } else { 0 }; + bytes + .get_mut(6..8) + .unwrap() + .copy_from_slice(&flag.to_le_bytes()); + bytes + .get_mut(central + 8..central + 10) + .unwrap() + .copy_from_slice(&flag.to_le_bytes()); + bytes + .get_mut(30..30 + raw.len()) + .unwrap() + .copy_from_slice(&raw); + bytes + .get_mut(central + 46..central + 46 + raw.len()) + .unwrap() + .copy_from_slice(&raw); + + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let error = store + .store_zip_code("safe", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert!(store.artifacts.is_empty()); + let mut entries = tokio::fs::read_dir(root.path()).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[rstest::rstest] + #[case::symlink(0o120_777)] + #[case::setuid(0o104_755)] + #[case::setgid(0o102_755)] + #[case::sticky(0o101_755)] + #[case::fifo(0o010_644)] + #[case::character_device(0o020_644)] + #[case::block_device(0o060_644)] + #[case::socket(0o140_644)] + #[case::directory_as_file(0o040_755)] + #[tokio::test] + async fn test_should_reject_hostile_zip_attributes_without_replacing_code(#[case] mode: u32) { + use std::io::Cursor; + + let root = tempfile::tempdir().unwrap(); + let code_dir = root.path().join("code"); + let sentinel = root.path().join("sentinel"); + tokio::fs::write(&sentinel, b"outside").await.unwrap(); + let store = FunctionStore::new(&code_dir); + let (old, _, _) = store + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"old")), + ) + .await + .unwrap(); + + // Encode the central-directory attributes directly: safe writer APIs + // deliberately strip special permission bits. The symlink payload would + // target the outside sentinel if materialized as a link. + let mut hostile = crate::test_zip::minimal_zip(sentinel.as_os_str().as_encoded_bytes()); + let central = hostile + .windows(4) + .position(|bytes| bytes == b"PK\x01\x02") + .unwrap(); + hostile + .get_mut(central + 4..central + 6) + .unwrap() + .copy_from_slice(&((3u16 << 8) | 0x14).to_le_bytes()); + hostile + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(mode << 16).to_le_bytes()); + let mut archive = zip::ZipArchive::new(Cursor::new(&hostile)).unwrap(); + assert_eq!( + archive.by_index(0).unwrap().external_attributes() >> 16, + mode + ); + + let error = store + .store_zip_code("safe", "$LATEST", &Bytes::from(hostile)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert_eq!(tokio::fs::read(&sentinel).await.unwrap(), b"outside"); + assert_eq!( + tokio::fs::read(old.join("fixture.txt")).await.unwrap(), + b"old" + ); + assert_eq!(store.artifacts.len(), 1); + let mut entries = tokio::fs::read_dir(&code_dir).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_some()); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_should_reject_duplicate_raw_zip_names_before_indexed_extraction() { + use std::io::{Cursor, Write}; + + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("first", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"one").unwrap(); + writer + .start_file("other", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"two").unwrap(); + let mut bytes = writer.finish().unwrap().into_inner(); + let local = bytes + .windows(4) + .rposition(|part| part == b"PK\x03\x04") + .unwrap(); + let central = bytes + .windows(4) + .rposition(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(local + 30..local + 35) + .unwrap() + .copy_from_slice(b"first"); + bytes + .get_mut(central + 46..central + 51) + .unwrap() + .copy_from_slice(b"first"); + let archive = zip::ZipArchive::new(Cursor::new(&bytes)).unwrap(); + assert_eq!( + archive.len(), + 1, + "upstream metadata deduplicates identical raw names" + ); + assert!(archive.has_duplicate_names()); + + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let error = store + .store_zip_code("duplicate", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert!(store.artifacts.is_empty()); + let mut entries = tokio::fs::read_dir(root.path()).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_should_preserve_dos_attribute_semantics_without_special_permissions() { + let mut bytes = crate::test_zip::minimal_zip(b"safe"); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(central + 4..central + 6) + .unwrap() + .copy_from_slice(&20u16.to_le_bytes()); + bytes + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(0o104_755u32 << 16).to_le_bytes()); + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let (path, _, _) = store + .store_zip_code("dos", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap(); + assert_eq!( + tokio::fs::read(path.join("fixture.txt")).await.unwrap(), + b"safe" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = tokio::fs::metadata(path.join("fixture.txt")).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o7000, 0); + } + } + #[test] fn test_should_reject_zip_that_expands_beyond_extraction_budget() { use std::io::Write as _; @@ -1954,8 +2670,8 @@ mod tests { } let tmp = tempfile::tempdir().unwrap(); - let err = - extract_zip(&buf, tmp.path(), 100).expect_err("expansion beyond budget must fail"); + let err = extract_zip(&buf, tmp.path(), 100, None) + .expect_err("expansion beyond budget must fail"); assert!( matches!(err, LambdaServiceError::InvalidZipFile { .. }), "got: {err:?}" @@ -1976,7 +2692,7 @@ mod tests { } let tmp = tempfile::tempdir().unwrap(); - extract_zip(&buf, tmp.path(), 1024).expect("within-budget zip extracts"); + extract_zip(&buf, tmp.path(), 1024, None).expect("within-budget zip extracts"); assert!(tmp.path().join("small.txt").exists()); } diff --git a/crates/rustack-lambda-core/src/storage/restore.rs b/crates/rustack-lambda-core/src/storage/restore.rs new file mode 100644 index 00000000..7589b78a --- /dev/null +++ b/crates/rustack-lambda-core/src/storage/restore.rs @@ -0,0 +1,308 @@ +//! Prepare every fallible record/artifact before the startup publication boundary. + +#[cfg(unix)] +use std::time::Duration; +use std::{ + collections::{BTreeMap, HashSet}, + iter::once, +}; + +#[cfg(unix)] +use tokio::time::Instant; + +#[cfg(unix)] +use super::restore_zip::ZipMeasurements; +use super::{ + FunctionRecord, FunctionStore, FunctionStoreSnapshot, StagedArtifact, ZipRestoreBudget, +}; +use crate::{ + artifact::MAX_ZIP_BYTES, + error::LambdaServiceError, + resolver::{FunctionName, Qualifier, resolve_function_ref}, +}; +#[cfg(unix)] +use crate::{ + artifact::{ImageArtifact, PinnedImage}, + docker::{DockerCodeArchive, DockerError, ImageLeasePort}, + function_config::validate_provided_settings, + provider::DockerArchiveProfile, +}; + +#[derive(Debug)] +pub(crate) struct PreparedImport<'a> { + store: &'a FunctionStore, + functions: BTreeMap, + artifacts: Vec>, +} +impl PreparedImport<'_> { + /// Infallible startup publication after all validation and ownership checks. + pub(crate) fn commit(self) { + self.store.functions.clear(); + for (name, record) in self.functions { + self.store.functions.insert(name, record); + } + for artifact in self.artifacts { + artifact.commit(); + } + self.store.changes.send_replace(()); + } + #[cfg(unix)] + pub(crate) fn commit_before(self, deadline: Instant) -> Result<(), LambdaServiceError> { + check_deadline(deadline)?; + self.commit(); + Ok(()) + } +} +impl FunctionStore { + pub(crate) fn check_snapshot_counts( + &self, + functions: usize, + revisions: usize, + ) -> Result<(), LambdaServiceError> { + if self.functions.len() > functions { + return Err(invalid("Image archive function count exceeds its profile")); + } + let mut count = 0usize; + for record in &self.functions { + count = count + .checked_add(1) + .and_then(|n| n.checked_add(record.versions.len())) + .ok_or_else(|| invalid("Image archive revision count overflow"))?; + if count > revisions { + return Err(invalid("Image archive revision count exceeds its profile")); + } + } + Ok(()) + } + pub(crate) fn check_snapshot_zip_bytes( + &self, + maximum: usize, + ) -> Result<(), LambdaServiceError> { + let mut remaining = maximum; + for function in &self.functions { + for row in once(&function.latest).chain(function.versions.values()) { + if let Some(bytes) = &row.zip_bytes { + if bytes.len() > MAX_ZIP_BYTES { + return Err(invalid("ZIP archive source is oversized")); + } + remaining = remaining + .checked_sub(bytes.len()) + .ok_or_else(|| invalid("ZIP archive source budget exceeded"))?; + } + } + } + Ok(()) + } + + #[cfg(unix)] + pub(crate) fn check_snapshot_zip_profile( + &self, + deadline: Instant, + ) -> Result<(), LambdaServiceError> { + self.check_snapshot_zip_bytes(DockerArchiveProfile::MAX_ZIP_DATA_BYTES)?; + let mut measurements = ZipMeasurements::default(); + for function in &self.functions { + for row in once(&function.latest).chain(function.versions.values()) { + check_deadline(deadline)?; + if let Some(bytes) = &row.zip_bytes { + measurements.charge(bytes)?; + } + } + } + check_deadline(deadline) + } + + pub(super) async fn prepare_snapshot( + &self, + snapshot: FunctionStoreSnapshot, + ) -> Result, LambdaServiceError> { + self.prepare_snapshot_inner(snapshot, None).await + } + async fn prepare_snapshot_inner( + &self, + snapshot: FunctionStoreSnapshot, + mut zip: Option, + ) -> Result, LambdaServiceError> { + if let Some(budget) = &zip { + budget.check()?; + } + self.validate_root().await?; + if let Some(budget) = &zip { + budget.check()?; + } + let mut staged = BTreeMap::new(); + let mut identities = HashSet::new(); + let mut artifacts = Vec::new(); + for function in snapshot.functions { + if !identities.insert(function.resource_id) { + return Err(invalid("Duplicate snapshot function resource identity")); + } + FunctionName::parse(&function.name)?; + let (name, qualifier) = resolve_function_ref(&function.arn)?; + if name != function.name || qualifier.is_some() { + return Err(invalid("Snapshot function ARN mismatch")); + } + for (alias, record) in &function.aliases { + Qualifier::parse(alias)?; + Qualifier::parse(&record.function_version)?; + } + let (record, allocations) = function.into_record(self, zip.as_mut()).await?; + artifacts.extend(allocations); + self.validate_record(&record)?; + if staged.insert(record.name.clone(), record).is_some() { + return Err(invalid("Duplicate snapshot function")); + } + } + Ok(PreparedImport { + store: self, + functions: staged, + artifacts, + }) + } + + #[cfg(unix)] + pub(crate) async fn prepare_docker_snapshot( + &self, + snapshot: FunctionStoreSnapshot, + images: &ImageLeasePort, + deadline: Instant, + profile: DockerArchiveProfile, + ) -> Result, LambdaServiceError> { + check_deadline(deadline)?; + let mut candidate = self + .prepare_snapshot_inner(snapshot, Some(ZipRestoreBudget::new(deadline))) + .await?; + // Validate all usable revisions and effective plans before any pin effect. + for record in candidate.functions.values() { + validate_restored_aliases(record)?; + for version in once(&record.latest).chain(record.versions.values()) { + if version.package_type == "Zip" { + let architecture = profile + .zip_architecture() + .ok_or_else(|| invalid("Docker ZIP archive runtime is unavailable"))?; + validate_provided_settings( + version.runtime.as_deref(), + version.handler.as_deref(), + )?; + if version.architectures.as_slice() != [architecture.as_str()] { + return Err(invalid("Docker ZIP archive platform mismatch")); + } + let path = version + .code_path + .as_ref() + .ok_or_else(|| invalid("Restored ZIP has no owned code"))?; + let artifact = self + .artifacts + .get(path) + .map(|artifact| artifact.value().clone()) + .ok_or_else(|| invalid("Restored ZIP artifact is missing"))?; + let conversion_deadline = Instant::now() + .checked_add(Duration::from_secs(600)) + .map_or(deadline, |limit| deadline.min(limit)); + DockerCodeArchive::prepare(artifact, architecture, conversion_deadline) + .await + .map_err(|error| match error { + DockerError::InvalidCode | DockerError::TooLarge => { + invalid("Invalid or oversized Docker ZIP archive candidate") + } + other => image_error(other), + })?; + check_deadline(deadline)?; + } else if version.deployment.state() == "Active" { + let saved = version.image.resolved().ok_or_else(|| { + invalid("Usable restored image has no immutable metadata") + })?; + saved + .metadata() + .execution + .plan(version.image_config.as_ref()) + .map_err(|_| invalid("Invalid restored image execution plan"))?; + } + } + } + let mut leases: BTreeMap, PinnedImage> = BTreeMap::new(); + for record in candidate.functions.values_mut() { + for version in once(&mut record.latest).chain(record.versions.values_mut()) { + check_deadline(deadline)?; + if version.package_type == "Zip" || version.deployment.state() != "Active" { + continue; + } + let saved = version + .image + .resolved() + .ok_or_else(|| invalid("Usable restored image has no immutable metadata"))?; + let key = serde_json::to_vec(saved) + .map_err(|_| invalid("Invalid restored image metadata"))?; + let image = if let Some(image) = leases.get(&key) { + image.clone() + } else { + let image = images + .restore(saved, version.image_config.as_ref(), deadline) + .await + .map_err(image_error)?; + leases.insert(key, image.clone()); + image + }; + if image.unpacked_size().get() != version.code_size { + return Err(invalid("Restored image storage proof mismatch")); + } + version.image = ImageArtifact::Pinned(image); + } + } + for record in candidate.functions.values() { + self.validate_record(record)?; + } + images.available().map_err(image_error)?; + check_deadline(deadline)?; + Ok(candidate) + } +} +#[cfg(unix)] +fn validate_restored_aliases(record: &FunctionRecord) -> Result<(), LambdaServiceError> { + for alias in record.aliases.values() { + let target = alias + .function_version + .parse::() + .map_err(|_| invalid("Invalid restored alias version"))?; + if !record.versions.contains_key(&target) { + return Err(invalid("Restored alias target is missing")); + } + if let Some(weights) = &alias.routing_config { + for (version, weight) in weights { + let target = version + .parse::() + .map_err(|_| invalid("Invalid restored alias routing"))?; + if !record.versions.contains_key(&target) + || !weight.is_finite() + || !(0.0..=1.0).contains(weight) + { + return Err(invalid("Invalid restored alias routing")); + } + } + } + } + Ok(()) +} + +fn invalid(message: &str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} +#[cfg(unix)] +pub(crate) fn check_deadline(deadline: Instant) -> Result<(), LambdaServiceError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|value| !value.is_zero()) + .ok_or_else(|| image_error(DockerError::Deadline))?; + if remaining > Duration::from_secs(3600) { + return Err(image_error(DockerError::Configuration)); + } + Ok(()) +} +#[cfg(unix)] +pub(crate) fn image_error(source: DockerError) -> LambdaServiceError { + LambdaServiceError::ImageRestore { + source: Box::new(source), + } +} diff --git a/crates/rustack-lambda-core/src/storage/restore_zip.rs b/crates/rustack-lambda-core/src/storage/restore_zip.rs new file mode 100644 index 00000000..a2eb91d2 --- /dev/null +++ b/crates/rustack-lambda-core/src/storage/restore_zip.rs @@ -0,0 +1,381 @@ +//! Bounded private ZIP staging under one original import deadline. + +use std::io::Cursor; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use bytes::Bytes; +use tokio::{ + task::spawn_blocking, + time::{Instant, timeout_at}, +}; +use zip::ZipArchive; + +use super::{ + FunctionStore, MAX_EXTRACTED_SIZE, StagedArtifact, ZipLayout, stage_zip_before, + validate_zip_directory, validated_zip_path, +}; +use crate::{ + artifact::{MAX_ZIP_BASE64_BYTES, MAX_ZIP_BYTES}, + error::LambdaServiceError, + provider::DockerArchiveProfile, +}; + +const EXPANDED: u64 = 512 * 1024 * 1024; +const NODES: usize = 20_000; +const SOURCES: usize = DockerArchiveProfile::MAX_ZIP_DATA_BYTES; + +fn invalid() -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: "Invalid or oversized Docker ZIP archive candidate".into(), + } +} +pub(super) fn check(deadline: Option) -> Result<(), LambdaServiceError> { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Err(LambdaServiceError::ResourceNotReady { + message: "ZIP archive import deadline exceeded".into(), + }); + } + Ok(()) +} +fn stage_error(error: LambdaServiceError) -> LambdaServiceError { + match error { + LambdaServiceError::InvalidParameter { .. } | LambdaServiceError::InvalidZipFile { .. } => { + invalid() + } + LambdaServiceError::Internal { .. } => LambdaServiceError::Internal { + message: "ZIP archive staging failed".into(), + }, + other => other, + } +} +#[derive(Debug, Clone, Copy)] +pub(super) struct Measurement { + expanded: u64, + nodes: usize, +} +fn measure(bytes: &Bytes) -> Result { + let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|_| invalid())?; + if archive.len() > 10_000 || archive.has_duplicate_names() { + return Err(invalid()); + } + let mut layout = ZipLayout::default(); + let mut expanded = 0_u64; + for n in 0..archive.len() { + let mut entry = archive.by_index(n).map_err(|_| invalid())?; + let path = validated_zip_path(&entry).map_err(stage_error)?; + layout.admit(&path, entry.is_dir()).map_err(stage_error)?; + if entry.is_dir() { + validate_zip_directory(&mut entry).map_err(stage_error)?; + } else { + expanded = expanded + .checked_add(entry.size()) + .filter(|n| *n <= MAX_EXTRACTED_SIZE) + .ok_or_else(invalid)?; + } + } + let nodes = layout.node_count().checked_add(3).ok_or_else(invalid)?; + Ok(Measurement { expanded, nodes }) +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct ZipMeasurements { + expanded: u64, + nodes: usize, + sources: usize, +} +impl Default for ZipMeasurements { + fn default() -> Self { + Self { + expanded: EXPANDED, + nodes: NODES, + sources: SOURCES, + } + } +} +impl ZipMeasurements { + pub(super) fn charge(&mut self, bytes: &Bytes) -> Result { + if bytes.len() > MAX_ZIP_BYTES { + return Err(invalid()); + } + let sources = self.sources.checked_sub(bytes.len()).ok_or_else(invalid)?; + let measured = measure(bytes)?; + let expanded = self + .expanded + .checked_sub(measured.expanded) + .ok_or_else(invalid)?; + let nodes = self.nodes.checked_sub(measured.nodes).ok_or_else(invalid)?; + self.sources = sources; + self.expanded = expanded; + self.nodes = nodes; + Ok(measured) + } +} + +#[cfg(test)] +#[derive(Debug)] +struct StageGate { + entered: tokio::sync::oneshot::Sender<()>, + release: std::sync::mpsc::Receiver<()>, +} + +#[derive(Debug)] +pub(super) struct ZipRestoreBudget { + deadline: Instant, + limits: ZipMeasurements, + #[cfg(test)] + gate: Option, +} +impl ZipRestoreBudget { + #[cfg(unix)] + pub(super) fn new(deadline: Instant) -> Self { + Self { + deadline, + limits: ZipMeasurements::default(), + #[cfg(test)] + gate: None, + } + } + pub(super) fn check(&self) -> Result<(), LambdaServiceError> { + check(Some(self.deadline)) + } + pub(super) async fn stage<'a>( + &mut self, + store: &'a FunctionStore, + encoded: String, + ) -> Result, LambdaServiceError> { + self.check()?; + if encoded.len() > MAX_ZIP_BASE64_BYTES { + return Err(invalid()); + } + timeout_at(self.deadline, store.validate_root()) + .await + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "ZIP archive import deadline exceeded".into(), + })??; + self.check()?; + let permit = store.staging.clone().try_acquire_owned().map_err(|_| { + LambdaServiceError::ResourceNotReady { + message: "ZIP staging capacity exhausted".into(), + } + })?; + let root = store.code_dir.clone(); + let deadline = self.deadline; + let mut limits = self.limits; + #[cfg(test)] + let gate = self.gate.take(); + let mut job = spawn_blocking(move || { + let retained = (encoded, permit); + check(Some(deadline))?; + let bytes = STANDARD + .decode(retained.0.as_bytes()) + .map(Bytes::from) + .map_err(|_| invalid())?; + let measured = limits.charge(&bytes)?; + check(Some(deadline))?; + let artifact = stage_zip_before(&root, bytes, measured.expanded, Some(deadline)) + .map_err(stage_error)?; + #[cfg(test)] + if let Some(gate) = gate { + let _ = gate.entered.send(()); + gate.release + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|_| invalid())?; + } + check(Some(deadline))?; + Ok::<_, LambdaServiceError>((artifact, limits)) + }); + let Ok(outcome) = timeout_at(deadline, &mut job).await else { + // Retain the original transaction until this same extraction joins. + drop(job.await); + return Err(LambdaServiceError::ResourceNotReady { + message: "ZIP archive import deadline exceeded".into(), + }); + }; + let (artifact, limits) = + outcome.map_err(|source| LambdaServiceError::ImageRestoreWorker { + source: source.into(), + })??; + // No detached worker has a store capability. Fence before registration. + self.check()?; + self.limits = limits; + store + .artifacts + .insert(artifact.code_root().to_path_buf(), artifact.clone()); + Ok(StagedArtifact::new(store, artifact)) + } +} + +#[cfg(all(test, unix))] +mod tests { + use std::{ + io::{Cursor, Write}, + path::Path, + sync::{Arc, mpsc}, + time::Duration, + }; + + use rstest::rstest; + use tokio::{ + fs, + sync::oneshot, + time::{sleep, sleep_until, timeout}, + }; + use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + + use super::*; + + fn source() -> String { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "tree/file", + SimpleFileOptions::default().compression_method(CompressionMethod::Deflated), + ) + .unwrap(); + writer.write_all(&[42_u8; 64]).unwrap(); + STANDARD.encode(writer.finish().unwrap().into_inner()) + } + async fn entries(path: &Path) -> usize { + let mut entries = fs::read_dir(path).await.unwrap(); + let mut count = 0; + while entries.next_entry().await.unwrap().is_some() { + count += 1; + } + count + } + fn held(deadline: Instant) -> (ZipRestoreBudget, oneshot::Receiver<()>, mpsc::Sender<()>) { + let (entered, observed) = oneshot::channel(); + let (release, receiver) = mpsc::channel(); + let mut budget = ZipRestoreBudget::new(deadline); + budget.gate = Some(StageGate { + entered, + release: receiver, + }); + (budget, observed, release) + } + #[rstest] + #[case::exact(64, 5, true)] + #[case::expanded(63, 5, false)] + #[case::nodes(64, 4, false)] + #[tokio::test] + async fn test_should_charge_expansion_and_wrapper_nodes_before_artifact_allocation( + #[case] expanded: u64, + #[case] nodes: usize, + #[case] accepted: bool, + ) { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let mut budget = ZipRestoreBudget::new(Instant::now() + Duration::from_secs(5)); + budget.limits.expanded = expanded; + budget.limits.nodes = nodes; + let stage = budget.stage(&store, source()).await; + assert_eq!(stage.is_ok(), accepted); + assert_eq!(entries(root.path()).await, usize::from(accepted)); + if accepted { + assert_eq!(budget.limits.expanded, 0); + assert_eq!(budget.limits.nodes, 0); + assert!(budget.stage(&store, source()).await.is_err()); + assert_eq!(entries(root.path()).await, 1); + } + drop(stage); + assert!(store.artifacts.is_empty()); + assert_eq!(entries(root.path()).await, 0); + } + #[tokio::test] + async fn test_should_charge_repeated_compressed_sources_and_preserve_first_pending_artifact() { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let encoded = source(); + let bytes = STANDARD.decode(&encoded).unwrap().len(); + let mut budget = ZipRestoreBudget::new(Instant::now() + Duration::from_secs(5)); + budget.limits.sources = bytes; + let first = budget.stage(&store, encoded.clone()).await.unwrap(); + assert!(budget.stage(&store, encoded).await.is_err()); + assert_eq!(entries(root.path()).await, 1); + assert!(first.artifact().code_root().is_absolute()); + drop(first); + assert_eq!(entries(root.path()).await, 0); + } + #[tokio::test] + async fn test_should_join_accepted_private_extraction_past_original_deadline_before_returning_error() + { + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let deadline = Instant::now() + Duration::from_secs(2); + let (mut budget, entered, release) = held(deadline); + let worker_store = store.clone(); + let job = + tokio::spawn(async move { budget.stage(&worker_store, source()).await.map(drop) }); + timeout(Duration::from_secs(3), entered) + .await + .unwrap() + .unwrap(); + sleep_until(deadline + Duration::from_millis(10)).await; + assert!(!job.is_finished()); + assert!(store.artifacts.is_empty()); + assert_eq!(store.staging.available_permits(), 3); + assert_eq!(entries(root.path()).await, 1); + release.send(()).unwrap(); + assert!(matches!( + timeout(Duration::from_secs(3), job).await.unwrap().unwrap(), + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + assert!(store.artifacts.is_empty()); + assert_eq!(store.staging.available_permits(), 4); + assert_eq!(entries(root.path()).await, 0); + } + #[tokio::test] + async fn test_should_retain_abandoned_worker_cleanup_without_late_store_registration() { + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let (mut budget, entered, release) = held(Instant::now() + Duration::from_secs(10)); + let worker_store = store.clone(); + let job = + tokio::spawn(async move { budget.stage(&worker_store, source()).await.map(drop) }); + timeout(Duration::from_secs(3), entered) + .await + .unwrap() + .unwrap(); + job.abort(); + assert!(job.await.unwrap_err().is_cancelled()); + assert_eq!(store.staging.available_permits(), 3); + assert!(store.artifacts.is_empty()); + release.send(()).unwrap(); + timeout(Duration::from_secs(3), async { + loop { + if entries(root.path()).await == 0 && store.staging.available_permits() == 4 { + break; + } + sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + assert!(store.artifacts.is_empty()); + } + #[tokio::test] + async fn test_should_probe_size_overflow_without_writing_beyond_the_charged_bytes() { + let root = tempfile::tempdir().unwrap(); + let mut bytes = STANDARD.decode(source()).unwrap(); + let central = bytes + .windows(4) + .position(|bytes| bytes == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(central + 24..central + 28) + .unwrap() + .copy_from_slice(&2_u32.to_le_bytes()); + let path = root.path().to_path_buf(); + let result = spawn_blocking(move || super::super::extract_zip(&bytes, &path, 2, None)) + .await + .unwrap(); + assert!(result.is_err()); + assert_eq!( + fs::metadata(root.path().join("tree/file")) + .await + .unwrap() + .len(), + 2 + ); + } +} diff --git a/crates/rustack-lambda-core/src/storage/snapshot_view.rs b/crates/rustack-lambda-core/src/storage/snapshot_view.rs new file mode 100644 index 00000000..923da06a --- /dev/null +++ b/crates/rustack-lambda-core/src/storage/snapshot_view.rs @@ -0,0 +1,115 @@ +//! Borrow archive projections while map entry guards protect dynamic values. + +use std::{borrow::Borrow, fmt}; + +use base64::{display::Base64Display, engine::general_purpose::STANDARD}; +use bytes::Bytes; +use dashmap::DashMap; +use serde::{ + Serialize, Serializer, + ser::{Error as _, SerializeSeq}, +}; + +use super::{ + EventSourceMappingRecord, EventSourceMappingStore, FunctionRecord, FunctionStore, LayerRecord, + LayerStore, +}; +use crate::{artifact::ImageArtifact, deployment::DeploymentState, error::LambdaServiceError}; + +type Guard = fn(&T) -> Result<(), LambdaServiceError>; +struct Records<'a, T> { + values: &'a DashMap, + guard: Guard, +} +impl Serialize for Records<'_, T> +where + T: Serialize + Send + Sync, +{ + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(self.values.len()))?; + for record in self.values { + (self.guard)(record.value()) + .map_err(|_| S::Error::custom("invalid image archive record"))?; + sequence.serialize_element(record.value())?; + } + sequence.end() + } +} +impl fmt::Debug for Records<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Records()") + } +} +#[derive(Debug, Serialize)] +pub(crate) struct FunctionSnapshotRef<'a> { + functions: Records<'a, FunctionRecord>, +} +#[derive(Debug, Serialize)] +pub(crate) struct LayerSnapshotRef<'a> { + layers: Records<'a, LayerRecord>, +} +#[derive(Debug, Serialize)] +pub(crate) struct MappingSnapshotRef<'a> { + mappings: Records<'a, EventSourceMappingRecord>, +} +impl FunctionStore { + pub(crate) fn borrow_snapshot(&self, guard: Guard) -> FunctionSnapshotRef<'_> { + FunctionSnapshotRef { + functions: Records { + values: &self.functions, + guard, + }, + } + } +} +impl LayerStore { + pub(crate) fn borrow_snapshot(&self, guard: Guard) -> LayerSnapshotRef<'_> { + LayerSnapshotRef { + layers: Records { + values: &self.layers, + guard, + }, + } + } +} +impl EventSourceMappingStore { + pub(crate) fn borrow_snapshot(&self) -> MappingSnapshotRef<'_> { + MappingSnapshotRef { + mappings: Records { + values: &self.mappings, + guard: |_| Ok(()), + }, + } + } +} +pub(super) fn image( + value: &ImageArtifact, + serializer: S, +) -> Result { + value.resolved().serialize(serializer) +} +pub(super) fn zip>>( + value: B, + serializer: S, +) -> Result { + if let Some(bytes) = value.borrow() { + serializer.collect_str(&Base64Display::new(bytes, &STANDARD)) + } else { + serializer.serialize_none() + } +} +pub(super) fn deployment( + value: &DeploymentState, + serializer: S, +) -> Result { + #[derive(Serialize)] + struct State<'a> { + state: &'a str, + deployment: &'a DeploymentState, + } + State { + state: value.state(), + deployment: value, + } + .serialize(serializer) +} diff --git a/crates/rustack-lambda-core/src/storage/zip_layout.rs b/crates/rustack-lambda-core/src/storage/zip_layout.rs new file mode 100644 index 00000000..d48b2fbd --- /dev/null +++ b/crates/rustack-lambda-core/src/storage/zip_layout.rs @@ -0,0 +1,114 @@ +//! Shared bounded filesystem-tree validation, including implicit ZIP parents. + +use std::{ + collections::{BTreeMap, HashSet}, + path::{Path, PathBuf}, +}; + +use crate::error::LambdaServiceError; + +const MAX_NODES: usize = 10_000; + +#[derive(Debug, Default)] +pub(crate) struct ZipLayout { + nodes: BTreeMap, + explicit: HashSet, +} + +impl ZipLayout { + pub(super) fn node_count(&self) -> usize { + self.nodes.len() + } + + /// Validate a member and return newly required directories, root first. + pub(crate) fn admit( + &mut self, + path: &Path, + directory: bool, + ) -> Result, LambdaServiceError> { + if path.file_name().is_none() || !self.explicit.insert(path.to_owned()) { + return Err(invalid()); + } + let mut parents: Vec<&Path> = path + .ancestors() + .skip(1) + .filter(|parent| parent.file_name().is_some()) + .collect(); + parents.reverse(); + let mut created = Vec::new(); + for parent in parents { + match self.nodes.get(parent) { + Some(false) => return Err(invalid()), + Some(true) => {} + None => { + self.insert(parent, true)?; + created.push(parent.to_owned()); + } + } + } + match self.nodes.get(path) { + Some(true) if directory => {} + Some(_) => return Err(invalid()), + None => { + self.insert(path, directory)?; + if directory { + created.push(path.to_owned()); + } + } + } + Ok(created) + } + fn insert(&mut self, path: &Path, directory: bool) -> Result<(), LambdaServiceError> { + if self.nodes.len() >= MAX_NODES { + return Err(invalid()); + } + self.nodes.insert(path.to_owned(), directory); + Ok(()) + } +} + +fn invalid() -> LambdaServiceError { + LambdaServiceError::InvalidZipFile { + message: "Invalid or oversized ZIP filesystem tree".into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_bound_the_complete_tree_including_implicit_directories() { + let mut layout = ZipLayout::default(); + assert_eq!( + layout.admit(Path::new("root/branch/file"), false).unwrap(), + vec![PathBuf::from("root"), PathBuf::from("root/branch")] + ); + assert_eq!( + layout.admit(Path::new("root/branch"), true).unwrap(), + [] as [PathBuf; 0] + ); + for n in 3..MAX_NODES { + layout.admit(Path::new(&format!("f{n}")), false).unwrap(); + } + assert_eq!(layout.nodes.len(), MAX_NODES); + assert!(layout.admit(Path::new("one-more"), false).is_err()); + } + + #[test] + fn test_should_reject_duplicate_and_parent_file_conflicts_in_either_order() { + for reverse in [false, true] { + let mut layout = ZipLayout::default(); + let (first, second) = if reverse { + ("parent/file", "parent") + } else { + ("parent", "parent/file") + }; + layout.admit(Path::new(first), false).unwrap(); + assert!(layout.admit(Path::new(second), false).is_err()); + } + let mut layout = ZipLayout::default(); + layout.admit(Path::new("directory"), true).unwrap(); + assert!(layout.admit(Path::new("directory"), true).is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs index 24ebb4ed..8b6e60b7 100644 --- a/crates/rustack-lambda-core/src/work.rs +++ b/crates/rustack-lambda-core/src/work.rs @@ -1,36 +1,90 @@ -//! Bounded, supervised invocation lifecycle. No detached Event tasks. +//! Bounded invocation admission and a single owner for execution/cleanup futures. + use std::{ + collections::HashMap, sync::{ - Arc, + Arc, OnceLock, atomic::{AtomicBool, AtomicUsize, Ordering}, }, time::Duration, }; -use dashmap::DashMap; use tokio::{ - sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore, oneshot}, - task::{AbortHandle, JoinSet}, + sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinHandle, JoinSet}, + time::{Instant, timeout}, }; +use typed_builder::TypedBuilder; use crate::{ + diagnostics::{ + ACCEPTED_LIMIT, DEFAULT_FUNCTION_LIMIT, EXECUTION_LIMIT, InvocationWorkLimits, + InvocationWorkStatus, + }, error::LambdaServiceError, - executor::{Executor, InvokeRequest, InvokeResponse}, + executor::{Executor, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::{InvocationContext, InvocationHandle, InvocationSubmission, Settlement}, }; #[derive(Debug)] pub(crate) struct WorkManager { - tasks: Mutex>, - accepting: AtomicBool, + accepting: Arc, capacity: Arc, global: Arc, - functions: DashMap>, + drain: Arc, + supervisor: OnceLock, +} + +#[derive(Debug)] +struct SupervisorHandle { + inbox: mpsc::Sender, + stop: watch::Sender, + state: watch::Receiver, + task: Mutex>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SupervisorState { + Running, + Stopped { failed: bool }, +} + +#[derive(Debug, TypedBuilder)] +struct Submission { + executor: Arc, + request: InvokeRequest, + context: InvocationContext, + cancellation: watch::Sender, + reserved: usize, + asynchronous: bool, + slot: OwnedSemaphorePermit, + admitted: oneshot::Sender>, + response: oneshot::Sender>, + settlement: watch::Sender, +} + +#[derive(Debug, TypedBuilder)] +struct Supervisor { + inbox: mpsc::Receiver, + stop: watch::Receiver, + drain: Arc, + accepting: Arc, + global: Arc, + state: watch::Sender, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + cancellations: HashMap>, + #[builder(default)] + functions: HashMap>, } + #[derive(Debug, Default)] struct FunctionCapacity { active: AtomicUsize, changed: Notify, } + #[derive(Debug)] struct FunctionPermit(Arc); impl Drop for FunctionPermit { @@ -39,8 +93,11 @@ impl Drop for FunctionPermit { self.0.changed.notify_waiters(); } } + +#[cfg(test)] #[derive(Debug)] -pub(crate) struct CancelOnDrop(pub(crate) AbortHandle); +pub(crate) struct CancelOnDrop(pub(crate) tokio::task::AbortHandle); +#[cfg(test)] impl Drop for CancelOnDrop { fn drop(&mut self) { self.0.abort(); @@ -52,9 +109,6 @@ fn unavailable(message: &str) -> LambdaServiceError { message: message.into(), } } -fn throttled() -> LambdaServiceError { - LambdaServiceError::TooManyRequests -} impl FunctionCapacity { fn try_acquire(self: &Arc, limit: usize) -> Result { @@ -66,9 +120,10 @@ impl FunctionCapacity { None } }) - .map_err(|_| throttled())?; + .map_err(|_| LambdaServiceError::TooManyRequests)?; Ok(FunctionPermit(Arc::clone(self))) } + async fn acquire(self: &Arc, limit: usize) -> FunctionPermit { loop { let changed = self.changed.notified(); @@ -85,260 +140,704 @@ impl FunctionCapacity { impl WorkManager { pub(crate) fn new() -> Self { Self { - tasks: Mutex::new(JoinSet::new()), - accepting: AtomicBool::new(true), - capacity: Arc::new(Semaphore::new(128)), - global: Arc::new(Semaphore::new(32)), - functions: DashMap::new(), + accepting: Arc::new(AtomicBool::new(true)), + capacity: Arc::new(Semaphore::new(ACCEPTED_LIMIT)), + global: Arc::new(Semaphore::new(EXECUTION_LIMIT)), + drain: Arc::new(Notify::new()), + supervisor: OnceLock::new(), + } + } + + pub(crate) fn status(&self) -> InvocationWorkStatus { + InvocationWorkStatus { + accepted_invocations: ACCEPTED_LIMIT.saturating_sub(self.capacity.available_permits()), + active_executions: EXECUTION_LIMIT.saturating_sub(self.global.available_permits()), + accepting: self.accepting.load(Ordering::Acquire), + limits: InvocationWorkLimits { + accepted_invocations: ACCEPTED_LIMIT, + active_executions: EXECUTION_LIMIT, + default_function_executions: DEFAULT_FUNCTION_LIMIT, + }, } } + fn supervisor(&self) -> &SupervisorHandle { + self.supervisor.get_or_init(|| { + let (inbox, receiver) = mpsc::channel(ACCEPTED_LIMIT); + let (stop, cancellation) = watch::channel(false); + let (state, snapshot) = watch::channel(SupervisorState::Running); + let supervisor = Supervisor::builder() + .inbox(receiver) + .stop(cancellation) + .drain(Arc::clone(&self.drain)) + .accepting(Arc::clone(&self.accepting)) + .global(Arc::clone(&self.global)) + .state(state) + .build(); + let task = tokio::spawn(supervisor.run()); + SupervisorHandle { + inbox, + stop, + state: snapshot, + task: Mutex::new(Some(task)), + } + }) + } + + #[cfg(test)] pub(crate) async fn submit( &self, executor: Arc, - req: InvokeRequest, + request: InvokeRequest, + reserved: Option, + asynchronous: bool, + caller_deadline: Option, + ) -> Result { + let mut submission = self.submit_owned( + executor, + request, + reserved, + asynchronous, + caller_deadline, + None, + )?; + submission.admitted().await?; + Ok(submission.into_handle()) + } + + pub(crate) fn submit_owned( + &self, + executor: Arc, + request: InvokeRequest, reserved: Option, asynchronous: bool, - ) -> Result< - ( - oneshot::Receiver>, - AbortHandle, - ), - LambdaServiceError, - > { - executor.available()?; - let limit = usize::try_from(reserved.unwrap_or(8)) - .map_err(|_| unavailable("Invalid reserved concurrency"))?; - if limit == 0 { - return Err(throttled()); + caller_deadline: Option, + external: Option>, + ) -> Result { + executor.available(&request)?; + if !self.accepting.load(Ordering::Acquire) { + return Err(unavailable("Lambda is quiescing")); + } + let reserved = match reserved { + Some(reserved) => usize::try_from(reserved) + .map_err(|_| unavailable("Invalid reserved concurrency"))?, + None => DEFAULT_FUNCTION_LIMIT, + }; + if reserved == 0 { + return Err(LambdaServiceError::TooManyRequests); } let slot = Arc::clone(&self.capacity) .try_acquire_owned() - .map_err(|_| throttled())?; - self.functions.retain(|_, capacity| { - Arc::strong_count(capacity) > 1 || capacity.active.load(Ordering::Acquire) != 0 + .map_err(|_| LambdaServiceError::TooManyRequests)?; + let total = request + .revision + .timeout + .saturating_add(Duration::from_secs(10)); + let deadline = Instant::now() + .checked_add(total) + .ok_or_else(|| unavailable("Invalid invocation budget"))?; + let deadline = caller_deadline.map_or(deadline, |caller| caller.min(deadline)); + let (cancellation, context) = + InvocationContext::channel_with_cancellation(deadline, external); + context.check()?; + let (admitted, accepted) = oneshot::channel(); + let (response, receiver) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let handle = + InvocationHandle::new(context.clone(), cancellation.clone(), receiver, settled); + let submission = Submission::builder() + .executor(executor) + .request(request) + .context(context.clone()) + .cancellation(cancellation) + .reserved(reserved) + .asynchronous(asynchronous) + .slot(slot) + .admitted(admitted) + .response(response) + .settlement(settlement) + .build(); + self.supervisor() + .inbox + .try_send(submission) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => LambdaServiceError::TooManyRequests, + mpsc::error::TrySendError::Closed(_) => unavailable("Lambda supervisor stopped"), + })?; + Ok(InvocationSubmission::new(handle, accepted)) + } + + pub(crate) fn cancel_all(&self) { + self.accepting.store(false, Ordering::Release); + if let Some(supervisor) = self.supervisor.get() { + supervisor.stop.send_replace(true); + } + self.drain.notify_one(); + } + + pub(crate) async fn quiesce(&self, budget: Duration) -> Result<(), LambdaServiceError> { + self.accepting.store(false, Ordering::Release); + self.drain.notify_one(); + let Some(supervisor) = self.supervisor.get() else { + return Ok(()); + }; + let mut state = supervisor.state.clone(); + let clean = { + let result = timeout( + budget, + state.wait_for(|state| matches!(state, SupervisorState::Stopped { .. })), + ) + .await; + match result { + Ok(Ok(state)) => matches!(*state, SupervisorState::Stopped { failed: false }), + _ => false, + } + }; + if !clean { + supervisor.stop.send_replace(true); + return Err(unavailable( + "Lambda quiesce did not confirm all invocation cleanup", + )); + } + let mut task = supervisor.task.lock().await; + if let Some(task) = task.as_mut() { + task.await + .map_err(|_| unavailable("Lambda supervisor failed"))?; + } + task.take(); + Ok(()) + } +} + +impl Drop for WorkManager { + fn drop(&mut self) { + // Fallback only: the detached actor keeps cleanup futures alive until settlement. + // Normal provider shutdown awaits quiescence before dropping this owner. + self.cancel_all(); + } +} + +impl Supervisor { + fn cancel_all(&self) { + for cancellation in self.cancellations.values() { + cancellation.send_replace(true); + } + } + + fn admit(&mut self, submission: Submission) { + let result = if self.accepting.load(Ordering::Acquire) { + submission.context.check().map_err(LambdaServiceError::from) + } else { + Err(unavailable("Lambda is quiescing")) + }; + if let Err(error) = result { + reject(submission, error); + return; + } + self.functions.retain(|_, function| { + Arc::strong_count(function) > 1 || function.active.load(Ordering::Acquire) != 0 }); let function = Arc::clone( self.functions - .entry(req.function_name.clone()) - .or_insert_with(|| Arc::new(FunctionCapacity::default())) - .value(), + .entry(submission.request.revision.function_name.clone()) + .or_default(), ); - let permits = if asynchronous { + let permits = if submission.asynchronous { None } else { - Some(( - function.try_acquire(limit)?, - Arc::clone(&self.global) - .try_acquire_owned() - .map_err(|_| throttled())?, - )) - }; - let mut tasks = self.tasks.lock().await; - while let Some(result) = tasks.try_join_next() { - if let Err(error) = result { - tracing::error!(%error, "Lambda worker failed"); + match function + .try_acquire(submission.reserved) + .and_then(|function| { + Arc::clone(&self.global) + .try_acquire_owned() + .map(|global| (function, global)) + .map_err(|_| LambdaServiceError::TooManyRequests) + }) { + Ok(permits) => Some(permits), + Err(error) => { + reject(submission, error); + return; + } } - } - if !self.accepting.load(Ordering::Acquire) { - return Err(unavailable("Lambda is quiescing")); - } - let (tx, rx) = oneshot::channel(); + }; + let Submission { + executor, + request, + context, + cancellation, + reserved, + slot, + admitted, + response, + settlement, + .. + } = submission; let global = Arc::clone(&self.global); - let handle = tasks.spawn(async move { - let _slot = slot; - let result = execute(executor, req, function, global, limit, permits).await; - match &result { - Ok(_) => tracing::debug!("Lambda invocation completed"), - Err(error) => tracing::warn!(%error, "Lambda invocation failed"), - } - if tx.send(result).is_err() { - tracing::debug!("Lambda invocation completed without a waiting caller"); - } + let worker = self.jobs.spawn(async move { + let result = execute( + executor, request, context, function, global, reserved, permits, + ) + .await; + let clean = !matches!(&result, Err(ExecutorError::CleanupFailed(_))); + // Backend cleanup completes before permits and the admission slot are released. + drop(slot); + settlement.send_replace(if clean { + Settlement::Clean + } else { + Settlement::Uncertain + }); + let _ = response.send(result); + clean }); - Ok((rx, handle)) + self.cancellations.insert(worker.id(), cancellation); + let _ = admitted.send(Ok(())); } - pub(crate) async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { - self.accepting.store(false, Ordering::Release); - // Own the JoinSet across awaits so cancellation of quiesce drops/aborts every task. - let mut tasks = std::mem::take(&mut *self.tasks.lock().await); - let drained = tokio::time::timeout(timeout, async { - while let Some(result) = tasks.join_next().await { - if let Err(error) = result { - return Err(unavailable(&format!("Lambda task failed: {error}"))); - } + async fn run(mut self) { + let mut failed = false; + loop { + if !self.accepting.load(Ordering::Acquire) || *self.stop.borrow() { + self.inbox.close(); } - // Includes workers aborted by a previously cancelled quiesce future. - let _all_capacity = Arc::clone(&self.capacity) - .acquire_many_owned(128) - .await - .map_err(|_| unavailable("Lambda accepted-work capacity closed"))?; - Ok(()) - }) - .await; - match drained { - Ok(Ok(())) => Ok(()), - result => { - tasks.abort_all(); - while let Some(result) = tasks.join_next().await { - if let Err(error) = result { - tracing::warn!(%error, "Lambda task cancelled during quiesce"); + if *self.stop.borrow() { + self.cancel_all(); + } + if self.inbox.is_closed() && self.inbox.is_empty() && self.jobs.is_empty() { + break; + } + tokio::select! { + biased; + _ = self.stop.changed(), if !*self.stop.borrow() => { self.accepting.store(false, Ordering::Release); } + () = self.drain.notified() => {} + completed = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => { + match completed { + Some(Ok((id, clean))) => { + self.cancellations.remove(&id); + if !clean { failed = true; } + } + Some(Err(error)) => { + self.cancellations.remove(&error.id()); + failed = true; + tracing::error!("Lambda invocation worker terminated without confirmed settlement"); + } + None => {} } + if failed { self.accepting.store(false, Ordering::Release); self.cancel_all(); } } - match result { - Ok(Err(error)) => Err(error), - _ => Err(unavailable( - "Lambda quiesce deadline exceeded; remaining invocations cancelled", - )), + submission = self.inbox.recv(), if !self.inbox.is_closed() || !self.inbox.is_empty() => { + if let Some(submission) = submission { self.admit(submission); } } } } + self.accepting.store(false, Ordering::Release); + self.state.send_replace(SupervisorState::Stopped { failed }); } } +fn reject(submission: Submission, error: LambdaServiceError) { + let Submission { + slot, + admitted, + response, + settlement, + .. + } = submission; + drop(slot); + settlement.send_replace(Settlement::Clean); + let _ = response.send(Err(ExecutorError::Cancelled)); + let _ = admitted.send(Err(error)); +} + async fn execute( executor: Arc, - req: InvokeRequest, + request: InvokeRequest, + context: InvocationContext, function: Arc, global: Arc, limit: usize, permits: Option<(FunctionPermit, OwnedSemaphorePermit)>, -) -> Result { - let (_function, _global) = match permits { - Some(permits) => permits, - None => ( - function.acquire(limit).await, - global - .acquire_owned() - .await - .map_err(|_| unavailable("Lambda execution closed"))?, - ), +) -> Result { + let acquired = async { + match permits { + Some(permits) => Ok::<_, ExecutorError>(permits), + None => Ok(( + function.acquire(limit).await, + global + .acquire_owned() + .await + .map_err(|_| ExecutorError::Cancelled)?, + )), + } }; - let deadline = req.timeout.saturating_add(Duration::from_secs(10)); - match tokio::time::timeout(deadline, executor.invoke(req)).await { - Ok(result) => result.map_err(LambdaServiceError::from), - Err(_) => Err(unavailable("Lambda invocation deadline exceeded")), - } + let (_function, _global) = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = acquired => result?, + }; + context.check()?; + executor.invoke(request, context).await } #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::sync::atomic::AtomicBool; use async_trait::async_trait; - use bytes::Bytes; + use tokio::sync::Semaphore; use super::*; - use crate::executor::{ExecutorError, PackageType}; #[derive(Debug)] - struct WaitingExecutor; + struct ProbeExecutor { + started: Semaphore, + cancelling: Semaphore, + cleanup: Semaphore, + hold_cleanup: AtomicBool, + calls: AtomicUsize, + } + impl ProbeExecutor { + fn new(hold_cleanup: bool) -> Self { + Self { + started: Semaphore::new(0), + cancelling: Semaphore::new(0), + cleanup: Semaphore::new(0), + hold_cleanup: AtomicBool::new(hold_cleanup), + calls: AtomicUsize::new(0), + } + } + async fn started(&self, count: u32) { + timeout(Duration::from_secs(2), self.started.acquire_many(count)) + .await + .unwrap() + .unwrap() + .forget(); + } + async fn cancelling(&self) { + timeout(Duration::from_secs(2), self.cancelling.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + } + } #[async_trait] - impl Executor for WaitingExecutor { - async fn invoke(&self, _request: InvokeRequest) -> Result { - std::future::pending().await + impl Executor for ProbeExecutor { + async fn invoke( + &self, + _request: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.calls.fetch_add(1, Ordering::AcqRel); + self.started.add_permits(1); + let reason = context.cancelled().await; + self.cancelling.add_permits(1); + if self.hold_cleanup.load(Ordering::Acquire) { + self.cleanup + .acquire() + .await + .map_err(|_| ExecutorError::CleanupFailed("Probe cleanup closed".into()))? + .forget(); + } + Err(reason.error()) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) } - async fn shutdown(&self) {} } fn request(name: &str) -> InvokeRequest { - InvokeRequest { - function_name: name.into(), - function_arn: "arn".into(), - qualifier: "$LATEST".into(), - runtime: None, - handler: None, - architectures: vec![], - package_type: PackageType::Zip, - code_root: None, - code_zip: None, - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_mins(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, + let mut request = InvokeRequest::fixture(); + let revision = Arc::make_mut(&mut request.revision); + revision.function_name = name.into(); + revision.timeout = Duration::from_secs(60); + request + } + + #[tokio::test] + async fn test_should_observe_actual_accepted_active_and_retained_cleanup_without_admission() { + let work = WorkManager::new(); + let idle = work.status(); + assert!(work.supervisor.get().is_none()); + assert_eq!( + ( + idle.accepted_invocations, + idle.active_executions, + idle.accepting + ), + (0, 0, true) + ); + let serialized = serde_json::to_value(idle).unwrap(); + assert_eq!( + serialized, + serde_json::json!({"acceptedInvocations":0,"activeExecutions":0,"accepting":true, + "limits":{"acceptedInvocations":128,"activeExecutions":32,"defaultFunctionExecutions":8}}) + ); + let executor = Arc::new(ProbeExecutor::new(true)); + for _ in 0..3 { + work.submit(executor.clone(), request("one"), Some(1), true, None) + .await + .unwrap() + .detach(); } + executor.started(1).await; + let active = work.status(); + assert_eq!( + (active.accepted_invocations, active.active_executions), + (3, 1) + ); + work.cancel_all(); + executor.cancelling().await; + timeout(Duration::from_secs(2), async { + while work.status().accepted_invocations != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let retained = work.status(); + assert_eq!( + ( + retained.accepted_invocations, + retained.active_executions, + retained.accepting + ), + (1, 1, false) + ); + executor.cleanup.add_permits(1); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + let settled = work.status(); + assert_eq!( + ( + settled.accepted_invocations, + settled.active_executions, + settled.accepting + ), + (0, 0, false) + ); } + #[tokio::test] - async fn test_should_bound_event_set_before_accepting_and_release_on_cancel() { + async fn test_should_bound_event_admission_and_cancel_queued_work_without_starting_it() { let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); for _ in 0..128 { - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor.clone(), request("bounded"), None, true, None) .await - .unwrap(); + .unwrap() + .detach(); } + executor.started(8).await; + assert_eq!( + ( + work.status().accepted_invocations, + work.status().active_executions + ), + (128, 8) + ); + assert_eq!(executor.calls.load(Ordering::Acquire), 8); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor.clone(), request("bounded"), None, true, None) .await, Err(LambdaServiceError::TooManyRequests) )); - tokio::task::yield_now().await; - assert!( - work.functions - .get("bounded") - .unwrap() - .active - .load(Ordering::Acquire) - <= 8 - ); - assert!(work.global.available_permits() >= 24); - assert!(work.quiesce(Duration::ZERO).await.is_err()); + work.cancel_all(); + work.quiesce(Duration::from_secs(2)).await.unwrap(); assert_eq!(work.capacity.available_permits(), 128); assert_eq!(work.global.available_permits(), 32); - assert_eq!( - work.functions - .get("bounded") - .unwrap() - .active - .load(Ordering::Acquire), - 0 - ); + assert_eq!(executor.calls.load(Ordering::Acquire), 8); assert!( - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor, request("bounded"), None, true, None) .await .is_err() ); } + #[tokio::test] - async fn test_should_enforce_function_global_and_zero_reserved_execution_limits() { + async fn test_should_enforce_shared_function_global_and_zero_reserved_execution_limits() { let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("zero"), Some(0), true) + work.submit(executor.clone(), request("zero"), Some(0), true, None) .await, Err(LambdaServiceError::TooManyRequests) )); + let mut handles = Vec::new(); for name in ["a", "b", "c", "d"] { for _ in 0..8 { - work.submit(Arc::new(WaitingExecutor), request(name), None, false) - .await - .unwrap(); + handles.push( + work.submit(executor.clone(), request(name), None, false, None) + .await + .unwrap(), + ); } assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request(name), None, false) + work.submit(executor.clone(), request(name), None, false, None) .await, Err(LambdaServiceError::TooManyRequests) )); } + assert_eq!(work.global.available_permits(), 0); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("e"), Some(32), false) + work.submit(executor, request("e"), Some(32), false, None) .await, Err(LambdaServiceError::TooManyRequests) )); - assert_eq!(work.global.available_permits(), 0); - assert!(work.quiesce(Duration::ZERO).await.is_err()); + drop(handles); + work.quiesce(Duration::from_secs(2)).await.unwrap(); assert_eq!(work.global.available_permits(), 32); } + #[tokio::test] - async fn test_should_release_capacity_when_sync_caller_cancels() { + async fn test_should_hold_permits_until_cancelled_backend_cleanup_is_confirmed() { let work = WorkManager::new(); - let (_response, handle) = work - .submit(Arc::new(WaitingExecutor), request("cancel"), None, false) + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit(executor.clone(), request("cleanup"), Some(1), false, None) + .await + .unwrap(); + executor.started(1).await; + handle.cancel(); + executor.cancelling().await; + assert_eq!(work.capacity.available_permits(), 127); + assert_eq!(work.global.available_permits(), 31); + assert!(matches!( + work.submit(executor.clone(), request("cleanup"), Some(1), false, None) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + executor.cleanup.add_permits(1); + handle + .cancel_and_wait(Instant::now() + Duration::from_secs(2)) .await .unwrap(); - drop(CancelOnDrop(handle)); - assert!(work.quiesce(Duration::from_secs(1)).await.is_err()); assert_eq!(work.capacity.available_permits(), 128); assert_eq!(work.global.available_permits(), 32); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + } + + #[tokio::test] + async fn test_should_keep_cleanup_owned_after_handle_deadline_and_aborted_quiesce_waiter() { + let work = Arc::new(WorkManager::new()); + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit(executor.clone(), request("retained"), None, false, None) + .await + .unwrap(); + executor.started(1).await; + assert!(matches!( + handle.cancel_and_wait(Instant::now()).await, + Err(ExecutorError::CleanupFailed(_)) + )); + executor.cancelling().await; + let waiting = Arc::clone(&work); + let task = tokio::spawn(async move { waiting.quiesce(Duration::from_secs(30)).await }); + tokio::task::yield_now().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(work.capacity.available_permits(), 127); + executor.cleanup.add_permits(1); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[tokio::test] + async fn test_should_cancel_on_handle_drop_and_reject_already_expired_admission() { + let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); + assert!(matches!( + work.submit( + executor.clone(), + request("expired"), + None, + false, + Some(Instant::now()) + ) + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + let handle = work + .submit(executor.clone(), request("drop"), None, false, None) + .await + .unwrap(); + executor.started(1).await; + drop(handle); + executor.cancelling().await; + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[tokio::test(start_paused = true)] + async fn test_should_return_at_caller_deadline_while_retaining_backend_cleanup_capacity() { + let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit( + executor.clone(), + request("deadline"), + None, + false, + Some(Instant::now() + Duration::from_secs(1)), + ) + .await + .unwrap(); + executor.started(1).await; + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!( + handle.wait().await, + Err(ExecutorError::DeadlineExceeded) + )); + executor.cancelling().await; + assert_eq!(work.global.available_permits(), 31); + assert_eq!(work.capacity.available_permits(), 127); + assert!(work.quiesce(Duration::ZERO).await.is_err()); + executor.cleanup.add_permits(1); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.global.available_permits(), 32); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[derive(Debug)] + struct PanickingExecutor; + #[async_trait] + impl Executor for PanickingExecutor { + async fn invoke( + &self, + _request: InvokeRequest, + _context: InvocationContext, + ) -> Result { + panic!("intentional supervised panic"); + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } + } + #[tokio::test] + async fn test_should_never_report_successful_quiescence_after_a_worker_panic() { + let work = WorkManager::new(); + let handle = work + .submit( + Arc::new(PanickingExecutor), + request("panic"), + None, + false, + None, + ) + .await + .unwrap(); + assert!(matches!( + handle.wait().await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(work.quiesce(Duration::from_secs(2)).await.is_err()); + assert!( + work.submit( + Arc::new(ProbeExecutor::new(false)), + request("after-panic"), + None, + false, + None + ) + .await + .is_err() + ); } } diff --git a/crates/rustack-lambda-core/tests/common/docker.rs b/crates/rustack-lambda-core/tests/common/docker.rs new file mode 100644 index 00000000..579e0b42 --- /dev/null +++ b/crates/rustack-lambda-core/tests/common/docker.rs @@ -0,0 +1,133 @@ +//! Shared explicit-daemon fixture. Readiness/probes are not implied by setup. +use std::{env, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::{ + artifact::{Architecture, PinnedImage}, + docker::{ + DirectoryLease, DockerPort, DockerWire, EnvironmentConfig, EnvironmentManager, + EnvironmentSettings, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OwnerFilter, + }, +}; +use tempfile::TempDir; +use tokio::time::Instant; +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +pub(crate) struct Fixture { + pub(crate) owner: EnvironmentManager, + pub(crate) images: ImageLeaseManager, + pub(crate) wire: DockerWire, + pub(crate) docker: DockerPort, + pub(crate) lease: DirectoryLease, + pub(crate) relay: Option, + pub(crate) function: Option, + pub(crate) directory: Option, +} +impl Fixture { + pub(crate) async fn new() -> Result { + let socket = + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket required")?; + let wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let directory = tempfile::tempdir()?; + let lease = DirectoryLease::acquire(directory.path().into(), deadline()).await?; + let docker = wire.port().bind_directory(&lease, deadline()).await?; + let images = ImageLeaseManager::start( + docker.clone(), + ImagePinScope::new(lease.clone()), + ImageLeaseConfig::default(), + )?; + let arch = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let config = EnvironmentConfig::try_from( + EnvironmentSettings::builder() + .environments(1) + .memory_bytes(256 * 1024 * 1024) + .io(Duration::from_secs(60)) + .start(Duration::from_secs(60)) + .build(), + )?; + let owner = EnvironmentManager::start(docker.clone(), lease.clone(), config)?; + let mut fixture = Self { + owner, + images, + wire, + docker, + lease, + relay: None, + function: None, + directory: Some(directory), + }; + if let Err(error) = fixture.prepare(arch).await { + let cleanup = fixture.cleanup().await; + return Err(error.context(format!( + "setup cleanup: {}", + if cleanup.is_ok() { + "confirmed" + } else { + "unresolved" + } + ))); + } + Ok(fixture) + } + async fn prepare(&mut self, arch: Architecture) -> Result<()> { + let relay_reference = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let relay = self + .images + .port() + .prepare(&relay_reference, arch, None, deadline()) + .await?; + self.relay = Some(relay); + let suffix = if arch == Architecture::Arm64 { + "arm64" + } else { + "amd64" + }; + let function = self + .images + .port() + .prepare( + &format!("rustack-p0-handler:{suffix}").parse()?, + arch, + None, + deadline(), + ) + .await?; + self.function = Some(function); + Ok(()) + } + + pub(crate) async fn cleanup(&mut self) -> Result<()> { + let result = async { + self.owner.shutdown(deadline()).await?; + ensure!( + self.owner.port().status().allocated == 0, + "allocation reservation remains" + ); + let containers = self + .docker + .list_containers( + OwnerFilter::new(self.lease.owner_id(), Some(self.lease.run_id()), None)?, + deadline(), + ) + .await?; + ensure!(containers.is_empty(), "owned containers remain"); + self.relay.take(); + self.function.take(); + self.images.shutdown(deadline()).await?; + self.wire.shutdown().await?; + Ok(()) + } + .await; + if result.is_err() { + if let Some(directory) = self.directory.take() { + let _ = directory.keep(); + } + } + result + } +} diff --git a/crates/rustack-lambda-core/tests/common/mod.rs b/crates/rustack-lambda-core/tests/common/mod.rs new file mode 100644 index 00000000..c4fdd744 --- /dev/null +++ b/crates/rustack-lambda-core/tests/common/mod.rs @@ -0,0 +1 @@ +pub(crate) mod docker; diff --git a/crates/rustack-lambda-core/tests/docker_endpoint.rs b/crates/rustack-lambda-core/tests/docker_endpoint.rs new file mode 100644 index 00000000..5f99e2ef --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_endpoint.rs @@ -0,0 +1,235 @@ +//! Explicit container-origin endpoint qualification against a controlled gateway peer. +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{ + env, + net::{IpAddr, SocketAddr}, + sync::atomic::Ordering, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use rstest::rstest; +use rustack_lambda_core::{ + artifact::ImageReference, + docker::{ + ContainerPolicy, DockerWire, EndpointProbeConfig, EnvironmentProbeConfig, + EnvironmentProbeError, GatewayInstanceId, NetworkSelection, ProbeFailure, ProbeSettings, + RelayBudgets, probe_endpoint, probe_environment, + }, +}; +use tokio::{ + net::TcpListener, + time::{Instant, sleep, timeout}, +}; + +mod common; +#[path = "docker_endpoint/gateway.rs"] +mod gateway; +use common::docker::Fixture; +use gateway::Gateway; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +#[rstest] +#[case::matching(0)] +#[case::wrong_instance(1)] +#[case::unavailable_listener(2)] +#[case::abandoned(3)] +#[tokio::test] +#[ignore = "requires explicit Docker socket, pinned runtime, operator bridge and function-visible \ + host"] +async fn test_should_qualify_actual_container_endpoint_and_retain_physical_retirement( + #[case] scenario: u8, +) -> Result<()> { + let host: IpAddr = env::var("RUSTACK_RUNTIME_GATEWAY_HOST") + .context("explicit function-visible host IP required")? + .parse()?; + let network: NetworkSelection = env::var("RUSTACK_RUNTIME_NETWORK") + .context("explicit operator bridge required")? + .parse()?; + let source_reference: ImageReference = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let mut fixture = Fixture::new().await?; + let mut source_id = None; + let evidence = fixture + .directory + .as_ref() + .map(|root| root.path().to_owned()); + let mut gateway = None; + let result: Result<()> = async { + source_id = Some( + fixture + .docker + .inspect_image(&source_reference, deadline()) + .await? + .id + .context("source identity")?, + ); + let network_proof = network.qualify(&fixture.docker, deadline()).await?; + let policy = ContainerPolicy::new( + network, + 1000, + 1000, + 333_333_333, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?; + let base = EnvironmentProbeConfig::builder() + .directory(fixture.lease.clone()) + .relay(fixture.relay.as_ref().context("pinned runtime")?.clone()) + .environments(fixture.owner.port()) + .policy(policy) + .network(network_proof) + .settings(ProbeSettings::new( + Duration::from_secs(60), + Duration::from_secs(60), + )?) + .build(); + let resources = probe_environment(base.clone(), deadline()).await?; + ensure!(resources.matches(&base), "resource evidence stale"); + ensure!( + fixture.owner.port().status().allocated == 0, + "resource proof before retirement" + ); + let (port, expected) = if scenario == 2 { + let reserved = TcpListener::bind("0.0.0.0:0").await?; + let port = reserved.local_addr()?.port(); + drop(reserved); + (port, GatewayInstanceId::generate_owned(deadline()).await?) + } else { + gateway = Some(Gateway::start(scenario == 3).await?); + let gateway = gateway.as_ref().context("gateway peer")?; + ( + gateway.address.port(), + if scenario == 1 { + GatewayInstanceId::generate_owned(deadline()).await? + } else { + gateway.instance + }, + ) + }; + let endpoint = format!("http://{}", SocketAddr::new(host, port)).parse()?; + let config = EndpointProbeConfig::new(base.clone(), resources.clone(), endpoint, expected)?; + if scenario == 3 { + let job = tokio::spawn(probe_endpoint(config, deadline())); + let observation: Result<()> = async { + gateway + .as_mut() + .context("held gateway")? + .observed(deadline()) + .await?; + ensure!( + fixture.owner.port().status().allocated == 1, + "no running endpoint environment" + ); + Ok(()) + } + .await; + job.abort(); + let cancelled = job.await.is_err_and(|error| error.is_cancelled()); + observation?; + ensure!(cancelled, "probe caller was not cancelled"); + fixture.owner.shutdown(deadline()).await?; + timeout(Duration::from_secs(10), async { + while !gateway + .as_ref() + .is_some_and(|gateway| gateway.closed.load(Ordering::Acquire)) + { + sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("abandoned HTTP IO not closed")?; + } else { + let observed = probe_endpoint(config.clone(), deadline()).await; + if scenario == 0 { + let proof = observed?; + ensure!( + proof.matches(&config) && proof.report().instance_id() == expected, + "endpoint receipt differs" + ); + let changed = EndpointProbeConfig::new( + base, + resources, + format!("http://{}", SocketAddr::new(host, port)).parse()?, + GatewayInstanceId::generate_owned(deadline()).await?, + )?; + ensure!( + !proof.matches(&changed), + "proof matched another gateway instance" + ); + } else { + ensure!( + matches!( + observed, + Err(EnvironmentProbeError::Rejected(ProbeFailure::Outcome)) + ), + "unavailable/wrong gateway did not produce a consumed probe failure" + ); + } + } + ensure!( + fixture.owner.port().status().allocated == 0, + "endpoint proof/failure before physical retirement" + ); + if let Some(gateway) = &gateway { + ensure!( + gateway.requests.load(Ordering::Acquire) == 1, + "endpoint probe replayed" + ); + } + Ok(()) + } + .await; + let cleanup = fixture.cleanup().await; + let source_audit: Result<()> = if cleanup.is_ok() && source_id.is_some() { + async { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET")?; + let mut wire = + DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let result: Result<()> = async { + let port = wire + .port() + .bind_directory(&fixture.lease, deadline()) + .await?; + ensure!( + port.inspect_image(&source_reference, deadline()) + .await? + .id + .as_deref() + == source_id.as_deref(), + "source image changed during endpoint cleanup" + ); + Ok(()) + } + .await; + let closed = wire.shutdown().await; + result?; + closed?; + Ok(()) + } + .await + } else { + Ok(()) + }; + let peer_cleanup = match &mut gateway { + Some(gateway) => gateway.shutdown().await, + None => Ok(()), + }; + if result.is_err() || cleanup.is_err() || peer_cleanup.is_err() || source_audit.is_err() { + if let Some(root) = fixture.directory.take() { + eprintln!("retained endpoint evidence: {}", root.keep().display()); + } else if let Some(root) = evidence { + eprintln!("retained endpoint evidence: {}", root.display()); + } + } + cleanup?; + peer_cleanup?; + source_audit?; + result +} diff --git a/crates/rustack-lambda-core/tests/docker_endpoint/gateway.rs b/crates/rustack-lambda-core/tests/docker_endpoint/gateway.rs new file mode 100644 index 00000000..b91c9cf0 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_endpoint/gateway.rs @@ -0,0 +1,154 @@ +//! Controlled gateway protocol peer; actual binary route has separate socket tests. + +use std::{ + io, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::{GatewayInstanceId, GatewayReply}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +#[derive(Debug, TypedBuilder)] +pub(super) struct Gateway { + pub address: SocketAddr, + pub instance: GatewayInstanceId, + seen: watch::Receiver, + pub requests: Arc, + pub closed: Arc, + stop: watch::Sender, + task: Option>>, + settled: Option>>, +} +impl Gateway { + pub async fn start(hold: bool) -> Result { + let listener = TcpListener::bind("0.0.0.0:0").await?; + let address = listener.local_addr()?; + let instance = + GatewayInstanceId::generate_owned(Instant::now() + Duration::from_secs(5)).await?; + let (stop, mut stopped) = watch::channel(false); + let (observed, seen) = watch::channel(0); + let requests = Arc::new(AtomicUsize::new(0)); + let counter = requests.clone(); + let closed = Arc::new(AtomicBool::new(false)); + let eof = closed.clone(); + let bytes = serde_json::to_vec(&GatewayReply::new(instance))?; + let task = tokio::spawn(async move { + let work = async { + loop { + let (mut stream, _) = listener.accept().await?; + let head = + read_head(&mut stream, Instant::now() + Duration::from_secs(10)).await?; + ensure!( + head.starts_with(b"GET /_rustack/runtime-probe HTTP/1.1\r\n"), + "wrong endpoint request" + ); + let raw: String = instance.into(); + ensure!( + !String::from_utf8_lossy(&head).contains(&raw), + "expected identity leaked into GET" + ); + let count = counter.fetch_add(1, Ordering::AcqRel).saturating_add(1); + ensure!(count <= 2, "probe retried excessively"); + observed.send_replace(count); + if hold { + let mut byte = [0]; + let result = timeout_at( + Instant::now() + Duration::from_secs(60), + stream.read(&mut byte), + ) + .await?; + ensure!( + matches!(result, Ok(0)) + || result.is_err_and(|error| matches!( + error.kind(), + io::ErrorKind::ConnectionReset | io::ErrorKind::BrokenPipe + )), + "abandoned connection still transmitted data" + ); + eof.store(true, Ordering::Release); + } else { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nCache-Control: \ + no-store\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + stream.write_all(head.as_bytes()).await?; + stream.write_all(&bytes).await?; + stream.shutdown().await?; + } + } + }; + tokio::select! {biased;_ = stopped.changed()=>Ok(()),result=work=>result,} + }); + Ok(Self::builder() + .address(address) + .instance(instance) + .seen(seen) + .requests(requests) + .closed(closed) + .stop(stop) + .task(Some(task)) + .settled(None) + .build()) + } + pub async fn observed(&mut self, deadline: Instant) -> Result<()> { + while *self.seen.borrow() == 0 { + timeout_at(deadline, self.seen.changed()) + .await + .context("gateway request observer expired")??; + } + Ok(()) + } + pub async fn shutdown(&mut self) -> Result<()> { + self.stop.send_replace(true); + if self.settled.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(Instant::now() + Duration::from_secs(10), task) + .await + .context("gateway join deadline")? + .context("gateway worker failed") + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settled = Some(result.map_err(Arc::new)); + } + match &self.settled { + Some(Ok(())) => Ok(()), + Some(Err(error)) => Err(anyhow::anyhow!("gateway cleanup failed: {error:#}")), + None => anyhow::bail!("gateway cleanup unsettled"), + } + } +} +impl Drop for Gateway { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} +async fn read_head(stream: &mut TcpStream, deadline: Instant) -> Result> { + timeout_at(deadline, async { + let mut bytes = Vec::new(); + loop { + ensure!(bytes.len() < 4096, "oversized fixture request"); + bytes.push(stream.read_u8().await?); + if bytes.ends_with(b"\r\n\r\n") { + return Ok(bytes); + } + } + }) + .await + .context("fixture request deadline")? +} diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs new file mode 100644 index 00000000..500398aa --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -0,0 +1,445 @@ +//! Real container lifecycle qualification of the retained environment owner. +#![cfg(unix)] +#![forbid(unsafe_code)] +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use bytes::Bytes; +use rustack_lambda_core::{ + artifact::DeploymentArtifact, + docker::{ + ContainerPolicy, EnvironmentError, EnvironmentHandle, EnvironmentPlan, EnvironmentPool, + FunctionLogs, PoolConfig, RelayBudgets, + }, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, +}; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, HandlerTimeout, Payload}, + session::{Invocation, RuntimeResult}, +}; +use serde_json::{Value, json}; +use tokio::time::{Instant, timeout}; +use uuid::Uuid; +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +mod common; +use common::docker::Fixture; +impl Fixture { + pub(crate) fn revision(&self) -> Result> { + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("environment-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture( + self.function + .as_ref() + .context("function image")? + .resolved() + .metadata() + .architecture, + ) + .artifact(DeploymentArtifact::Image( + self.function.as_ref().context("function image")?.clone(), + )) + .image_config(None) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ])) + .timeout(Duration::from_secs(120)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build(); + Ok(Arc::new(revision)) + } + pub(crate) fn plan(&self) -> Result { + self.plan_revision(self.revision()?) + } + pub(crate) fn plan_revision( + &self, + revision: Arc, + ) -> Result { + Ok(EnvironmentPlan::new( + self.lease.clone(), + self.relay.as_ref().context("relay image")?.clone(), + revision, + ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?, + )?) + } +} + +fn invocation(event: &Value) -> Result { + Ok(Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(120), + "arn:aws:lambda:us-east-1:000000000000:function:environment-fixture:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?) +} +async fn call(handle: &mut EnvironmentHandle, event: &Value, failed: bool) -> Result { + let response = handle.invoke(invocation(event)?, true).await?; + match response.result { + RuntimeResult::Success(payload) if !failed => Ok(serde_json::from_slice(payload.as_ref())?), + RuntimeResult::FunctionError(payload) if failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + _ => anyhow::bail!("unexpected handler result"), + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_own_real_initialization_warm_calls_function_error_and_cleanup() -> Result<()> { + let mut fixture = Fixture::new().await?; + let result=async{ + let mut handle=fixture.owner.port().spawn(fixture.plan()?,deadline())?;handle.wait_ready(deadline()).await?; + ensure!(matches!(fixture.owner.port().spawn(fixture.plan()?,deadline()),Err(EnvironmentError::Busy)),"global environment bound not enforced"); + let first=call(&mut handle,&json!({"value":35,"padding":"x".repeat(6*1024*1024-4096),"response_bytes":6*1024*1024-4096}),false).await?; + ensure!(first["computed"]==42&&first["counter"]==1,"computed result differs");ensure!(first["large"].as_str().is_some_and(|value|value.len()==6*1024*1024-4096),"large result differs"); + let second=call(&mut handle,&json!({"value":0}),false).await?;ensure!(second["computed"]==7&&second["counter"]==2,"warm result differs"); + let failure=call(&mut handle,&json!({"fail":true}),true).await?;ensure!(failure["errorType"]=="ValueError","FunctionError differs");handle.shutdown(deadline()).await?;Ok(()) + }.await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_retire_running_code_after_abandoned_call_and_join_all_owned_io() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let result = async { + let plan = fixture.plan()?; + let key = plan.function_key().clone(); + let mut handle = fixture.owner.port().spawn(plan, deadline())?; + handle.wait_ready(deadline()).await?; + let mut observed = FunctionLogs::start( + fixture.docker.attach_logs(&key, deadline()).await?, + Duration::from_secs(60), + 262_144, + )?; + let call = tokio::spawn(async move { + handle + .invoke(invocation(&json!({"value":1,"sleep":true})).unwrap(), false) + .await + }); + let observation = timeout(Duration::from_secs(30), async { + loop { + let snapshot = observed.snapshot(deadline()).await?; + if snapshot + .tail + .windows(14) + .any(|part| part == b"P0_BUSY_START=") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("handler did not start"); + call.abort(); + let joined = call.await; + let retired = fixture.owner.shutdown(deadline()).await; + let logs = observed.shutdown().await; + observation??; + ensure!( + matches!(joined,Err(ref error) if error.is_cancelled()), + "abandoned call was not cancelled" + ); + retired?; + logs?; + Ok(()) + } + .await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_pool_real_warm_calls_and_evict_capacity_across_revision_changes() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let original = fixture.revision()?; + let port = pool.port(); + for expected in 1..=2 { + let response = port + .invoke( + fixture.plan_revision(original.clone())?, + invocation(&json!({"value":35}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("unexpected function failure"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["computed"] == 42 && value["counter"] == expected, + "pool did not preserve warm counter" + ); + } + let mut changed = (*original).clone(); + changed.revision_id = Uuid::new_v4().to_string(); + changed + .environment + .insert("TEST_VALUE".into(), "changed".into()); + let changed = Arc::new(changed); + for revision in [changed, original.clone()] { + let response = port + .invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":0,"policy":true}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("unexpected function failure"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["computed"] == 7 && value["counter"] == 1, + "revision or evicted environment was reused" + ); + } + let failed = port + .invoke( + fixture.plan_revision(original.clone())?, + invocation(&json!({"fail":true}))?, + false, + ) + .await?; + let RuntimeResult::FunctionError(payload) = failed.result else { + anyhow::bail!("pool hid FunctionError"); + }; + let error: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + error["errorType"] == "ValueError", + "pool error payload differs" + ); + let response = port + .invoke( + fixture.plan_revision(original)?, + invocation(&json!({"value":0}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("replacement invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["counter"] == 1, + "failed environment was returned to warm pool" + ); + Ok(()) + } + .await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} + +#[rstest::rstest] +#[case::abandoned(false)] +#[case::awaited_settlement(true)] +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_join_pool_retirement_after_cancelling_an_observed_running_handler( + #[case] awaited: bool, +) -> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let revision = fixture.revision()?; + let plan = fixture.plan_revision(revision.clone())?; + let key = plan.function_key().clone(); + let port = pool.port(); + let initial = port + .invoke(plan, invocation(&json!({"value":0}))?, false) + .await?; + ensure!( + matches!(initial.result, RuntimeResult::Success(_)), + "warm setup failed" + ); + let plan = fixture.plan_revision(revision.clone())?; + let event = invocation(&json!({"value":1,"sleep":true}))?; + let mut observed = FunctionLogs::start( + fixture.docker.attach_logs(&key, deadline()).await?, + Duration::from_secs(60), + 262_144, + )?; + let mut call = match port.start(plan, event, false) { + Ok(call) => call, + Err(error) => { + observed.shutdown().await?; + return Err(error.into()); + } + }; + let observation = timeout(Duration::from_secs(30), async { + loop { + let snapshot = observed.snapshot(deadline()).await?; + if snapshot + .tail + .windows(14) + .any(|part| part == b"P0_BUSY_START=") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("pooled handler did not start"); + let cancellation = if awaited { + call.cancel_and_wait(deadline()).await + } else { + drop(call); + pool.shutdown(deadline()).await + }; + let logs = observed.shutdown().await; + observation??; + cancellation?; + logs?; + ensure!( + fixture.owner.port().status().allocated == 0, + "call returned before environment retirement" + ); + ensure!( + matches!( + fixture.docker.inspect_container(&key, deadline()).await, + Err(rustack_lambda_core::docker::DockerError::Status( + http::StatusCode::NOT_FOUND + )) + ), + "cancelled function still exists" + ); + if awaited { + ensure!( + port.status().accepting, + "per-call cancellation stopped the pool" + ); + let replacement = port + .invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":2}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = replacement.result else { + anyhow::bail!("replacement handler failed"); + }; + let output: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(output["counter"] == 1, "cancelled environment was reused"); + } + Ok(()) + } + .await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_enforce_captured_handler_timeout_and_replace_the_retired_pooled_environment() +-> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let mut revision = (*fixture.revision()?).clone(); + revision.timeout = Duration::from_secs(3); + let revision = Arc::new(revision); + let port = pool.port(); + let response = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":0}))?, + false, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("timeout fixture warm setup failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["counter"] == 1, "initial counter differs"); + let result = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":1,"sleep":true}))?.with_handler_timeout(HandlerTimeout::try_from(Duration::from_mins(15))?), + false, + ).await; + ensure!( + matches!(result, Err(EnvironmentError::Timeout(duration)) if duration == Duration::from_secs(3)), + "handler timeout was not preserved", + ); + ensure!(fixture.owner.port().status().allocated == 0, "timeout returned before resource settlement"); + let response = port.invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":35}))?, + false, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("timeout replacement invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["computed"] == 42 && value["counter"] == 1, "timed out environment was reused"); + Ok(()) + }.await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_isolate_requested_log_tails_across_logging_silent_and_uncaptured_warm_calls() +-> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let revision = fixture.revision()?; + let port = pool.port(); + for (capture, emit, counter) in [(true, true, 1), (false, true, 2), (true, false, 3), (true, true, 4)] { + let response = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":35,"logs":emit,"response_bytes":if emit {1024*1024} else {0}}))?, + capture, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("logging fixture invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["computed"] == 42 && value["counter"] == counter, "log windows prevented actual warm reuse"); + ensure!(response.logs.len() <= 4096, "log window exceeds response bound"); + if capture && emit { + for marker in [b"WIRE_STDOUT_CAPTURE".as_slice(), b"WIRE_STDERR_CAPTURE".as_slice()] { + ensure!(response.logs.windows(marker.len()).any(|bytes| bytes == marker), "observed invocation marker missing"); + } + } else { + ensure!(response.logs.is_empty(), "silent or uncaptured invocation inherited earlier logs"); + } + } + Ok(()) + }.await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} diff --git a/crates/rustack-lambda-core/tests/docker_executor.rs b/crates/rustack-lambda-core/tests/docker_executor.rs new file mode 100644 index 00000000..c2bf4e23 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_executor.rs @@ -0,0 +1,182 @@ +//! Real SDK/HTTP/provider execution; controlled fixture readiness is not startup evidence. +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use hyper::server::conn::http1; +use hyper_util::rt::TokioIo; +use rustack_lambda_core::{ + config::LambdaConfig, + deployment::ImageDeploymentConfig, + docker::{ContainerPolicy, PoolConfig, RelayBudgets}, + executor::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, Executor, ExecutorBackend, + }, + handler::RustackLambdaHandler, + provider::RustackLambda, + storage::FunctionStore, +}; +use rustack_lambda_http::{LambdaHttpConfig, LambdaHttpService}; +use tokio::{net::TcpListener, process::Command, sync::watch, task::JoinSet, time::timeout}; + +mod common; +use common::docker::Fixture; + +#[derive(Debug)] +struct CompositionOnlyReadiness; +impl DockerReadiness for CompositionOnlyReadiness { + fn check(&self) -> Result<(), DockerExecutorError> { + Ok(()) + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker images/socket and pinned boto3 fixture"] +async fn test_should_deploy_publish_alias_and_invoke_real_images_through_boto3_http() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let result = run(&fixture).await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} +async fn run(fixture: &Fixture) -> Result<()> { + let relay = fixture.relay.as_ref().context("relay image")?.clone(); + let executor = Arc::new(DockerExecutor::new( + DockerExecutorConfig::builder() + .directory(fixture.lease.clone()) + .relay(relay) + .policy(ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?) + .environments(fixture.owner.port()) + .readiness(Arc::new(CompositionOnlyReadiness)) + .settings(DockerExecutorSettings::new( + PoolConfig::default(), + Duration::from_secs(60), + )?) + .build(), + )?); + let result = serve(fixture, executor.clone()).await; + let retired = executor.shutdown().await; + result.and(retired.map_err(Into::into)) +} +async fn serve(fixture: &Fixture, executor: Arc) -> Result<()> { + let store = FunctionStore::new( + fixture + .directory + .as_ref() + .context("data directory")? + .path() + .join("code"), + ); + let config = LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }; + let provider = Arc::new( + RustackLambda::with_executor(store, config, executor.clone()) + .with_image_preparation(fixture.images.port(), ImageDeploymentConfig::default())?, + ); + let result = run_server(fixture, provider.clone()).await; + let quiesced = provider.quiesce(Duration::from_secs(60)).await; + let retired = executor.shutdown().await; + provider.shutdown().await; + result + .and(quiesced.map_err(Into::into)) + .and(retired.map_err(Into::into)) +} +async fn run_server(fixture: &Fixture, provider: Arc) -> Result<()> { + let image = fixture + .function + .as_ref() + .context("function image")? + .resolved() + .metadata(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let service = LambdaHttpService::new( + Arc::new(RustackLambdaHandler::new(provider)), + LambdaHttpConfig::default(), + ); + let (stop, mut stopped) = watch::channel(false); + let mut server = tokio::spawn(async move { + let mut jobs = JoinSet::new(); + let result = async { loop { + tokio::select! {biased; + _=stopped.wait_for(|stop|*stop)=>return Ok::<(),anyhow::Error>(()), + joined=jobs.join_next(),if !jobs.is_empty()=> {joined.unwrap()??;}, + stream=listener.accept(),if jobs.len()<8=> { + let (stream,_)=stream?;let service=service.clone(); + jobs.spawn(async move {http1::Builder::new().serve_connection(TokioIo::new(stream),service).await}); + } + } + } }.await; + jobs.abort_all(); + let mut workers_ok = true; + while let Some(joined) = jobs.join_next().await { + if let Err(error) = joined { + workers_ok &= error.is_cancelled(); + } + } + result?; + ensure!(workers_ok, "HTTP worker failed"); + Ok::<(), anyhow::Error>(()) + }); + let result = run_sdk( + &endpoint, + image.requested_reference.as_str(), + image.architecture.as_str(), + ) + .await; + stop.send_replace(true); + let joined = timeout(Duration::from_secs(60), &mut server).await; + let joined = match joined { + Ok(result) => result.context("HTTP supervisor join")?, + Err(error) => { + server.abort(); + let _ = server.await; + Err(error.into()) + } + }; + result.and(joined) +} +async fn run_sdk(endpoint: &str, image: &str, architecture: &str) -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut child = Command::new("uv"); + child + .args([ + "run", + "--no-project", + "--python", + "3.13", + "--with-requirements", + ]) + .arg(root.join("tests/workflows/requirements.txt")) + .arg("python") + .arg(root.join("tests/workflows/docker_lambda_sdk.py")) + .args([endpoint, image, architecture]) + .kill_on_drop(true); + let mut process = child.spawn().context("start pinned SDK fixture")?; + let result = timeout(Duration::from_secs(300), process.wait()).await; + let status = match result { + Ok(result) => result?, + Err(error) => { + let _ = process.kill().await; + let _ = process.wait().await; + return Err(error.into()); + } + }; + ensure!(status.success(), "SDK fixture failed: {status}"); + Ok(()) +} diff --git a/crates/rustack-lambda-core/tests/docker_probe.rs b/crates/rustack-lambda-core/tests/docker_probe.rs new file mode 100644 index 00000000..e51c8e92 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_probe.rs @@ -0,0 +1,112 @@ +//! Actual pinned Rust resource probe in disconnected and operator bridge networks. +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{env, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::{ + ContainerPolicy, EnvironmentProbeConfig, NetworkSelection, ProbeSettings, RelayBudgets, + probe_environment, +}; +use tokio::time::Instant; + +mod common; +use common::docker::Fixture; + +#[rstest::rstest] +#[case::disconnected(false)] +#[case::operator_bridge(true)] +#[tokio::test] +#[ignore = "requires explicit Docker runtime image/socket and operator probe bridge"] +async fn test_should_qualify_actual_kernel_limits_and_join_probe_retirement( + #[case] bridge: bool, +) -> Result<()> { + let mut fixture = Fixture::new().await?; + let result = async { + let selection: NetworkSelection = if bridge { + env::var("RUSTACK_RUNTIME_NETWORK") + .ok() + .filter(|value| !value.is_empty()) + .context("explicit operator probe bridge required")? + .parse()? + } else { + "none".parse()? + }; + let deadline = Instant::now() + Duration::from_secs(60); + let network = selection.qualify(&fixture.docker, deadline).await?; + ensure!(network.disconnected() != bridge, "network mode differs"); + let policy = ContainerPolicy::new( + selection, + 1000, + 1000, + 333_333_333, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?; + let config = EnvironmentProbeConfig::builder() + .directory(fixture.lease.clone()) + .relay(fixture.relay.as_ref().context("relay pin")?.clone()) + .environments(fixture.owner.port()) + .policy(policy) + .network(network) + .settings(ProbeSettings::new( + Duration::from_secs(60), + Duration::from_secs(60), + )?) + .build(); + let evidence = probe_environment(config.clone(), deadline).await?; + ensure!( + evidence.matches(&config), + "fresh probe does not match retained inputs" + ); + ensure!( + fixture.owner.port().status().allocated == 0, + "proof issued before physical retirement" + ); + let report = evidence.report(); + ensure!( + report.memory_max == Some(128 * 1024 * 1024) + && report.swap_max == Some(0) + && report.pids_max == Some(256), + "kernel limits differ" + ); + ensure!( + report.cpu_quota == Some(33_000) && report.cpu_period == 100_000, + "CPU rounding differs" + ); + ensure!( + report.euid != 0 + && report.egid != 0 + && report.cap_effective == 0 + && report.no_new_privileges + && report.seccomp_mode == 2, + "privilege isolation differs" + ); + ensure!( + report.root_readonly + && report.temporary.bytes == 512 * 1024 * 1024 + && report.temporary.mode == 0o1777, + "filesystem isolation differs" + ); + let mut changed = config.clone(); + changed.policy = ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + )?; + ensure!( + !evidence.matches(&changed), + "proof matched a different policy" + ); + Ok(()) + } + .await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs new file mode 100644 index 00000000..5efe2c51 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -0,0 +1,560 @@ +//! Explicit local Docker qualification of production HTTP/upgrade/control IO. + +#![cfg(unix)] +#![forbid(unsafe_code)] + +#[path = "docker_wire/code.rs"] +mod code; + +use std::{collections::HashMap, env, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use http::StatusCode; +use rustack_lambda_core::{ + artifact::{ + Architecture as ImageArchitecture, DeploymentArtifact, ImageReference, PinnedImage, + ResolvedImage, + }, + docker::{ + ContainerPolicy, DirectoryLease, DirectoryLeaseError, DockerError, DockerKey, DockerPort, + DockerSocket, DockerWire, EnvironmentPlan, FunctionLogs, ImageLeaseConfig, + ImageLeaseManager, ImagePinScope, ImageResolver, OutputChannel, OwnerFilter, RelayBudgets, + }, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, +}; +use rustack_lambda_model::types::ImageConfig; +use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + metadata::local_trace_id, + protocol::{Architecture, AttemptId, EnvironmentId, Payload}, + session::{Invocation, RuntimeResult}, +}; +use serde_json::{Value, json}; +use tempfile::TempDir; +use tokio::{ + io::AsyncWriteExt, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use uuid::Uuid; + +const IO: Duration = Duration::from_secs(60); +const LARGE: usize = 6 * 1024 * 1024 - 4096; +fn deadline() -> Instant { + Instant::now() + IO +} + +#[derive(Debug)] +struct Names { + relay: DockerKey, + function: DockerKey, + owner: Uuid, + run: Uuid, + environment: EnvironmentId, +} + +#[derive(Debug)] +struct Runtime { + client: ControlClient, + stop: watch::Sender, + pump: Option>>, + settlement: Option>, +} +impl Runtime { + async fn shutdown(&mut self) -> Result<()> { + let protocol = self + .client + .shutdown(deadline()) + .await + .map_err(anyhow::Error::from); + self.stop.send_replace(true); + let pump = if let Some(result) = &self.settlement { + result.clone() + } else { + let result = if let Some(pump) = &mut self.pump { + pump.await + .map_err(|_| DockerError::Worker) + .and_then(|result| result) + } else { + Ok(()) + }; + self.pump.take(); + self.settlement = Some(result.clone()); + result + }; + protocol.and(pump.map_err(anyhow::Error::from)) + } +} +impl Drop for Runtime { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[derive(Debug)] +struct Environment { + wire: DockerWire, + docker: DockerPort, + names: Names, + runtime: Option, + logs: Option, + images: ImageLeaseManager, + artifacts: Vec, + plan: Option, + lease: DirectoryLease, + directory: Option, +} +impl Environment { + async fn new() -> Result { + Self::new_with_code(false).await + } + + async fn new_with_code(code: bool) -> Result { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET") + .context("explicit test Docker socket required")?; + let socket: DockerSocket = format!("unix://{socket}").parse()?; + let wire = DockerWire::connect(socket, deadline()).await?; + let directory = tempfile::tempdir()?; + let lease = if code { + DirectoryLease::acquire_code(directory.path().to_path_buf(), deadline()).await? + } else { + DirectoryLease::acquire(directory.path().to_path_buf(), deadline()).await? + }; + let run = lease.run_id(); + let owner = lease.owner_id(); + let docker = wire.port().bind_directory(&lease, deadline()).await?; + let images = ImageLeaseManager::start( + docker.clone(), + ImagePinScope::new(lease.clone()), + ImageLeaseConfig::default(), + )?; + ensure!( + matches!( + DirectoryLease::acquire(directory.path().to_path_buf(), deadline()).await, + Err(DirectoryLeaseError::InUse) + ), + "image owner did not retain directory exclusion" + ); + Ok(Self { + wire, + docker, + names: Names { + relay: format!("rustack-wire-relay-{run}").parse()?, + function: format!("rustack-wire-function-{run}").parse()?, + owner, + run, + environment: EnvironmentId::default(), + }, + runtime: None, + logs: None, + images, + artifacts: Vec::new(), + plan: None, + lease, + directory: Some(directory), + }) + } + + async fn image(&mut self, reference: &str, architecture: &str) -> Result { + let reference: ImageReference = reference.parse()?; + let architecture = match architecture { + "arm64" => ImageArchitecture::Arm64, + "amd64" => ImageArchitecture::X86_64, + _ => bail!("unknown fixture architecture"), + }; + let resolved = ImageResolver::new(self.docker.clone()) + .resolve_local(&reference, architecture, None, deadline()) + .await?; + let saved: ResolvedImage = serde_json::from_slice(&serde_json::to_vec(&resolved)?)?; + let image = self.images.port().restore(&saved, None, deadline()).await?; + ensure!( + image.resolved().code_sha256()? == saved.code_sha256()?, + "restored image hash changed" + ); + self.artifacts.push(image.clone()); + Ok(image) + } + + async fn start(&mut self) -> Result<()> { + self.start_with(None, true).await + } + + async fn start_with(&mut self, overrides: Option, bootstrap: bool) -> Result<()> { + let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; + let (architecture, daemon_arch) = match arch.as_str() { + "arm64" => (Architecture::Arm64, "aarch64"), + "amd64" => (Architecture::X86_64, "x86_64"), + _ => bail!("architecture must be arm64 or amd64"), + }; + let relay = env::var("RUSTACK_RUNTIME_IMAGE").context("relay fixture image required")?; + let port = self.docker.clone(); + let info = port.info(deadline()).await?; + ensure!( + info.os_type.as_deref() == Some("linux") + && info.architecture.as_deref() == Some(daemon_arch), + "daemon platform mismatch" + ); + let relay = self.image(&relay, &arch).await?; + let function = self + .image(&format!("rustack-p0-handler:{arch}"), &arch) + .await?; + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("wire-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(function.resolved().metadata().architecture) + .artifact(DeploymentArtifact::Image(function)) + .image_config(overrides) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ("TEST_VALUE".into(), "from-captured-revision".into()), + ])) + .timeout(Duration::from_secs(120)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build(); + let plan = EnvironmentPlan::new( + self.lease.clone(), + relay, + Arc::new(revision), + ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new(IO, Duration::from_secs(600), IO)?, + )?, + )?; + self.names.relay = plan.relay_key().clone(); + self.names.function = plan.function_key().clone(); + self.names.environment = plan.environment_id(); + self.plan = Some(plan); + self.plan + .as_ref() + .context("allocation plan")? + .create_relay(&port, deadline()) + .await?; + self.attach(&port, architecture).await?; + port.start_container(&self.names.relay, deadline()).await?; + let client = &mut self.runtime.as_mut().context("runtime owner")?.client; + ensure!( + client.next(deadline()).await? == ClientEvent::RelayReady, + "relay handshake mismatch" + ); + self.plan + .as_ref() + .context("allocation plan")? + .create_function(&port, deadline()) + .await?; + self.logs = Some(FunctionLogs::start( + port.attach_logs(&self.names.function, deadline()).await?, + IO, + 262_144, + )?); + port.start_container(&self.names.function, deadline()) + .await?; + if bootstrap { + ensure!( + client.next(deadline()).await? == ClientEvent::BootstrapReady, + "function readiness mismatch" + ); + } + Ok(()) + } + + async fn attach(&mut self, port: &DockerPort, architecture: Architecture) -> Result<()> { + let attached = port.attach(&self.names.relay, deadline()).await?; + let (mut output, input) = attached.split(IO)?; + let (mut bridge, reader) = tokio::io::duplex(64 * 1024); + let client = ControlClient::start( + ClientConfig::new(self.names.environment, architecture, IO)?, + reader, + input, + )?; + let (stop, mut stopped) = watch::channel(false); + let pump = tokio::spawn(async move { + loop { + let chunk = tokio::select! { + biased; + () = async { let _ = stopped.wait_for(|stop| *stop).await; } => return Ok(()), + chunk = output.next(Instant::now() + Duration::from_secs(600)) => chunk?, + }; + match chunk { + Some(chunk) if chunk.channel == OutputChannel::Stdout => { + timeout_at(deadline(), bridge.write_all(&chunk.bytes)) + .await + .map_err(|_| DockerError::Deadline)??; + } + Some(_) => {} // Relay diagnostics are deliberately not copied into logs. + None => return Ok(()), + } + } + }); + self.runtime = Some(Runtime { + client, + stop, + pump: Some(pump), + settlement: None, + }); + Ok(()) + } + + async fn call(&mut self, event: &Value, failed: bool) -> Result { + let call = Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(120), + "arn:aws:lambda:us-east-1:000000000000:function:wire-fixture:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?; + let id = call.parts().0; + let client = &mut self.runtime.as_mut().context("runtime owner")?.client; + client.invoke(call, deadline()).await?; + match client + .next(Instant::now() + Duration::from_secs(120)) + .await? + { + ClientEvent::InvocationResult { attempt, result } if attempt == id => match result { + RuntimeResult::Success(payload) if !failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + RuntimeResult::FunctionError(payload) if failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + _ => bail!("unexpected function outcome"), + }, + _ => bail!("unexpected control outcome"), + } + } + + async fn cleanup(&mut self) -> Result<()> { + let port = self.docker.clone(); + let function = remove(&port, &self.names.function).await; + let logs = if let Some(logs) = &mut self.logs { + logs.shutdown().await.map(drop).map_err(anyhow::Error::from) + } else { + Ok(()) + }; + let runtime = if let Some(runtime) = &mut self.runtime { + runtime.shutdown().await + } else { + Ok(()) + }; + let relay = remove(&port, &self.names.relay).await; + let remaining = port + .list_containers( + OwnerFilter::new(self.names.owner, Some(self.names.run), None)?, + deadline(), + ) + .await; + let listed = remaining.map_err(anyhow::Error::from).and_then(|items| { + ensure!( + items.is_empty(), + "owned containers remain after wire cleanup" + ); + Ok(()) + }); + self.plan.take(); + self.artifacts.clear(); + let images = self + .images + .shutdown(deadline()) + .await + .map_err(anyhow::Error::from); + let joined = if images.is_ok() { + self.wire.shutdown().await.map_err(anyhow::Error::from) + } else { + Ok(()) + }; + let result = function + .and(logs) + .and(runtime) + .and(relay) + .and(listed) + .and(images) + .and(joined); + if result.is_err() { + // Uncertain cleanup still owns the namespace; retain its files for + // recovery rather than unlinking the live lock inode on test Drop. + if let Some(directory) = self.directory.take() { + let _retained = directory.keep(); + } + } + result + } +} + +async fn remove(port: &DockerPort, key: &DockerKey) -> Result<()> { + match port.remove_container(key, deadline()).await { + Ok(()) | Err(DockerError::Status(StatusCode::NOT_FOUND)) => Ok(()), + Err(error) => Err(error.into()), + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker wire qualification images/socket"] +async fn test_should_execute_real_ric_over_production_wire_with_large_binary_frames_and_cleanup() +-> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start().await?; + let event = + json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE,"logs":true,"policy":true}); + let computed = environment.call(&event, false).await?; + ensure!( + computed["computed"] == 42 + && computed["counter"] == 1 + && computed["inputBytes"] == LARGE, + "computed/large input mismatch" + ); + ensure!( + computed["large"].as_str().is_some_and( + |value| value.len() == LARGE && value.bytes().all(|byte| byte == b'r') + ), + "large output mismatch" + ); + ensure!( + computed["uid"] == 1000 + && computed["rootReadOnly"] == true + && computed["memoryMax"] == "201326592" + && computed["pidsMax"] == "256", + "function resource controls differ" + ); + ensure!( + computed["tmpBytes"] == 512 * 1024 * 1024_u64 + && computed["tmpMode"] == 0o1777 + && computed["swapMax"] == "0" + && computed["capEff"] == "0000000000000000" + && computed["noNewPrivs"] == "1" + && computed["taskRoot"] == "/var/task" + && computed["functionName"] == "wire-fixture" + && computed["functionVersion"] == "$LATEST" + && computed["region"] == "us-east-1" + && computed["configuredValue"] == "from-captured-revision", + "allocation policy runtime enforcement differs" + ); + let cpu = computed["cpuMax"].as_str().context("CPU quota report")?; + let cpu: Vec<_> = cpu.split_whitespace().collect(); + ensure!(matches!(cpu.as_slice(), [quota, period] if quota.parse::().is_ok_and(|quota| quota > 0 && period.parse::() == Ok(quota))), "CPU quota does not enforce one CPU"); + let warm = environment.call(&json!({"value":0}), false).await?; + ensure!( + warm["computed"] == 7 && warm["counter"] == 2, + "warm reuse mismatch" + ); + let failure = environment.call(&json!({"fail":true}), true).await?; + ensure!( + failure["errorType"] == "ValueError", + "function error mismatch" + ); + let logs = environment.logs.as_ref().context("function log capture")?; + timeout_at(deadline(), async { + loop { + let snapshot = logs.snapshot(deadline()).await?; + ensure!(snapshot.failure.is_none(), "log capture failed"); + if snapshot + .tail + .windows(19) + .any(|part| part == b"WIRE_STDOUT_CAPTURE") + && snapshot + .tail + .windows(19) + .any(|part| part == b"WIRE_STDERR_CAPTURE") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("function log receipt deadline")??; + Ok(()) + } + .await; + let cleanup = environment.cleanup().await; + result.and(cleanup) +} + +#[tokio::test] +#[ignore = "requires explicit Docker wire qualification images/socket"] +async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() -> Result<()> { + for reset in [false, true] { + let mut environment = Environment::new().await?; + let result = async { + let overrides = ImageConfig { + entry_point: reset.then(Vec::new), + command: Some(if reset { + vec!["/bin/true".into()] + } else { + vec![] + }), + working_directory: Some("/tmp".into()), + }; + environment + .start_with(Some(overrides.clone()), false) + .await?; + let port = environment.docker.clone(); + let expected = environment + .artifacts + .last() + .context("function artifact")? + .resolved() + .metadata() + .execution + .plan(Some(&overrides))?; + let actual = port + .inspect_container(&environment.names.function, deadline()) + .await? + .config + .context("actual create config")?; + ensure!( + actual.working_dir.as_deref() == Some("/tmp"), + "working directory override differs" + ); + if reset { + ensure!( + actual.entrypoint.as_deref().unwrap_or_default().is_empty(), + "daemon restored a cleared entrypoint" + ); + ensure!( + actual + .cmd + .as_deref() + .is_some_and(|command| command == ["/bin/true"]), + "reset command mismatch" + ); + } else { + ensure!( + actual + .entrypoint + .as_deref() + .is_some_and(|entrypoint| entrypoint == expected.entrypoint()), + "inherited entrypoint mismatch" + ); + ensure!( + actual.cmd.as_deref().unwrap_or_default().is_empty(), + "daemon restored a cleared command" + ); + } + let exit = port + .wait_container(&environment.names.function, deadline()) + .await?; + ensure!( + exit.status_code == if reset { 0 } else { 142 }, + "empty override execution exit differs" + ); + Ok(()) + } + .await; + let cleanup = environment.cleanup().await; + result.and(cleanup)?; + } + Ok(()) +} diff --git a/crates/rustack-lambda-core/tests/docker_wire/code.rs b/crates/rustack-lambda-core/tests/docker_wire/code.rs new file mode 100644 index 00000000..b2751a17 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_wire/code.rs @@ -0,0 +1,830 @@ +//! Actual typed ZIP transfer into a separately owned daemon-local code volume. + +use std::{ + collections::{BTreeMap, HashMap}, + env, + io::{Cursor, Write}, + time::Duration, +}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use http::StatusCode; +use rustack_lambda_core::{ + artifact::Architecture, + docker::{ + CodeRetirement, CodeVolumeConfig, CodeVolumeIdentity, CodeVolumeManager, DockerCodeArchive, + DockerError, DockerKey, FunctionLogs, OwnerFilter, PinnedCode, + models::{ + ContainerCreateBody, ContainerInspectResponse, ContainerStateStatusEnum, HostConfig, + Mount, MountType, MountVolumeOptions, + }, + }, + storage::FunctionStore, +}; +use serde_json::{Value, json}; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, + task::spawn_blocking, + time::{sleep, timeout_at}, +}; +use uuid::Uuid; +use zip::{ZipWriter, write::SimpleFileOptions}; + +use super::{Environment, IO, deadline}; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum VolumePhase { + Unissued, + Uncertain, + Known, +} +impl VolumePhase { + fn name(self) -> &'static str { + match self { + Self::Unissued => "unissued", + Self::Uncertain => "uncertain", + Self::Known => "known", + } + } +} +#[derive(Clone)] +enum ContainerPhase { + Unissued, + Uncertain, + Known(DockerKey), +} +impl ContainerPhase { + fn key(&self) -> Option<&DockerKey> { + if let Self::Known(key) = self { + Some(key) + } else { + None + } + } + fn record(&self) -> Value { + match self { + Self::Unissued => json!({"state":"unissued"}), + Self::Uncertain => json!({"state":"uncertain"}), + Self::Known(key) => json!({"state":"known","id":key.as_str()}), + } + } +} + +struct Fixture { + environment: Environment, + volume: DockerKey, + loader: DockerKey, + function: DockerKey, + labels: BTreeMap, + image: String, + hash: String, + volume_phase: VolumePhase, + loader_phase: ContainerPhase, + function_phase: ContainerPhase, + manager: Option, + ready: Option, + retirement: Option, + native_zip_sha256: Option, +} + +impl Fixture { + async fn new() -> Result { + Self::new_with_manager(false).await + } + + async fn new_with_manager(managed: bool) -> Result { + let mut environment = Environment::new_with_code(managed).await?; + let root = environment + .directory + .as_mut() + .context("code fixture evidence")?; + root.disable_cleanup(true); + eprintln!("ZIP code evidence: {}", root.path().display()); + let id = Uuid::new_v4(); + let labels = BTreeMap::from([ + ( + "rustack.lambda.ownerId".into(), + environment.names.owner.to_string(), + ), + ( + "rustack.lambda.runId".into(), + environment.names.run.to_string(), + ), + ("rustack.lambda.environmentId".into(), id.to_string()), + ("rustack.test.code".into(), id.to_string()), + ]); + Ok(Self { + environment, + volume: format!("rustack-code-{id}").parse()?, + loader: format!("rustack-code-loader-{id}").parse()?, + function: format!("rustack-code-consumer-{id}").parse()?, + labels, + image: String::new(), + hash: String::new(), + volume_phase: VolumePhase::Unissued, + loader_phase: ContainerPhase::Unissued, + function_phase: ContainerPhase::Unissued, + manager: None, + ready: None, + retirement: None, + native_zip_sha256: None, + }) + } + + async fn record(&self) -> Result<()> { + let root = self + .environment + .directory + .as_ref() + .context("code evidence root")? + .path(); + let record = serde_json::to_vec(&json!({"schemaVersion":1,"volume":self.volume.as_str(), + "loader":self.loader.as_str(),"function":self.function.as_str(),"labels":self.labels, + "image":self.image,"zipSha256":self.hash,"nativeZipSha256":self.native_zip_sha256,"volumeState":self.volume_phase.name(), + "loaderState":self.loader_phase.record(),"functionState":self.function_phase.record()}))?; + timeout_at(deadline(), async { + let pending = root.join("code-intent.pending"); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&pending) + .await?; + file.write_all(&record).await?; + file.sync_all().await?; + drop(file); + fs::rename(&pending, root.join("code-intent.json")).await?; + fs::File::open(root).await?.sync_all().await + }) + .await??; + Ok(()) + } + + fn body(&self, loader: bool) -> ContainerCreateBody { + let mut labels: HashMap = self.labels.clone().into_iter().collect(); + labels.insert( + "rustack.lambda.role".into(), + if loader { + "code-loader" + } else { + "code-consumer" + } + .into(), + ); + ContainerCreateBody { + image: Some(self.image.clone()), + user: Some(if loader { "65534:65534" } else { "1000:1000" }.into()), + entrypoint: Some(vec![ + if loader { + "/never-start-this-loader" + } else { + "/var/task/bootstrap" + } + .into(), + ]), + cmd: Some(Vec::new()), + working_dir: Some("/var/task".into()), + labels: Some(labels), + tty: Some(false), + host_config: Some(HostConfig { + network_mode: Some("none".into()), + readonly_rootfs: Some(true), + memory: Some( + if self.native_zip_sha256.is_some() { + 512 + } else { + 128 + } * 1024 + * 1024, + ), + nano_cpus: Some(1_000_000_000), + pids_limit: Some(64), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + tmpfs: Some(HashMap::from([( + "/tmp".into(), + if self.native_zip_sha256.is_some() { + // The separately owned Linux verifier executes Native children here. + "rw,exec,nosuid,nodev,size=536870912" + } else { + "rw,nosuid,nodev,size=536870912" + } + .into(), + )])), + mounts: Some(vec![Mount { + typ: Some(MountType::VOLUME), + source: Some(self.volume.as_str().into()), + target: Some("/var/task".into()), + read_only: Some(!loader), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..MountVolumeOptions::default() + }), + ..Mount::default() + }]), + ..HostConfig::default() + }), + ..ContainerCreateBody::default() + } + } + + fn owned(&self, inspected: &ContainerInspectResponse, loader: bool) -> Result { + let name = if loader { &self.loader } else { &self.function }; + let config = inspected + .config + .as_ref() + .context("owned container config")?; + let expected = self.body(loader); + let id = inspected.id.as_deref().context("owned container ID")?; + ensure!( + id.len() == 64 + && id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "noncanonical container ID" + ); + ensure!( + inspected.name.as_deref() == Some(format!("/{}", name.as_str()).as_str()) + && config.image.as_deref() == Some(self.image.as_str()) + && expected.labels.as_ref().is_some_and(|required| config + .labels + .as_ref() + .is_some_and(|actual| required + .iter() + .all(|(key, value)| actual.get(key) == Some(value)))), + "foreign code fixture container" + ); + let mounts = inspected.mounts.as_ref().context("actual code mount")?; + ensure!( + mounts + .iter() + .any(|mount| mount.name.as_deref() == Some(self.volume.as_str()) + && mount.destination.as_deref() == Some("/var/task") + && mount.rw == Some(loader)), + "actual volume mount mode/name mismatch" + ); + Ok(id.parse()?) + } + + async fn observe(&mut self, loader: bool) -> Result { + let name = if loader { &self.loader } else { &self.function }; + let inspected = self + .environment + .docker + .inspect_container(name, deadline()) + .await?; + let id = self.owned(&inspected, loader)?; + if loader { + self.loader_phase = ContainerPhase::Known(id.clone()); + } else { + self.function_phase = ContainerPhase::Known(id.clone()); + } + self.record().await?; + Ok(id) + } + + async fn prepare_code(&mut self) -> Result<(DockerCodeArchive, Architecture)> { + let arch = env::var("RUSTACK_RUNTIME_ARCH")?; + let image = self + .environment + .image(&format!("rustack-custom-ric:a-{arch}"), &arch) + .await?; + image + .resolved() + .metadata() + .image_id + .as_str() + .clone_into(&mut self.image); + let architecture = image.resolved().metadata().architecture; + let path = env::var("RUSTACK_ZIP_CODE_BOOTSTRAP") + .context("explicit cross-built ZIP probe required")?; + let bytes = timeout_at(deadline(), async { + let mut file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + (&mut file) + .take(20 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= 20 * 1024 * 1024, "probe source cap"); + Ok::<_, anyhow::Error>(bytes) + }) + .await??; + let zip = timeout_at(deadline(), spawn_blocking(move || package(&bytes))).await???; + let root = self + .environment + .directory + .as_ref() + .context("artifact root")? + .path() + .join("artifacts"); + let store = FunctionStore::new(&root); + let (path, hash, _) = store + .store_zip_code("code-probe", "$LATEST", &Bytes::from(zip)) + .await?; + let artifact = store.capture_artifact(&path).await?; + self.hash = hash; + timeout_at( + deadline(), + fs::write(path.join("bootstrap"), b"mutated host extraction"), + ) + .await??; + let code = DockerCodeArchive::prepare(artifact, architecture, deadline()).await?; + ensure!( + code.len() > 1024 * 1024 && code.code_sha256() == self.hash, + "large immutable transfer identity" + ); + Ok((code, architecture)) + } + + async fn prepare_native_code(&mut self) -> Result<(DockerCodeArchive, Architecture)> { + let arch = env::var("RUSTACK_RUNTIME_ARCH")?; + let image = self + .environment + .image(&env::var("RUSTACK_ZIP_BASE_IMAGE")?, &arch) + .await?; + image + .resolved() + .metadata() + .image_id + .as_str() + .clone_into(&mut self.image); + let architecture = image.resolved().metadata().architecture; + let zip = bounded_file(&env::var("RUSTACK_PROVIDED_ZIP_A")?, 50 * 1024 * 1024).await?; + let original_hash = rustack_lambda_core::storage::compute_sha256(&zip); + eprintln!("Native original ZIP digest: {original_hash}"); + self.native_zip_sha256 = Some(original_hash.clone()); + let bootstrap = + bounded_file(&env::var("RUSTACK_ZIP_NATIVE_VERIFIER")?, 50 * 1024 * 1024).await?; + let manifest = serde_json::to_vec( + &json!({"architecture": if arch == "arm64" { "arm64" } else { "x86_64" }, + "zipSha256":original_hash,"codeMarker":"A"}), + )?; + let packaged = timeout_at( + deadline(), + spawn_blocking(move || { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (name, data, mode) in [ + ("bootstrap", bootstrap, 0o755), + ("input.zip", zip, 0o644), + ("manifest.json", manifest, 0o644), + ] { + writer.start_file(name, SimpleFileOptions::default().unix_permissions(mode))?; + writer.write_all(&data)?; + } + Ok::<_, anyhow::Error>(writer.finish()?.into_inner()) + }), + ) + .await???; + let root = self + .environment + .directory + .as_ref() + .context("Native artifact evidence")? + .path() + .join("artifacts"); + let store = FunctionStore::new(root); + let (path, hash, _) = store + .store_zip_code("native-verifier", "$LATEST", &Bytes::from(packaged)) + .await?; + self.hash = hash; + let artifact = store.capture_artifact(&path).await?; + Ok(( + DockerCodeArchive::prepare(artifact, architecture, deadline()).await?, + architecture, + )) + } + + async fn exercise(&mut self) -> Result<()> { + let (code, architecture) = self.prepare_code().await?; + self.volume_phase = VolumePhase::Uncertain; + self.record().await?; + self.environment + .docker + .create_volume(&self.volume, &self.labels, deadline()) + .await?; + self.volume_phase = VolumePhase::Known; + self.record().await?; + self.loader_phase = ContainerPhase::Uncertain; + self.record().await?; + self.environment + .docker + .create_container_on_platform(&self.loader, &self.body(true), architecture, deadline()) + .await?; + let loader = self.observe(true).await?; + self.require_unstarted(&loader).await?; + self.environment + .docker + .upload_code(&loader, &code, deadline()) + .await?; + self.require_unstarted(&loader).await?; + self.retire(true).await?; + self.function_phase = ContainerPhase::Uncertain; + self.record().await?; + self.environment + .docker + .create_container_on_platform( + &self.function, + &self.body(false), + architecture, + deadline(), + ) + .await?; + let function = self.observe(false).await?; + self.consume_probe(&function).await?; + Ok(()) + } + + async fn exercise_managed(&mut self) -> Result<()> { + let (code, architecture) = if self.native_zip_sha256.is_some() { + self.prepare_native_code().await? + } else { + self.prepare_code().await? + }; + let base = self + .environment + .artifacts + .last() + .context("source pin")? + .clone(); + let manager = CodeVolumeManager::start( + self.environment.docker.clone(), + self.environment.lease.clone(), + CodeVolumeConfig::default(), + )?; + let port = manager.port(); + self.manager = Some(manager); + let preparation = port.start_preparation( + CodeVolumeIdentity::new(Uuid::new_v4(), Uuid::new_v4())?, + code, + base, + deadline() + Duration::from_secs(120), + )?; + self.retirement = Some(preparation.retirement()); + let ready = preparation.wait().await?; + self.volume = ready.volume_name().clone(); + ensure!( + ready.base_image_id().as_str() == self.image, + "changed base image" + ); + let mount = ready.mount()?; + self.ready = Some(ready); + ensure!( + port.status().volumes == 1 && port.status().preparing == 0, + "Ready admission" + ); + self.function_phase = ContainerPhase::Uncertain; + self.record().await?; + let mut body = self.body(false); + body.host_config.as_mut().context("consumer policy")?.mounts = Some(vec![mount]); + self.environment + .docker + .create_container_on_platform(&self.function, &body, architecture, deadline()) + .await?; + let function = self.observe(false).await?; + self.consume_probe(&function).await?; + ensure!( + port.status().volumes == 1, + "consumer must retain code ownership" + ); + Ok(()) + } + + async fn consume_probe(&mut self, function: &DockerKey) -> Result<()> { + self.environment.logs = Some(FunctionLogs::start( + self.environment + .docker + .attach_logs(function, deadline()) + .await?, + IO, + 65_536, + )?); + self.environment + .docker + .start_container(function, deadline()) + .await?; + let exit = self + .environment + .docker + .wait_container(function, deadline()) + .await?; + let logs = self.environment.logs.as_mut().context("probe log owner")?; + timeout_at(deadline(), async { + loop { + let snapshot = logs.snapshot(deadline()).await?; + if snapshot.closed { + ensure!(snapshot.failure.is_none(), "probe log failure"); + break; + } + sleep(Duration::from_millis(10)).await; + } + Ok::<_, anyhow::Error>(()) + }) + .await??; + let tail = logs.shutdown().await?.tail; + let evidence = self + .environment + .directory + .as_ref() + .context("probe evidence root")? + .path() + .join("code-output.log"); + timeout_at(deadline(), fs::write(evidence, &tail)).await??; + ensure!( + exit.status_code == 0 && exit.error.is_none(), + "code probe exited unsuccessfully" + ); + let output: Value = serde_json::from_slice(&tail)?; + if let Some(hash) = &self.native_zip_sha256 { + return verify_native(&output, hash); + } + ensure!( + output + == json!({"computed":42,"readOnly":true,"tmpWritable":true,"metadata":[ + {"uid":0,"gid":0,"mode":0o555},{"uid":0,"gid":0,"mode":0o555},{"uid":0,"gid":0,"mode":0o444}]}), + "actual ZIP code/ownership/mode computation mismatch" + ); + Ok(()) + } + + async fn require_unstarted(&self, key: &DockerKey) -> Result<()> { + let inspected = self + .environment + .docker + .inspect_container(key, deadline()) + .await?; + self.owned(&inspected, true)?; + ensure!( + inspected + .state + .is_some_and(|state| state.running == Some(false) + && state.status == Some(ContainerStateStatusEnum::CREATED)), + "loader was started" + ); + Ok(()) + } + + async fn retire(&mut self, loader: bool) -> Result<()> { + let (phase, name) = if loader { + (self.loader_phase.clone(), self.loader.clone()) + } else { + (self.function_phase.clone(), self.function.clone()) + }; + if matches!(phase, ContainerPhase::Unissued) { + return Ok(()); + } + let known = phase.key().cloned(); + let lookup = known.as_ref().unwrap_or(&name); + let inspected = match self + .environment + .docker + .inspect_container(lookup, deadline()) + .await + { + Err(DockerError::Status(StatusCode::NOT_FOUND)) if known.is_some() => return Ok(()), + result => result?, + }; + let id = self.owned(&inspected, loader)?; + ensure!( + known.as_ref().is_none_or(|known| known == &id), + "changed known container identity" + ); + if loader { + self.loader_phase = ContainerPhase::Known(id.clone()); + } else { + self.function_phase = ContainerPhase::Known(id.clone()); + } + self.record().await?; + match inspected.state.and_then(|state| state.running) { + Some(true) => { + let _ = self + .environment + .docker + .kill_container(&id, deadline()) + .await; + self.environment + .docker + .wait_container(&id, deadline()) + .await?; + } + Some(false) => {} + None => bail!("missing code container state"), + } + let _ = self + .environment + .docker + .remove_container(&id, deadline()) + .await; + ensure!( + matches!( + self.environment + .docker + .inspect_container(&id, deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ), + "code container retirement unproven" + ); + Ok(()) + } + + async fn cleanup(&mut self) -> Result<()> { + self.retire(false).await?; + if let Some(manager) = &mut self.manager { + self.ready.take(); + if let Some(retirement) = &mut self.retirement { + retirement.wait(deadline()).await?; + } + manager.shutdown(deadline()).await?; + } + self.retire(true).await?; + if self.volume_phase != VolumePhase::Unissued { + match self + .environment + .docker + .inspect_volume(&self.volume, deadline()) + .await + { + Err(DockerError::Status(StatusCode::NOT_FOUND)) + if self.volume_phase == VolumePhase::Known => {} + result => { + let volume = result?; + ensure!( + volume.name == self.volume.as_str() + && volume.driver == "local" + && volume.options.is_empty() + && volume.labels == self.labels.clone().into_iter().collect(), + "foreign code volume" + ); + self.volume_phase = VolumePhase::Known; + self.record().await?; + let _ = self + .environment + .docker + .remove_volume(&self.volume, deadline()) + .await; + ensure!( + matches!( + self.environment + .docker + .inspect_volume(&self.volume, deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ), + "code volume retirement unproven" + ); + } + } + } + ensure!( + self.environment + .docker + .list_volumes( + OwnerFilter::new(self.environment.names.owner, None, None)?, + deadline() + ) + .await? + .is_empty(), + "owned code volumes remain" + ); + self.environment.cleanup().await + } +} + +#[tokio::test] +#[ignore = "requires actual retained code owner; run make test-lambda-docker"] +async fn test_should_prepare_code_with_retained_owner_and_consume_it_read_only() -> Result<()> { + let mut fixture = Fixture::new_with_manager(true).await?; + let result = timeout_at( + deadline() + Duration::from_secs(180), + fixture.exercise_managed(), + ) + .await + .context("managed code fixture observer expired") + .and_then(|result| result); + let cleanup = fixture.cleanup().await; + result?; + cleanup?; + fixture + .environment + .directory + .as_mut() + .context("code evidence root")? + .disable_cleanup(false); + Ok(()) +} + +fn package(bootstrap: &[u8]) -> Result> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file( + "bootstrap", + SimpleFileOptions::default().unix_permissions(0o755), + )?; + writer.write_all(bootstrap)?; + writer.start_file( + "data/input", + SimpleFileOptions::default().unix_permissions(0o644), + )?; + writer.write_all(b"21")?; + writer.start_file( + "large-data", + SimpleFileOptions::default().unix_permissions(0o644), + )?; + writer.write_all(&vec![b'x'; 2 * 1024 * 1024])?; + Ok(writer.finish()?.into_inner()) +} + +#[tokio::test] +#[ignore = "requires actual code-volume transport; run make test-lambda-docker-wire"] +async fn test_should_upload_zip_code_to_a_private_volume_and_consume_it_read_only() -> Result<()> { + let mut fixture = Fixture::new().await?; + let result = timeout_at(deadline() + Duration::from_secs(180), fixture.exercise()) + .await + .context("code fixture observer expired") + .and_then(|result| result); + let cleanup = fixture.cleanup().await; + result?; + cleanup?; + fixture + .environment + .directory + .as_mut() + .context("code evidence root")? + .disable_cleanup(false); + Ok(()) +} + +async fn bounded_file(path: &str, maximum: u64) -> Result> { + timeout_at(deadline(), async { + let metadata = fs::symlink_metadata(path).await?; + ensure!( + metadata.is_file() && metadata.len() <= maximum, + "bounded regular source file" + ); + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(maximum.checked_add(1).context("source cap overflow")?) + .read_to_end(&mut bytes) + .await?; + ensure!(u64::try_from(bytes.len())? <= maximum, "source byte cap"); + Ok(bytes) + }) + .await? +} + +fn verify_native(output: &Value, hash: &str) -> Result<()> { + ensure!( + output["schemaVersion"] == 1 + && output["kind"] == "native-same-zip" + && output["joined"] == true + && output["zipSha256"] == hash, + "Native original ZIP identity/join proof" + ); + let observed = output["observed"] + .as_array() + .context("Native computed observations")?; + ensure!(observed.len() == 2, "Native cold/warm inventory"); + for (index, n) in [42, 46].into_iter().enumerate() { + let result = observed.get(index).context("Native observation")?; + ensure!( + result["computed"] == n + && result["codeMarker"] == "A" + && result["providedCounter"].as_u64() == Some(u64::try_from(index)? + 1), + "actual Native computation/reuse" + ); + } + let cold = observed.first().context("Native cold observation")?; + let warm = observed.get(1).context("Native warm observation")?; + ensure!( + cold["pid"].as_u64().is_some_and(|pid| pid > 1) && warm["pid"] == cold["pid"], + "actual reused Native child identity" + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires actual Linux Native verifier and exact provided ZIP; run make \ + test-lambda-docker"] +async fn test_should_execute_the_same_provided_zip_bytes_through_real_linux_native_and_join_children() +-> Result<()> { + let mut fixture = Fixture::new_with_manager(true).await?; + fixture.native_zip_sha256 = Some(String::new()); + let result = timeout_at( + deadline() + Duration::from_secs(180), + fixture.exercise_managed(), + ) + .await + .context("Native same ZIP resource observer expired") + .and_then(|result| result); + let cleanup = fixture.cleanup().await; + result?; + cleanup?; + fixture + .environment + .directory + .as_mut() + .context("Native evidence root")? + .disable_cleanup(false); + Ok(()) +} diff --git a/crates/rustack-lambda-http/src/dispatch.rs b/crates/rustack-lambda-http/src/dispatch.rs index 5f53c776..1e1c1645 100644 --- a/crates/rustack-lambda-http/src/dispatch.rs +++ b/crates/rustack-lambda-http/src/dispatch.rs @@ -32,6 +32,8 @@ pub trait LambdaHandler: Send + Sync + 'static { } /// Dispatch a Lambda operation to the handler. +/// # Errors +/// Returns the handler error for invalid requests, unavailable resources or failed operations. pub async fn dispatch_operation( handler: &H, op: LambdaOperation, diff --git a/crates/rustack-lambda-http/src/lib.rs b/crates/rustack-lambda-http/src/lib.rs index f8e5ee76..082cf6f4 100644 --- a/crates/rustack-lambda-http/src/lib.rs +++ b/crates/rustack-lambda-http/src/lib.rs @@ -7,7 +7,7 @@ //! - **Service**: Hyper `Service` implementation for the Lambda protocol //! - **Response helpers**: JSON success/error response formatting //! -//! Unlike `awsJson1.1` services (SSM, DynamoDB), Lambda uses REST-style +//! Unlike `awsJson1.1` services (SSM, `DynamoDB`), Lambda uses REST-style //! routing where each operation has a unique method + path combination. //! Path parameters (e.g., `{FunctionName}`) are extracted and passed to //! the handler. diff --git a/crates/rustack-lambda-http/src/router.rs b/crates/rustack-lambda-http/src/router.rs index 5c343f46..4afffc46 100644 --- a/crates/rustack-lambda-http/src/router.rs +++ b/crates/rustack-lambda-http/src/router.rs @@ -115,7 +115,7 @@ fn normalize_date_prefix(path: &str) -> Option { let (date_part, rest) = trimmed.split_once('/')?; // Validate it looks like a date: YYYY-MM-DD (10 chars). - if date_part.len() != 10 || date_part.as_bytes()[4] != b'-' || date_part.as_bytes()[7] != b'-' { + if !matches!(date_part.as_bytes(), [_, _, _, _, b'-', _, _, b'-', _, _]) { return None; } diff --git a/crates/rustack-lambda-model/src/error.rs b/crates/rustack-lambda-model/src/error.rs index 00768287..52e2f4e0 100644 --- a/crates/rustack-lambda-model/src/error.rs +++ b/crates/rustack-lambda-model/src/error.rs @@ -14,6 +14,8 @@ pub enum LambdaErrorCode { ResourceNotFoundException, /// Resource already exists. ResourceConflictException, + /// An optimistic update precondition did not match. + PreconditionFailedException, /// Request payload too large. RequestTooLargeException, /// Unsupported media type. @@ -39,6 +41,7 @@ impl LambdaErrorCode { Self::InvalidRequestContentException => "InvalidRequestContentException", Self::ResourceNotFoundException => "ResourceNotFoundException", Self::ResourceConflictException => "ResourceConflictException", + Self::PreconditionFailedException => "PreconditionFailedException", Self::RequestTooLargeException => "RequestTooLargeException", Self::UnsupportedMediaTypeException => "UnsupportedMediaTypeException", Self::InvalidRuntimeException => "InvalidRuntimeException", @@ -57,6 +60,7 @@ impl LambdaErrorCode { } Self::ResourceNotFoundException => http::StatusCode::NOT_FOUND, Self::ResourceConflictException => http::StatusCode::CONFLICT, + Self::PreconditionFailedException => http::StatusCode::PRECONDITION_FAILED, Self::RequestTooLargeException => http::StatusCode::PAYLOAD_TOO_LARGE, Self::UnsupportedMediaTypeException => http::StatusCode::UNSUPPORTED_MEDIA_TYPE, Self::InvalidRuntimeException | Self::ResourceNotReadyException => { diff --git a/crates/rustack-lambda-model/src/input.rs b/crates/rustack-lambda-model/src/input.rs index 248928f2..544c4bde 100644 --- a/crates/rustack-lambda-model/src/input.rs +++ b/crates/rustack-lambda-model/src/input.rs @@ -12,7 +12,7 @@ use crate::types::{ /// Input for `CreateFunction`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct CreateFunctionInput { /// Function name (1-140 characters). pub function_name: String, @@ -71,7 +71,7 @@ pub struct CreateFunctionInput { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, /// Code signing config ARN (stored, not enforced). @@ -81,8 +81,14 @@ pub struct CreateFunctionInput { /// Input for `UpdateFunctionCode`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct UpdateFunctionCodeInput { + /// Validate parameters without modifying code or publishing a version. + #[serde(skip_serializing_if = "Option::is_none")] + pub dry_run: Option, + /// Revision ID precondition for an optimistic update. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, /// Base64-encoded zip file. #[serde(skip_serializing_if = "Option::is_none")] pub zip_file: Option, @@ -111,8 +117,11 @@ pub struct UpdateFunctionCodeInput { /// Input for `UpdateFunctionConfiguration`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct UpdateFunctionConfigurationInput { + /// Revision ID precondition for an optimistic update. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, /// Runtime. #[serde(skip_serializing_if = "Option::is_none")] pub runtime: Option, @@ -155,7 +164,7 @@ pub struct UpdateFunctionConfigurationInput { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, } @@ -304,7 +313,7 @@ pub struct PublishLayerVersionInput { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct CreateEventSourceMappingInput { - /// ARN of the event source (e.g., SQS queue, DynamoDB stream, Kinesis stream). + /// ARN of the event source (e.g., SQS queue, `DynamoDB` stream, Kinesis stream). pub event_source_arn: String, /// Function name or ARN. pub function_name: String, @@ -424,3 +433,22 @@ pub struct EventInvokeConfigInput { #[serde(skip_serializing_if = "Option::is_none")] pub destination_config: Option, } + +#[cfg(test)] +mod revision_tests { + use super::{UpdateFunctionCodeInput, UpdateFunctionConfigurationInput}; + + #[test] + fn test_should_preserve_explicit_revision_preconditions_in_both_update_wire_shapes() { + let json = r#"{"RevisionId":"expected-revision"}"#; + let code: UpdateFunctionCodeInput = serde_json::from_str(json).unwrap(); + let configuration: UpdateFunctionConfigurationInput = serde_json::from_str(json).unwrap(); + assert_eq!(code.revision_id.as_deref(), Some("expected-revision")); + assert_eq!( + configuration.revision_id.as_deref(), + Some("expected-revision") + ); + assert_eq!(serde_json::to_string(&code).unwrap(), json); + assert_eq!(serde_json::to_string(&configuration).unwrap(), json); + } +} diff --git a/crates/rustack-lambda-model/src/types.rs b/crates/rustack-lambda-model/src/types.rs index 4af421cc..8dd54d6a 100644 --- a/crates/rustack-lambda-model/src/types.rs +++ b/crates/rustack-lambda-model/src/types.rs @@ -97,7 +97,7 @@ pub enum PackageType { /// Function architecture. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum Architecture { - /// x86_64 architecture. + /// `x86_64` architecture. #[serde(rename = "x86_64")] #[default] X86_64, @@ -227,7 +227,7 @@ pub enum InvokeMode { /// Function code input for `CreateFunction` and `UpdateFunctionCode`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct FunctionCode { /// Base64-encoded zip file contents. #[serde(skip_serializing_if = "Option::is_none")] @@ -269,7 +269,7 @@ pub struct FunctionCodeLocation { /// Environment variable configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct Environment { /// Environment variable key-value pairs. #[serde(skip_serializing_if = "Option::is_none")] @@ -302,7 +302,7 @@ pub struct EnvironmentError { /// VPC configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct VpcConfig { /// Subnet IDs. #[serde(skip_serializing_if = "Option::is_none")] @@ -329,7 +329,7 @@ pub struct VpcConfigResponse { /// Tracing configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct TracingConfig { /// Tracing mode. #[serde(skip_serializing_if = "Option::is_none")] @@ -347,7 +347,7 @@ pub struct TracingConfigResponse { /// Dead letter queue configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct DeadLetterConfig { /// Target ARN (SQS queue or SNS topic). #[serde(skip_serializing_if = "Option::is_none")] @@ -356,7 +356,7 @@ pub struct DeadLetterConfig { /// Ephemeral storage configuration. #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct EphemeralStorage { /// Size in MB (512-10240). pub size: u32, @@ -370,7 +370,7 @@ impl Default for EphemeralStorage { /// Image configuration override. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct ImageConfig { /// Entry point override. #[serde(skip_serializing_if = "Option::is_none")] @@ -409,7 +409,7 @@ pub struct ImageConfigError { /// Logging configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct LoggingConfig { /// Log format. #[serde(skip_serializing_if = "Option::is_none")] @@ -425,16 +425,16 @@ pub struct LoggingConfig { pub log_group: Option, } -/// SnapStart configuration (stored, not enforced). +/// `SnapStart` configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct SnapStart { /// Apply on. #[serde(skip_serializing_if = "Option::is_none")] pub apply_on: Option, } -/// SnapStart response. +/// `SnapStart` response. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct SnapStartResponse { @@ -627,7 +627,7 @@ pub struct FunctionConfiguration { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart response. + /// `SnapStart` response. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, /// Image configuration response. diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml new file mode 100644 index 00000000..0aa37ffa --- /dev/null +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "rustack-lambda-runtime" +description = "Shared Lambda runtime session and bounded relay protocol for Rustack" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +getrandom = { workspace = true } +url = { workspace = true } +rustls = { workspace = true } +tokio-rustls = { workspace = true } +webpki-roots = { workspace = true } +bytes = { workspace = true, features = ["serde"] } +ciborium = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +typed-builder = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-core = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +rcgen = { workspace = true } +anyhow = { workspace = true } +proptest = { workspace = true } +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-runtime/src/client.rs b/crates/rustack-lambda-runtime/src/client.rs new file mode 100644 index 00000000..ac84fe81 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/client.rs @@ -0,0 +1,491 @@ +//! Ordered host-side relay protocol; external resources remain host-owned. +//! +//! Partial framing lives in retained stream workers. Selecting and cancelling +//! `next()` never cancels those workers or replays a business invocation. + +use std::time::Duration; + +use tokio::{ + io::{AsyncRead, AsyncWrite}, + time::{Instant, timeout_at}, +}; + +use crate::{ + metadata::deadline_unix_ms, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, HandlerTimeout, Message, + Outcome, Payload, + }, + relay::RelayError, + session::{Invocation, RuntimeResult}, + transport::{InputEnd, Settings, Transport, after, preserve_worker_failure}, +}; + +/// Validated host identity, expected architecture and control IO budget. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ClientConfig { + environment: EnvironmentId, + architecture: Architecture, + io_budget: Duration, +} + +impl ClientConfig { + /// Validate a nonzero control budget of at most sixty seconds. + /// + /// # Errors + /// Returns Configuration when the budget is outside its allowed range. + /// + /// ``` + /// # use rustack_lambda_runtime::{client::ClientConfig, protocol::{Architecture, EnvironmentId}}; + /// let config = ClientConfig::new(EnvironmentId::default(), Architecture::Arm64, + /// std::time::Duration::from_secs(5))?; + /// # Ok::<(), rustack_lambda_runtime::relay::RelayError>(()) + /// ``` + pub fn new( + environment: EnvironmentId, + architecture: Architecture, + io_budget: Duration, + ) -> Result { + if io_budget.is_zero() || io_budget > Duration::from_secs(60) { + return Err(RelayError::Configuration); + } + Ok(Self { + environment, + architecture, + io_budget, + }) + } +} + +/// One validated relay event, with payload diagnostics redacted. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ClientEvent { + /// Hello/architecture and Ready were received in order. + RelayReady, + /// The function issued its first valid Runtime API poll. + BootstrapReady, + /// One matching attempt completed; `FunctionError` retires this session. + InvocationResult { + /// Correlated invocation identity. + attempt: AttemptId, + /// Raw success or function error, without an implicit retry. + result: RuntimeResult, + }, + /// A matching delivered handler clock expired; this session is retiring. + InvocationTimedOut(AttemptId), + /// Initialization failed; this environment cannot be reused. + InitError(Payload), + /// The relay invalidated the attempt; external code may still be running. + Cancelled(AttemptId), + /// Close was acknowledged after the peer joined its Runtime API. + Closed, +} + +#[derive(Debug)] +enum Phase { + AwaitHello, + AwaitReady, + Initializing, + Idle, + Busy(AttemptId), + Cancelling { + attempt: AttemptId, + outcome_seen: bool, + }, + Retiring, + Closed, + Failed(RelayError), +} + +#[derive(Debug)] +struct State { + phase: Phase, + closing: bool, + writing: bool, + delivered: bool, + handler_timeout: Option, +} + +/// One host control session with retained IO and fail-closed send semantics. +/// +/// Drop signals its stream supervisor. Normal environment shutdown must call +/// shutdown and separately terminate/remove every external resource. +#[derive(Debug)] +#[non_exhaustive] +pub struct ControlClient { + config: ClientConfig, + transport: Transport, + state: State, + last_attempt: Option, + settlement: Option>, +} + +impl ControlClient { + /// Own asynchronously cancellable streams and start their retained workers. + /// + /// # Errors + /// Returns Worker if no Tokio runtime is entered. + /// + /// ```no_run + /// # async fn example(config: rustack_lambda_runtime::client::ClientConfig) -> Result<(), rustack_lambda_runtime::relay::RelayError> { + /// let (reader, _) = tokio::io::duplex(1024); + /// let (writer, _) = tokio::io::duplex(1024); + /// let mut client = rustack_lambda_runtime::client::ControlClient::start(config, reader, writer)?; + /// let first = client.next(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # Ok(()) } + /// ``` + pub fn start(config: ClientConfig, reader: R, writer: W) -> Result + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + Ok(Self { + config, + transport: Transport::start( + reader, + writer, + Settings { + silence: Duration::from_hours(1), + io_budget: config.io_budget, + input_end: InputEnd::CloseAcknowledgement, + }, + )?, + state: State { + phase: Phase::AwaitHello, + closing: false, + writing: false, + delivered: false, + handler_timeout: None, + }, + last_attempt: None, + settlement: None, + }) + } + + /// Whether a healthy, bootstrapped session can accept a new attempt now. + /// + /// ```no_run + /// # fn inspect(client: &rustack_lambda_runtime::client::ControlClient) { + /// let may_invoke = client.is_idle(); + /// # } + /// ``` + #[must_use] + pub fn is_idle(&self) -> bool { + matches!(self.state.phase, Phase::Idle) + && !self.state.closing + && !self.state.writing + && !self.transport.finished() + } + + /// Receive one validated event; cancelling this future preserves framing. + /// + /// # Errors + /// Returns a cached protocol/IO failure or invalidates the session at the + /// absolute deadline. It rejects calls after an interrupted send. + /// + /// ```no_run + /// # async fn receive(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let event = client.next(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn next(&mut self, deadline: Instant) -> Result { + self.check()?; + loop { + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + let received = timeout_at(deadline, self.transport.recv()).await; + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + let frame = match received { + Ok(Ok(frame)) => frame, + Ok(Err(error)) => return Err(self.fail(error)), + Err(_) => return Err(self.fail(RelayError::Deadline)), + }; + if frame.environment() != self.config.environment { + return Err(self.fail(RelayError::Scope)); + } + match self.accept(frame.into_message()) { + Ok(Some(event)) => return Ok(event), + Ok(None) => {} + Err(error) => return Err(self.fail(error)), + } + } + } + + /// Flush one attempt after first-poll readiness; never reconnect or replay. + /// + /// # Errors + /// Returns invalid order/identity, expired metadata or transport failure. + /// Dropping this send future poisons later business operations. + /// + /// ```no_run + /// # async fn invoke(client: &mut rustack_lambda_runtime::client::ControlClient, invocation: rustack_lambda_runtime::session::Invocation) { + /// let sent = client.invoke(invocation, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn invoke( + &mut self, + invocation: Invocation, + deadline: Instant, + ) -> Result<(), RelayError> { + self.check()?; + let (attempt, call_deadline, function_arn, trace_id, payload) = invocation.parts(); + if self.state.closing + || !matches!(self.state.phase, Phase::Idle) + || self.last_attempt == Some(attempt) + { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + let deadline_unix_ms = match deadline_unix_ms(call_deadline) { + Ok(deadline) + if call_deadline.saturating_duration_since(Instant::now()) + <= Duration::from_hours(1) => + { + deadline + } + _ => return Err(self.fail(RelayError::Deadline)), + }; + self.last_attempt = Some(attempt); + self.state.handler_timeout = invocation.handler_timeout(); + self.state.phase = Phase::Busy(attempt); + self.send( + Message::Invoke { + attempt, + deadline_unix_ms, + handler_timeout_ms: invocation.handler_timeout(), + function_arn: function_arn.clone(), + trace_id: trace_id.clone(), + payload: payload.clone(), + }, + deadline.min(call_deadline), + ) + .await + } + + /// Invalidate the active or just-completed last attempt; await its event next. + /// + /// # Errors + /// Rejects another identity, overlapping cancellation or a closing session. + /// IO failure invalidates the connection without asserting process exit. + /// + /// ```no_run + /// # async fn cancel(client: &mut rustack_lambda_runtime::client::ControlClient, id: rustack_lambda_runtime::protocol::AttemptId) { + /// let sent = client.cancel(id, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn cancel( + &mut self, + attempt: AttemptId, + deadline: Instant, + ) -> Result<(), RelayError> { + self.check()?; + if self.state.closing || self.last_attempt != Some(attempt) { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + let outcome_seen = match self.state.phase { + Phase::Busy(active) if active == attempt => false, + Phase::Idle | Phase::Retiring => true, + _ => return Err(self.fail(RelayError::UnexpectedMessage)), + }; + self.state.phase = Phase::Cancelling { + attempt, + outcome_seen, + }; + self.send(Message::Cancel { attempt }, deadline).await + } + + /// Flush Close while retaining validation of preceding peer events. + /// + /// # Errors + /// Requires ordered Hello/Ready and a healthy, uninterrupted transport. + /// + /// ```no_run + /// # async fn close(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let sent = client.begin_close(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn begin_close(&mut self, deadline: Instant) -> Result<(), RelayError> { + self.check()?; + if matches!(self.state.phase, Phase::AwaitHello | Phase::AwaitReady) { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + if !self.state.closing { + self.state.closing = true; + self.send(Message::Close, deadline).await?; + } + Ok(()) + } + + /// Complete Close/Ack and join IO, retaining ownership across dropped waiters. + /// + /// # Errors + /// Caches protocol, deadline, uncertain-write and worker failures. A failed + /// handshake still stops/joins stream workers; external cleanup is separate. + /// + /// ```no_run + /// # async fn shutdown(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let clean = client.shutdown(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), RelayError> { + if let Some(result) = &self.settlement { + return result.clone(); + } + let outcome = async { + if matches!(self.state.phase, Phase::Closed) { + return Ok(()); + } + self.check()?; + while matches!(self.state.phase, Phase::AwaitHello | Phase::AwaitReady) { + self.next(deadline).await?; + } + self.begin_close(deadline).await?; + while !matches!(self.state.phase, Phase::Closed) { + self.next(deadline).await?; + } + Ok(()) + } + .await; + let io = self.transport.shutdown().await; + let outcome = preserve_worker_failure(outcome, io); + self.settlement = Some(outcome.clone()); + outcome + } + + fn check(&mut self) -> Result<(), RelayError> { + if let Phase::Failed(error) = &self.state.phase { + return Err(error.clone()); + } + if self.state.writing { + return Err(self.fail(RelayError::Output)); + } + if matches!(self.state.phase, Phase::Closed) { + return Err(RelayError::UnexpectedMessage); + } + Ok(()) + } + + fn fail(&mut self, error: RelayError) -> RelayError { + self.state.phase = Phase::Failed(error.clone()); + self.transport.stop(); + error + } + + async fn send(&mut self, message: Message, deadline: Instant) -> Result<(), RelayError> { + let deadline = deadline.min(after(self.config.io_budget)?); + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + self.state.writing = true; + let result = self + .transport + .send(Frame::new(self.config.environment, message), deadline) + .await; + match result { + Ok(()) => { + self.state.writing = false; + Ok(()) + } + Err(error) => Err(self.fail(error)), + } + } + + fn accept(&mut self, message: Message) -> Result, RelayError> { + match message { + Message::Hello { architecture } + if matches!(self.state.phase, Phase::AwaitHello) + && architecture == self.config.architecture => + { + self.state.phase = Phase::AwaitReady; + Ok(None) + } + Message::Ready if matches!(self.state.phase, Phase::AwaitReady) => { + self.state.phase = Phase::Initializing; + Ok(Some(ClientEvent::RelayReady)) + } + Message::BootstrapReady if matches!(self.state.phase, Phase::Initializing) => { + self.state.phase = Phase::Idle; + Ok(Some(ClientEvent::BootstrapReady)) + } + Message::InvocationResult { + attempt, + outcome, + payload, + } => match &mut self.state.phase { + Phase::Busy(active) if *active == attempt => { + self.state.delivered = true; + let result = match outcome { + Outcome::Success => { + self.state.phase = Phase::Idle; + RuntimeResult::Success(payload) + } + Outcome::FunctionError => { + self.state.phase = Phase::Retiring; + RuntimeResult::FunctionError(payload) + } + }; + Ok(Some(ClientEvent::InvocationResult { attempt, result })) + } + Phase::Cancelling { + attempt: active, + outcome_seen, + } if *active == attempt && !*outcome_seen => { + self.state.delivered = true; + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + }, + Message::InvocationTimedOut { attempt } if self.state.handler_timeout.is_some() => { + match &mut self.state.phase { + Phase::Busy(active) if *active == attempt => { + self.state.delivered = true; + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::InvocationTimedOut(attempt))) + } + Phase::Cancelling { + attempt: active, + outcome_seen, + } if *active == attempt && !*outcome_seen => { + self.state.delivered = true; + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + } + } + Message::InitError { payload } if !self.state.delivered => { + match &mut self.state.phase { + Phase::Initializing | Phase::Idle | Phase::Busy(_) => { + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::InitError(payload))) + } + Phase::Cancelling { outcome_seen, .. } if !*outcome_seen => { + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + } + } + Message::Ack(Acknowledgement::Cancel(attempt)) if matches!(self.state.phase, Phase::Cancelling { attempt: active, .. } if active == attempt) => + { + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::Cancelled(attempt))) + } + Message::Ack(Acknowledgement::Close) + if self.state.closing && !matches!(self.state.phase, Phase::Cancelling { .. }) => + { + self.state.phase = Phase::Closed; + Ok(Some(ClientEvent::Closed)) + } + _ => Err(RelayError::UnexpectedMessage), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/client/tests.rs b/crates/rustack-lambda-runtime/src/client/tests.rs new file mode 100644 index 00000000..8656a1e8 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/client/tests.rs @@ -0,0 +1,712 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncWriteExt, DuplexStream, duplex}, + net::TcpStream, + task::JoinHandle, +}; + +use super::*; +use crate::{ + http::tests::{Reply, request}, + metadata::local_trace_id, + protocol::{encode, read_frame, write_frame}, + relay::{RelayConfig, RelayHandle, start}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(30), + "arn:aws:lambda:us-east-1:000000000000:function:test:prod" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"event-secret")).unwrap(), + ) + .unwrap() +} + +struct Peer { + client: ControlClient, + environment: EnvironmentId, + input: DuplexStream, + output: DuplexStream, +} + +impl Peer { + fn new(output_size: usize) -> Self { + Self::with_limits(16, output_size, Duration::from_secs(1)) + } + + fn with_limits(input_size: usize, output_size: usize, budget: Duration) -> Self { + let environment = EnvironmentId::default(); + let config = ClientConfig::new(environment, Architecture::Arm64, budget).unwrap(); + let (reader, output) = duplex(input_size); + let (input, writer) = duplex(output_size); + let client = ControlClient::start(config, reader, writer).unwrap(); + Self { + client, + environment, + input, + output, + } + } + + async fn send(&mut self, message: Message) { + write_frame( + &mut self.output, + &Frame::new(self.environment, message), + deadline(), + ) + .await + .unwrap(); + } + + async fn read(&mut self) -> Message { + let frame = read_frame(&mut self.input, deadline()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.environment(), self.environment); + frame.into_message() + } + + async fn ready(&mut self, bootstrap: bool) { + self.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + self.send(Message::Ready).await; + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + if bootstrap { + self.send(Message::BootstrapReady).await; + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + } + } + + async fn close(mut self) { + self.client.begin_close(deadline()).await.unwrap(); + assert_eq!(self.read().await, Message::Close); + self.send(Message::Ack(Acknowledgement::Close)).await; + self.client.shutdown(deadline()).await.unwrap(); + self.client.shutdown(deadline()).await.unwrap(); + assert!(!self.client.is_idle()); + } + + async fn failed(mut self) { + assert!(self.client.shutdown(deadline()).await.is_err()); + assert!(self.client.shutdown(deadline()).await.is_err()); + assert!(!self.client.is_idle()); + } +} + +struct Harness { + client: ControlClient, + relay: RelayHandle, + api: std::net::SocketAddr, +} + +impl Harness { + async fn new() -> Self { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let environment = EnvironmentId::default(); + let relay_config = RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(2), + ) + .unwrap(); + let config = + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(); + let (writer, relay_reader) = duplex(24); + let (relay_writer, reader) = duplex(24); + let relay = start(api, relay_config, relay_reader, relay_writer).unwrap(); + let mut client = ControlClient::start(config, reader, writer).unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + assert!(!client.is_idle()); + Self { + client, + relay, + api: address, + } + } + + async fn bootstrap(&mut self) -> JoinHandle { + let address = self.api; + let poll = tokio::spawn(async move { + request( + address, + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await + }); + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + assert!(self.client.is_idle()); + poll + } + + async fn close(mut self) { + self.client.shutdown(deadline()).await.unwrap(); + self.relay.wait().await.unwrap(); + assert!(TcpStream::connect(self.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_pair_client_with_real_relay_http_and_reuse_matching_attempts() { + let mut harness = Harness::new().await; + let mut poll = Some(harness.bootstrap().await); + for index in 0..3 { + let poll = poll.take().unwrap_or_else(|| { + let address = harness.api; + tokio::spawn(async move { + request( + address, + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await + }) + }); + let call = invocation(); + let attempt = call.parts().0; + harness.client.invoke(call, deadline()).await.unwrap(); + let event = poll.await.unwrap(); + assert_eq!(event.status, 200); + assert_eq!(event.body, b"event-secret"); + let suffix = if index == 2 { "error" } else { "response" }; + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/{suffix}"), + "", + b"computed-secret" + ) + .await + .status, + 202 + ); + let expected = Payload::try_from(Bytes::from_static(b"computed-secret")).unwrap(); + let result = if index == 2 { + RuntimeResult::FunctionError(expected) + } else { + RuntimeResult::Success(expected) + }; + assert_eq!( + harness.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { attempt, result } + ); + assert_eq!(harness.client.is_idle(), index != 2); + } + harness.close().await; +} + +#[tokio::test] +async fn test_should_forward_init_failure_before_bootstrap_and_close_both_owners() { + let mut harness = Harness::new().await; + assert_eq!( + request( + harness.api, + "POST", + "/2018-06-01/runtime/init/error", + "", + b"init-secret" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.client.next(deadline()).await.unwrap(), ClientEvent::InitError(payload) if payload.as_ref() == b"init-secret") + ); + assert!(!harness.client.is_idle()); + harness.close().await; +} + +#[tokio::test] +async fn test_should_reject_foreign_scope_architecture_and_out_of_order_handshakes() { + for mode in 0..6 { + let mut peer = Peer::new(8192); + match mode { + 0 => write_frame( + &mut peer.output, + &Frame::new(EnvironmentId::default(), Message::Ready), + deadline(), + ) + .await + .unwrap(), + 1 => { + peer.send(Message::Hello { + architecture: Architecture::X86_64, + }) + .await; + } + 2 => peer.send(Message::Ready).await, + 3 => peer.send(Message::BootstrapReady).await, + 4 => peer.send(Message::Close).await, + _ => { + peer.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + peer.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + } + } + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_reject_foreign_and_duplicate_results_and_cancel_ack_ids() { + for mode in 0..3 { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + assert!(matches!(peer.read().await, Message::Invoke { attempt: id, .. } if id == attempt)); + let outcome = Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::from_static(b"result")).unwrap(), + }; + match mode { + 0 => { + peer.send(Message::InvocationResult { + attempt: AttemptId::default(), + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + } + 1 => { + peer.send(outcome.clone()).await; + assert!(matches!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { .. } + )); + peer.send(outcome).await; + } + _ => { + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(AttemptId::default()))) + .await; + } + } + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_discard_only_one_late_result_during_cancellation_and_require_matching_ack() { + for completed in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::from_static(b"late")).unwrap(), + }) + .await; + if completed { + assert!(matches!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { .. } + )); + } + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + assert!(!peer.client.is_idle()); + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + peer.close().await; + } +} + +#[tokio::test] +async fn test_should_validate_pending_events_before_close_and_reject_early_close_ack() { + for mode in 0..4 { + let mut peer = Peer::new(8192); + peer.ready(mode != 0).await; + let call = invocation(); + let attempt = call.parts().0; + if mode != 0 { + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + } + if mode >= 2 { + peer.client.cancel(attempt, deadline()).await.unwrap(); + peer.read().await; + } + peer.client.begin_close(deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Close); + let preceding = match mode { + 0 => Message::BootstrapReady, + 1 => Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }, + _ => Message::Ack(Acknowledgement::Cancel(attempt)), + }; + if mode == 3 { + peer.send(Message::Ack(Acknowledgement::Close)).await; + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } else { + peer.send(preceding).await; + peer.send(Message::Ack(Acknowledgement::Close)).await; + peer.client.shutdown(deadline()).await.unwrap(); + } + } +} + +#[tokio::test] +async fn test_should_keep_partial_framing_when_a_next_waiter_is_cancelled() { + let mut peer = Peer::new(8192); + peer.ready(false).await; + let bytes = encode(&Frame::new(peer.environment, Message::BootstrapReady)).unwrap(); + peer.output + .write_all(bytes.get(..2).unwrap()) + .await + .unwrap(); + let mut waiting = Box::pin(peer.client.next(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + peer.output + .write_all(bytes.get(2..).unwrap()) + .await + .unwrap(); + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + peer.close().await; +} + +#[tokio::test] +async fn test_should_fail_closed_after_a_cancelled_send_without_losing_io_ownership() { + let mut peer = Peer::new(1); + peer.ready(true).await; + let mut sending = Box::pin(peer.client.invoke(invocation(), deadline())); + assert!( + sending + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(sending); + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::Output) + )); + peer.failed().await; +} + +#[tokio::test] +async fn test_should_resume_close_after_its_waiter_is_dropped() { + let mut peer = Peer::new(8192); + peer.ready(false).await; + peer.client.begin_close(deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Close); + let mut waiting = Box::pin(peer.client.shutdown(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + peer.send(Message::Ack(Acknowledgement::Close)).await; + peer.client.shutdown(deadline()).await.unwrap(); + peer.client.shutdown(deadline()).await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_partial_frames_and_cache_receive_deadline_failures() { + for partial in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(false).await; + if partial { + peer.output.write_all(&[0]).await.unwrap(); + } + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_reject_local_invocation_before_bootstrap_and_reused_attempt_identity() { + for bootstrap in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(bootstrap).await; + let call = invocation(); + if bootstrap { + let attempt = call.parts().0; + peer.client.invoke(call.clone(), deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + peer.client.next(deadline()).await.unwrap(); + } + assert!(peer.client.invoke(call, deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_a_queued_frame_when_a_pending_waiter_resumes_after_its_deadline() { + let mut peer = Peer::with_limits(1, 8192, Duration::from_secs(60)); + peer.ready(false).await; + let expires = Instant::now() + Duration::from_secs(1); + let mut waiting = Box::pin(peer.client.next(expires)); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + write_frame( + &mut peer.output, + &Frame::new(peer.environment, Message::BootstrapReady), + deadline(), + ) + .await + .unwrap(); + // Capacity one forces the reader to finish the preceding frame before this + // byte fits. Its next partial-frame budget is sixty seconds, beyond the + // caller's one-second deadline, so a valid queued frame wins an inner poll. + peer.output.write_all(&[0]).await.unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!(waiting.await, Err(RelayError::Deadline))); + peer.failed().await; +} + +#[tokio::test] +async fn test_should_reject_initialization_errors_after_a_result_proves_delivery() { + for mode in 0..4 { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let mut attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + peer.client.next(deadline()).await.unwrap(); + if mode == 1 || mode == 3 { + let next = invocation(); + attempt = next.parts().0; + peer.client.invoke(next, deadline()).await.unwrap(); + peer.read().await; + } + if mode >= 2 { + peer.client.cancel(attempt, deadline()).await.unwrap(); + peer.read().await; + } + peer.send(Message::InitError { + payload: Payload::try_from(Bytes::from_static(b"late-init-secret")).unwrap(), + }) + .await; + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + peer.failed().await; + } +} + +#[test] +fn test_should_validate_client_io_budget() { + for budget in [Duration::ZERO, Duration::from_secs(61)] { + assert!(ClientConfig::new(EnvironmentId::default(), Architecture::Arm64, budget).is_err()); + } +} + +#[tokio::test] +async fn test_should_preserve_transport_worker_failure_after_a_prior_scope_error() { + let environment = EnvironmentId::default(); + let (reader, mut peer) = duplex(4096); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + tokio::io::sink(), + ) + .unwrap(); + peer.write_all( + &encode(&Frame::new( + EnvironmentId::default(), + Message::Hello { + architecture: Architecture::Arm64, + }, + )) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + client.next(deadline()).await, + Err(RelayError::Scope) + )); + client.transport.abort_owner(); + assert!(matches!( + client.shutdown(deadline()).await, + Err(RelayError::Worker) + )); + assert!(matches!( + client.shutdown(deadline()).await, + Err(RelayError::Worker) + )); +} + +#[rstest::rstest] +#[case::idle("idle")] +#[case::initializing("initializing")] +#[case::without_budget("without_budget")] +#[case::foreign_attempt("foreign")] +#[case::duplicate("duplicate")] +#[case::init_after_timeout("init_after_timeout")] +#[tokio::test] +async fn test_should_reject_impossible_handler_timeout_events(#[case] condition: &str) { + let mut peer = Peer::new(4096); + peer.ready(condition != "initializing").await; + let call = invocation(); + let attempt = call.parts().0; + if !matches!(condition, "idle" | "initializing") { + let call = if condition == "without_budget" { + call + } else { + call.with_handler_timeout( + crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap(), + ) + }; + peer.client.invoke(call, deadline()).await.unwrap(); + let sent = peer.read().await; + assert!( + matches!( + sent, + Message::Invoke { + handler_timeout_ms: Some(_), + .. + } + ) || condition == "without_budget" + ); + } + let reported = if condition == "foreign" { + AttemptId::default() + } else { + attempt + }; + peer.send(Message::InvocationTimedOut { attempt: reported }) + .await; + if matches!(condition, "duplicate" | "init_after_timeout") { + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationTimedOut(attempt) + ); + assert!(!peer.client.is_idle()); + let message = if condition == "duplicate" { + Message::InvocationTimedOut { attempt } + } else { + Message::InitError { + payload: Payload::try_from(Bytes::new()).unwrap(), + } + }; + peer.send(message).await; + } + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + assert!(peer.client.shutdown(deadline()).await.is_err()); +} +#[tokio::test] +async fn test_should_discard_only_one_matching_handler_timeout_during_cancel_before_ack() { + let mut peer = Peer::new(4096); + peer.ready(true).await; + let call = invocation().with_handler_timeout( + crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap(), + ); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::InvocationTimedOut { attempt }).await; + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + assert!(!peer.client.is_idle()); + peer.close().await; +} diff --git a/crates/rustack-lambda-runtime/src/diagnostics.rs b/crates/rustack-lambda-runtime/src/diagnostics.rs new file mode 100644 index 00000000..ecdc868c --- /dev/null +++ b/crates/rustack-lambda-runtime/src/diagnostics.rs @@ -0,0 +1,147 @@ +//! Mandatory per-task dependency filter, independent of application `RUST_LOG`. +//! +//! Hyper's chunk decoder can put partial trailer bytes in TRACE fields. Keep +//! its verbose events/spans out before delegating to the application's subscriber. + +use tracing_core::{ + Dispatch, Event, Interest, Level, Metadata, Subscriber, dispatcher, + span::{Attributes, Current, Id, Record}, +}; + +#[derive(Debug)] +struct RuntimeSubscriber { + parent: Dispatch, +} + +fn allowed(metadata: &Metadata<'_>) -> bool { + let target = metadata.target(); + let tls = ["rustls", "tokio_rustls"].iter().any(|name| { + target == *name + || target + .strip_prefix(name) + .is_some_and(|suffix| suffix.starts_with("::")) + }); + if tls { + return false; + } + let dependency = ["hyper", "hyper_util", "bollard"].iter().any(|name| { + target == *name + || target + .strip_prefix(name) + .is_some_and(|suffix| suffix.starts_with("::")) + }); + !dependency || *metadata.level() <= Level::INFO +} + +impl Subscriber for RuntimeSubscriber { + fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest { + if allowed(metadata) { + self.parent.register_callsite(metadata) + } else { + Interest::never() + } + } + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + allowed(metadata) && self.parent.enabled(metadata) + } + fn new_span(&self, attributes: &Attributes<'_>) -> Id { + self.parent.new_span(attributes) + } + fn record(&self, id: &Id, values: &Record<'_>) { + self.parent.record(id, values); + } + fn record_follows_from(&self, id: &Id, follows: &Id) { + self.parent.record_follows_from(id, follows); + } + fn event(&self, event: &Event<'_>) { + // Recheck even when a different live dispatcher enabled this callsite. + if allowed(event.metadata()) { + self.parent.event(event); + } + } + fn enter(&self, id: &Id) { + self.parent.enter(id); + } + fn exit(&self, id: &Id) { + self.parent.exit(id); + } + fn clone_span(&self, id: &Id) -> Id { + self.parent.clone_span(id) + } + fn try_close(&self, id: Id) -> bool { + self.parent.try_close(id) + } + fn current_span(&self) -> Current { + self.parent.current_span() + } +} + +/// Capture application context while denying verbose transport dependency logs. +/// +/// The policy covers verbose Hyper/`hyper_util`/Bollard and all rustls/ +/// tokio-rustls events independently of `RUST_LOG`; TLS diagnostics may contain +/// operator server names or peer certificate data even at warning levels. +/// Apply the dispatch to construction and every poll/spawned transport worker. +/// +/// ``` +/// let dispatch = rustack_lambda_runtime::diagnostics::runtime_dispatch(); +/// tracing::dispatcher::with_default(&dispatch, || tracing::debug!("application event")); +/// ``` +#[must_use] +pub fn runtime_dispatch() -> Dispatch { + dispatcher::get_default(|parent| { + Dispatch::new(RuntimeSubscriber { + parent: parent.clone(), + }) + }) +} + +#[cfg(test)] +mod tests { + use std::{ + io::{self, Write}, + sync::{Arc, Mutex}, + }; + + use tracing::{Level, dispatcher}; + + use super::runtime_dispatch; + + #[derive(Debug, Clone)] + struct Capture(Arc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self.0.lock().unwrap(); + if output.len().saturating_add(bytes.len()) > 16_384 { + return Err(io::Error::other("fixture capture exceeded budget")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + #[test] + fn test_should_filter_all_tls_peer_events_while_preserving_application_diagnostics() { + let capture = Capture(Arc::default()); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + tracing::subscriber::with_default(subscriber, || { + let dispatch = runtime_dispatch(); + dispatcher::with_default(&dispatch, || { + tracing::trace!(target:"rustls","PRIVATE-TLS-TRACE"); + tracing::warn!(target:"rustls::client","PRIVATE-PEER-CERT"); + tracing::error!(target:"tokio_rustls","PRIVATE-SNI"); + tracing::info!("owned_application_event"); + }); + }); + let output = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); + assert!(output.contains("owned_application_event")); + assert!(!output.contains("PRIVATE")); + } +} diff --git a/crates/rustack-lambda-runtime/src/http.rs b/crates/rustack-lambda-runtime/src/http.rs new file mode 100644 index 00000000..76948018 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/http.rs @@ -0,0 +1,602 @@ +//! Loopback Runtime API adapter backed by the shared session actor. +//! +//! One supervisor owns the session, listener and bounded HTTP connection tasks. +//! Infrastructure diagnostics never include peer targets, headers or payloads. + +use std::{ + convert::Infallible, + io, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, + time::Duration, +}; + +use ::http::{ + HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri, + header::{CONNECTION, CONTENT_LENGTH, CONTENT_TYPE}, +}; +use bytes::Bytes; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::{body::Incoming, server::conn::http1, service::service_fn}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use thiserror::Error; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::{Semaphore, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use crate::{ + diagnostics::runtime_dispatch, + metadata::deadline_unix_ms, + protocol::{AttemptId, MAX_PAYLOAD_BYTES, Outcome, Payload}, + session::{Invocation, RuntimeSession, SessionError, SessionHandle}, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const HEADER_BYTES: usize = 16 * 1024; +const HEADER_COUNT: usize = 64; +const HEADER_VALUE_BYTES: usize = 256; +const XRAY_ERROR_CAUSE_BYTES: usize = 8 * 1024; +const XRAY_ERROR_CAUSE: &str = "lambda-runtime-function-xray-error-cause"; +const TARGET_BYTES: usize = 256; +const CONNECTIONS: usize = 8; +const BODY_READERS: usize = 4; +const INVOCATION_ID: &str = "lambda-runtime-invocation-id"; + +/// Adapter/supervision failures without invalid peer data in their diagnostics. +#[derive(Debug, Clone, Error)] +#[non_exhaustive] +pub enum ApiError { + /// Listener IO failure. + #[error("runtime API listener IO failed")] + Io(#[source] Arc), + /// Shared session failed. + #[error(transparent)] + Session(#[from] SessionError), + /// A supervised HTTP task failed or exceeded graceful shutdown. + #[error("runtime API connection failed")] + ConnectionFailed, + /// The server task failed to join. + #[error("runtime API supervisor failed")] + SupervisorFailed, + /// Runtime API listeners must remain environment-local. + #[error("runtime API requires a loopback address")] + NonLoopback, +} + +impl From for ApiError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} + +#[derive(Debug)] +struct ServerTask { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +impl ServerTask { + async fn shutdown(&mut self) -> Result<(), ApiError> { + self.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| ApiError::SupervisorFailed) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settlement = Some(result.clone()); + result + } +} + +impl Drop for ServerTask { + fn drop(&mut self) { + // Fallback: the retained supervisor still owns its session/connections. + // Normal shutdown joins this handle before the parent Tokio runtime stops. + self.stop.send_replace(true); + } +} + +/// Owned loopback listener and shared-session command port. +#[derive(Debug)] +#[non_exhaustive] +pub struct RuntimeApiHandle { + addr: SocketAddr, + session: SessionHandle, + server: ServerTask, + initialization_error: Option>, +} + +impl RuntimeApiHandle { + /// Bound listener address for `AWS_LAMBDA_RUNTIME_API`. + /// + /// ``` + /// # fn inspect(api: &rustack_lambda_runtime::http::RuntimeApiHandle) { + /// assert!(api.addr().ip().is_loopback()); + /// # } + /// ``` + #[must_use] + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Shared session port used by the executor or control relay. + /// + /// ``` + /// # fn inspect(api: &rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let session = api.session().clone(); + /// # } + /// ``` + #[must_use] + pub fn session(&self) -> &SessionHandle { + &self.session + } + + /// Transfer the initialization-error receiver at most once. + /// + /// ``` + /// # fn inspect(api: &mut rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let notification = api.take_initialization_error(); + /// assert!(api.take_initialization_error().is_none()); + /// # } + /// ``` + pub fn take_initialization_error(&mut self) -> Option> { + self.initialization_error.take() + } + + /// Stop admission and join all connection/session tasks. + /// + /// # Errors + /// Returns cached IO/session/supervisor failures; dropping a waiter retains ownership. + /// + /// ```no_run + /// # async fn stop(api: &mut rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let clean = api.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), ApiError> { + self.server.shutdown().await + } +} + +/// Start the default Native loopback adapter with a ten-minute idle lease. +/// +/// # Errors +/// Returns listener/session startup failures. +/// +/// ```no_run +/// # async fn example() -> Result<(), rustack_lambda_runtime::http::ApiError> { +/// let mut api = rustack_lambda_runtime::http::start().await?; +/// api.shutdown().await?; +/// # Ok(()) } +/// ``` +pub async fn start() -> Result { + start_at( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + Duration::from_mins(10), + ) + .await +} + +/// Bind an explicit loopback endpoint, shared by Native and the relay. +/// +/// # Errors +/// Rejects non-loopback addresses or listener/session startup failures. +/// +/// ```no_run +/// # async fn example() -> Result<(), rustack_lambda_runtime::http::ApiError> { +/// let mut api = rustack_lambda_runtime::http::start_at("127.0.0.1:9001".parse().map_err(|_| rustack_lambda_runtime::http::ApiError::NonLoopback)?, std::time::Duration::from_secs(60)).await?; +/// api.shutdown().await?; +/// # Ok(()) } +/// ``` +#[tracing::instrument(skip_all)] +pub async fn start_at( + addr: SocketAddr, + idle_lease: Duration, +) -> Result { + start_with_body_budget(addr, idle_lease, Arc::new(Semaphore::new(BODY_READERS))).await +} + +async fn start_with_body_budget( + addr: SocketAddr, + idle_lease: Duration, + body_budget: Arc, +) -> Result { + if !addr.ip().is_loopback() { + return Err(ApiError::NonLoopback); + } + let listener = timeout(REQUEST_TIMEOUT, TcpListener::bind(addr)) + .await + .map_err(|_| ApiError::ConnectionFailed)??; + let addr = listener.local_addr()?; + let mut owner = RuntimeSession::start(idle_lease)?; + let session = owner.handle().clone(); + let initialization_error = owner.take_initialization_error(); + let (stop, _) = watch::channel(false); + let task = tokio::spawn( + serve(listener, owner, idle_lease, stop.clone(), body_budget) + .with_subscriber(runtime_dispatch()), + ); + Ok(RuntimeApiHandle { + addr, + session, + server: ServerTask { + stop, + task: Some(task), + settlement: None, + }, + initialization_error, + }) +} + +async fn serve( + listener: TcpListener, + mut owner: RuntimeSession, + idle_lease: Duration, + stop: watch::Sender, + body_budget: Arc, +) -> Result<(), ApiError> { + let session = owner.handle().clone(); + let mut stopped = stop.subscribe(); + let mut connections = JoinSet::new(); + let mut failure = None; + loop { + tokio::select! { + biased; + () = async { let _ = stopped.wait_for(|stopped| *stopped).await; } => break, + () = session.closed() => break, + completed = connections.join_next(), if !connections.is_empty() => { + if !matches!(completed, Some(Ok(Ok(())))) { failure = Some(ApiError::ConnectionFailed); break; } + } + accepted = listener.accept() => { + match accepted { + Ok((socket, _)) if connections.len() < CONNECTIONS => { + let session = session.clone(); + let stop = stop.subscribe(); + let body_budget = Arc::clone(&body_budget); + connections.spawn(connection(socket, session, idle_lease, stop, body_budget).with_current_subscriber()); + } + Ok(_) => {} + Err(error) => { failure = Some(ApiError::from(error)); break; } + } + } + } + } + drop(listener); + stop.send_replace(true); + session.close(); + while let Some(result) = connections.join_next().await { + if !matches!(result, Ok(Ok(()))) { + failure.get_or_insert(ApiError::ConnectionFailed); + } + } + if let Err(error) = owner.shutdown().await { + failure.get_or_insert(ApiError::Session(error)); + } + failure.map_or(Ok(()), Err) +} + +#[tracing::instrument(skip_all)] +async fn connection( + socket: TcpStream, + session: SessionHandle, + idle_lease: Duration, + mut stop: watch::Receiver, + body_budget: Arc, +) -> Result<(), ApiError> { + let service_session = session.clone(); + let service = service_fn(move |request| { + handle( + service_session.clone(), + idle_lease, + request, + Arc::clone(&body_budget), + ) + }); + let mut builder = http1::Builder::new(); + builder + .timer(TokioTimer::new()) + .header_read_timeout(REQUEST_TIMEOUT) + .max_headers(HEADER_COUNT) + .max_buf_size(HEADER_BYTES); + let connection = builder.serve_connection(TokioIo::new(socket), service); + tokio::pin!(connection); + let result = tokio::select! { + biased; + () = async { let _ = stop.wait_for(|stopped| *stopped).await; } => { + connection.as_mut().graceful_shutdown(); + timeout(REQUEST_TIMEOUT, &mut connection).await.map_err(|_| ApiError::ConnectionFailed)? + } + result = &mut connection => result, + }; + // Rejected HTTP syntax and idle header timeouts close only that connection. + // The session bounds long polls/writes; genuine IO failures invalidate it. + match result { + Ok(()) => Ok(()), + Err(error) if error.is_timeout() || error.is_parse() => Ok(()), + Err(_) => { + session.close(); + Err(ApiError::ConnectionFailed) + } + } +} + +#[derive(Debug, Clone, Copy)] +enum Route { + Next, + Complete { + attempt: AttemptId, + outcome: Outcome, + }, + InitError, +} + +fn route(method: &Method, target: &Uri) -> Result { + if target + .path_and_query() + .is_some_and(|value| value.as_str().len() > TARGET_BYTES) + || target.scheme().is_some() + || target.authority().is_some() + || target.query().is_some() + { + return Err(StatusCode::BAD_REQUEST); + } + let path = target.path(); + if path == "/2018-06-01/runtime/invocation/next" { + return if method == Method::GET { + Ok(Route::Next) + } else { + Err(StatusCode::METHOD_NOT_ALLOWED) + }; + } + if path == "/2018-06-01/runtime/init/error" { + return if method == Method::POST { + Ok(Route::InitError) + } else { + Err(StatusCode::METHOD_NOT_ALLOWED) + }; + } + let rest = path + .strip_prefix("/2018-06-01/runtime/invocation/") + .ok_or(StatusCode::NOT_FOUND)?; + let (attempt, suffix) = rest.split_once('/').ok_or(StatusCode::NOT_FOUND)?; + let outcome = match suffix { + "response" => Outcome::Success, + "error" => Outcome::FunctionError, + _ => return Err(StatusCode::NOT_FOUND), + }; + if method != Method::POST { + return Err(StatusCode::METHOD_NOT_ALLOWED); + } + Ok(Route::Complete { + attempt: attempt.parse().map_err(|_| StatusCode::BAD_REQUEST)?, + outcome, + }) +} + +fn validate_headers(headers: &HeaderMap, route: Option<&Route>) -> Result<(), StatusCode> { + if headers.len() > HEADER_COUNT || headers.get_all(XRAY_ERROR_CAUSE).iter().count() > 1 { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + let error_route = matches!( + route, + Some(Route::Complete { + outcome: Outcome::FunctionError, + .. + }) + ); + let mut total = 0usize; + for (name, value) in headers { + let xray_cause = error_route && name == XRAY_ERROR_CAUSE; + let limit = if xray_cause { + XRAY_ERROR_CAUSE_BYTES + } else { + HEADER_VALUE_BYTES + }; + if value.as_bytes().len() > limit + || !value.as_bytes().iter().all(|byte| { + if xray_cause { + *byte >= 32 && *byte != 127 + } else { + (32..=126).contains(byte) + } + }) + { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + total = total + .checked_add(name.as_str().len()) + .and_then(|n| n.checked_add(value.as_bytes().len())) + .ok_or(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE)?; + if total > HEADER_BYTES { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + } + Ok(()) +} + +async fn handle( + session: SessionHandle, + idle_lease: Duration, + request: Request, + body_budget: Arc, +) -> Result>, Infallible> { + Ok(dispatch(&session, idle_lease, request, &body_budget) + .await + .unwrap_or_else(error_response)) +} + +async fn dispatch( + session: &SessionHandle, + idle_lease: Duration, + request: Request, + body_budget: &Arc, +) -> Result>, StatusCode> { + let route = route(request.method(), request.uri()); + validate_headers(request.headers(), route.as_ref().ok())?; + let route = route?; + let invocation_id = if matches!(route, Route::Complete { .. }) { + match parse_invocation_id(request.headers()) { + Ok(id) => id, + Err(_) => return Ok(invalid_invocation_id()), + } + } else { + None + }; + if matches!(route, Route::Complete { attempt, .. } if invocation_id.is_some_and(|id| id != attempt)) + { + return Ok(invalid_invocation_id()); + } + let limit = if matches!(route, Route::Next) { + 0 + } else { + MAX_PAYLOAD_BYTES + }; + if request + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > limit as u64) + { + return Err(StatusCode::PAYLOAD_TOO_LARGE); + } + let body_permit = if matches!(route, Route::Next) { + None + } else { + Some( + body_budget + .clone() + .try_acquire_owned() + .map_err(|_| StatusCode::TOO_MANY_REQUESTS)?, + ) + }; + let deadline = Instant::now() + .checked_add(REQUEST_TIMEOUT) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let body = timeout_at(deadline, Limited::new(request.into_body(), limit).collect()) + .await + .map_err(|_| StatusCode::REQUEST_TIMEOUT)? + .map_err(|error| { + if error.is::() { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + } + })?; + if body.trailers().is_some() { + return Err(StatusCode::BAD_REQUEST); + } + let payload = Payload::try_from(body.to_bytes()).map_err(|_| StatusCode::PAYLOAD_TOO_LARGE)?; + match route { + Route::Next => { + let deadline = Instant::now() + .checked_add(idle_lease) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let invocation = session.next(deadline).await.map_err(status)?; + match invocation_response(&invocation) { + Ok(response) => Ok(response), + Err(error) => { + session.close(); + Err(error) + } + } + } + Route::Complete { attempt, outcome } => { + let completed = session + .complete_with_body_permit( + attempt, + invocation_id, + outcome, + payload, + deadline, + body_permit, + ) + .await; + if matches!(completed, Err(SessionError::UnknownAttempt)) && invocation_id.is_some() { + return Ok(invalid_invocation_id()); + } + completed.map_err(status)?; + Ok(response(StatusCode::ACCEPTED, Bytes::new(), false)) + } + Route::InitError => { + session + .initialization_error_with_body_permit(payload, deadline, body_permit) + .await + .map_err(status)?; + Ok(response(StatusCode::ACCEPTED, Bytes::new(), false)) + } + } +} + +fn parse_invocation_id(headers: &HeaderMap) -> Result, StatusCode> { + if headers.get_all(INVOCATION_ID).iter().count() > 1 { + return Err(StatusCode::BAD_REQUEST); + } + headers + .get(INVOCATION_ID) + .map(|value| { + value + .to_str() + .ok() + .and_then(|value| value.parse().ok()) + .ok_or(StatusCode::BAD_REQUEST) + }) + .transpose() +} + +fn invocation_response(invocation: &Invocation) -> Result>, StatusCode> { + let (attempt, deadline, arn, trace, payload) = invocation.parts(); + let deadline = deadline_unix_ms(deadline).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Response::builder() + .status(StatusCode::OK) + .header("Lambda-Runtime-Aws-Request-Id", attempt.to_string()) + .header(INVOCATION_ID, attempt.to_string()) + .header("Lambda-Runtime-Deadline-Ms", deadline.get().to_string()) + .header("Lambda-Runtime-Invoked-Function-Arn", arn.as_ref()) + .header("Lambda-Runtime-Trace-Id", trace.as_ref()) + .header(CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(payload.clone()))) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) +} + +fn status(error: SessionError) -> StatusCode { + match error { + SessionError::Busy => StatusCode::CONFLICT, + SessionError::UnknownAttempt | SessionError::InvalidMetadata => StatusCode::BAD_REQUEST, + SessionError::Deadline | SessionError::HandlerTimeout => StatusCode::REQUEST_TIMEOUT, + _ => StatusCode::SERVICE_UNAVAILABLE, + } +} + +fn response(status: StatusCode, body: Bytes, close: bool) -> Response> { + let mut response = Response::new(Full::new(body)); + *response.status_mut() = status; + if close { + response + .headers_mut() + .insert(CONNECTION, HeaderValue::from_static("close")); + } + response +} + +fn error_response(status: StatusCode) -> Response> { + response(status, Bytes::new(), true) +} + +fn invalid_invocation_id() -> Response> { + response(StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"errorType":"InvalidInvocationId","errorMessage":"Invocation ID does not match active invocation"}"#), true) +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/crates/rustack-lambda-runtime/src/http/tests.rs b/crates/rustack-lambda-runtime/src/http/tests.rs new file mode 100644 index 00000000..4e671082 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/http/tests.rs @@ -0,0 +1,901 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use rstest::rstest; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, +}; + +use super::*; +use crate::{metadata::local_trace_id, session::RuntimeResult}; + +#[tokio::test] +async fn test_should_retain_five_completed_keepalive_connections_for_default_node_ric() { + let cutoff = Instant::now() + Duration::from_secs(4); + let mut api = start().await.unwrap(); + let mut retained = Vec::new(); + for _ in 0..5 { + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + api.session().next(call.parts().1).await.unwrap(); + let mut stream = TcpStream::connect(api.addr()).await.unwrap(); + let request = format!( + "POST /2018-06-01/runtime/invocation/{}/response HTTP/1.1\r\nHost: \ + {}\r\nContent-Length: 0\r\n\r\n", + ticket.attempt(), + api.addr() + ); + stream.write_all(request.as_bytes()).await.unwrap(); + let head = timeout_at(cutoff, async { + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + assert!(head.len() < 16384); + head.push(stream.read_u8().await.unwrap()); + } + String::from_utf8(head).unwrap() + }) + .await + .unwrap(); + assert!(head.starts_with("HTTP/1.1 202")); + assert!(!head.to_ascii_lowercase().contains("connection: close")); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::Success(payload) if payload.as_ref().is_empty()) + ); + retained.push(stream); + } + assert!( + Instant::now() < cutoff, + "keepalive proof exceeded the original pre-idle-timeout cutoff" + ); + assert_eq!(retained.len(), 5); + drop(retained); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_bound_four_actual_body_readers_without_consuming_the_current_attempt() { + let body_budget = Arc::new(Semaphore::new(4)); + let mut api = start_with_body_budget( + "127.0.0.1:0".parse().unwrap(), + Duration::from_secs(60), + Arc::clone(&body_budget), + ) + .await + .unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + api.session().next(call.parts().1).await.unwrap(); + let mut held = Vec::new(); + let foreign = AttemptId::default(); + for _ in 0..4 { + let mut stream = TcpStream::connect(api.addr()).await.unwrap(); + stream + .write_all( + format!( + "POST /2018-06-01/runtime/invocation/{foreign}/error HTTP/1.1\r\nHost: \ + {}\r\nConnection: close\r\nContent-Length: 1\r\n\r\n", + api.addr() + ) + .as_bytes(), + ) + .await + .unwrap(); + held.push(stream); + } + timeout(Duration::from_secs(2), async { + while body_budget.available_permits() != 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + let path = format!( + "/2018-06-01/runtime/invocation/{}/response", + ticket.attempt() + ); + assert_eq!( + request(api.addr(), "POST", &path, "", b"not-accepted") + .await + .status, + 429 + ); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"" + ) + .await + .status, + 409 + ); + for mut stream in held { + stream.write_all(b"x").await.unwrap(); + let mut reply = Vec::new(); + timeout( + Duration::from_secs(3), + stream.take(16385).read_to_end(&mut reply), + ) + .await + .unwrap() + .unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 400")); + } + assert_eq!(body_budget.available_permits(), 4); + assert_eq!( + request(api.addr(), "POST", &path, "", b"original-result") + .await + .status, + 202 + ); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::Success(payload) if payload.as_ref()==b"original-result") + ); + api.shutdown().await.unwrap(); +} + +#[rstest] +#[case(8192, true)] +#[case(8193, false)] +fn test_should_bound_the_exact_xray_error_cause(#[case] bytes: usize, #[case] accepted: bool) { + let route = Route::Complete { + attempt: AttemptId::default(), + outcome: Outcome::FunctionError, + }; + let mut headers = HeaderMap::new(); + headers.insert( + XRAY_ERROR_CAUSE, + HeaderValue::from_str(&"x".repeat(bytes)).unwrap(), + ); + assert_eq!(validate_headers(&headers, Some(&route)).is_ok(), accepted); +} + +#[rstest] +#[case("GET", "/2018-06-01/runtime/invocation/next", XRAY_ERROR_CAUSE)] +#[case("POST", "/2018-06-01/runtime/init/error", XRAY_ERROR_CAUSE)] +#[case( + "POST", + "/2018-06-01/runtime/invocation/12345678-1234-4234-8234-123456789012/response", + XRAY_ERROR_CAUSE +)] +#[case( + "GET", + "/2018-06-01/runtime/invocation/12345678-1234-4234-8234-123456789012/error", + XRAY_ERROR_CAUSE +)] +#[case( + "POST", + "/2018-06-01/runtime/invocation/not-a-uuid/error", + XRAY_ERROR_CAUSE +)] +#[case( + "POST", + "/2018-06-01/runtime/invocation/12345678-1234-4234-8234-123456789012/error", + "x-xray-error-cause" +)] +fn test_should_not_grant_xray_allowance_to_other_headers_or_routes( + #[case] method: &str, + #[case] path: &str, + #[case] name: &'static str, +) { + let parsed = route(&method.parse().unwrap(), &path.parse().unwrap()); + let mut headers = HeaderMap::new(); + headers.insert(name, HeaderValue::from_str(&"x".repeat(257)).unwrap()); + assert_eq!( + validate_headers(&headers, parsed.as_ref().ok()), + Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE) + ); +} + +#[test] +fn test_should_bound_xray_opaque_bytes_duplicates_count_and_aggregate() { + let route = Route::Complete { + attempt: AttemptId::default(), + outcome: Outcome::FunctionError, + }; + let mut headers = HeaderMap::new(); + headers.insert( + XRAY_ERROR_CAUSE, + HeaderValue::from_bytes(&[0x80, 0xff]).unwrap(), + ); + assert!(validate_headers(&headers, Some(&route)).is_ok()); + headers.insert(XRAY_ERROR_CAUSE, HeaderValue::from_bytes(b"x\tx").unwrap()); + assert!(validate_headers(&headers, Some(&route)).is_err()); + headers.insert(XRAY_ERROR_CAUSE, HeaderValue::from_static("first")); + headers.append(XRAY_ERROR_CAUSE, HeaderValue::from_static("second")); + assert!(validate_headers(&headers, Some(&route)).is_err()); + headers.clear(); + headers.insert( + XRAY_ERROR_CAUSE, + HeaderValue::from_str(&"x".repeat(8192)).unwrap(), + ); + for _ in 0..30 { + headers.append("x-other", HeaderValue::from_str(&"o".repeat(256)).unwrap()); + } + headers.append("x-other", HeaderValue::from_str(&"o".repeat(255)).unwrap()); + assert!(validate_headers(&headers, Some(&route)).is_ok()); + headers.append("x-other", HeaderValue::from_str(&"o".repeat(256)).unwrap()); + assert_eq!( + validate_headers(&headers, Some(&route)), + Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE) + ); + headers.clear(); + for _ in 0..65 { + headers.append("x-other", HeaderValue::from_static("")); + } + assert_eq!( + validate_headers(&headers, Some(&route)), + Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE) + ); +} + +#[tokio::test] +async fn test_should_preserve_attempt_through_rejected_xray_then_deliver_original_error() { + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"" + ) + .await + .status, + 200 + ); + let path = format!("/2018-06-01/runtime/invocation/{}/error", ticket.attempt()); + for (headers, expected) in [ + (format!("{XRAY_ERROR_CAUSE}: {}\r\n", "x".repeat(8193)), 431), + ( + format!("{XRAY_ERROR_CAUSE}: x\r\n{XRAY_ERROR_CAUSE}: y\r\n"), + 431, + ), + (format!("{XRAY_ERROR_CAUSE}: x\tx\r\n"), 431), + (format!("{XRAY_ERROR_CAUSE}: x\x7fx\r\n"), 400), + (format!("{XRAY_ERROR_CAUSE}: x\0x\r\n"), 400), + ] { + assert_eq!( + request(api.addr(), "POST", &path, &headers, b"rejected-error") + .await + .status, + expected + ); + } + let cause = format!( + "{XRAY_ERROR_CAUSE}: {}诊断\r\n", + "private-xray-cause".repeat(200) + ); + let mismatch = format!("{cause}{INVOCATION_ID}: {}\r\n", AttemptId::default()); + assert_eq!( + request(api.addr(), "POST", &path, &mismatch, b"wrong-error") + .await + .status, + 400 + ); + let foreign = format!( + "/2018-06-01/runtime/invocation/{}/error", + AttemptId::default() + ); + assert_eq!( + request(api.addr(), "POST", &foreign, &cause, b"foreign-error") + .await + .status, + 400 + ); + let payload = b"original-error\0\xff"; + let accepted = format!("{cause}{INVOCATION_ID}: {}\r\n", ticket.attempt()); + assert_eq!( + request(api.addr(), "POST", &path, &accepted, payload) + .await + .status, + 202 + ); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::FunctionError(bytes) if bytes.as_ref()==payload) + ); + api.shutdown().await.unwrap(); +} + +#[derive(Debug)] +pub(crate) struct Reply { + pub(crate) status: u16, + headers: Vec<(String, String)>, + pub(crate) body: Vec, +} + +#[derive(Debug, Clone)] +struct LogWriter(Arc>>); + +impl io::Write for LogWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self.0.lock().unwrap(); + if output.len().saturating_add(bytes.len()) > 2 * 1024 * 1024 { + return Err(io::Error::other("log capture capacity exceeded")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Reply { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str())) + } +} + +async fn raw(addr: SocketAddr, request: Vec) -> Reply { + timeout(Duration::from_secs(3), async { + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(&request).await.unwrap(); + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).await.unwrap(); + let position = bytes + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .unwrap(); + let headers = String::from_utf8(bytes.get(..position).unwrap().to_vec()).unwrap(); + let mut lines = headers.lines(); + let status = lines + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .parse() + .unwrap(); + let headers = lines + .map(|line| { + let (key, value) = line.split_once(':').unwrap(); + (key.into(), value.trim().into()) + }) + .collect(); + Reply { + status, + headers, + body: bytes.get(position + 4..).unwrap().to_vec(), + } + }) + .await + .unwrap() +} + +pub(crate) async fn request( + addr: SocketAddr, + method: &str, + path: &str, + headers: &str, + body: &[u8], +) -> Reply { + let mut bytes = format!( + "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\nContent-Length: \ + {}\r\n{headers}\r\n", + body.len() + ) + .into_bytes(); + bytes.extend_from_slice(body); + raw(addr, bytes).await +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(10), + "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"secret-event\0\xff")).unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn test_should_round_trip_binary_invocations_and_supply_complete_runtime_metadata() { + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + let response = request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + assert_eq!(response.status, 200); + assert_eq!(response.body, call.parts().4.as_ref()); + assert_eq!( + response.header("Lambda-Runtime-Aws-Request-Id").unwrap(), + ticket.attempt().to_string() + ); + assert_eq!( + response.header(INVOCATION_ID).unwrap(), + ticket.attempt().to_string() + ); + assert_eq!( + response.header("Lambda-Runtime-Invoked-Function-Arn"), + Some("arn:aws:lambda:us-east-1:000000000000:function:echo:prod") + ); + assert!( + response + .header("Lambda-Runtime-Trace-Id") + .unwrap() + .ends_with("Sampled=0") + ); + assert!( + response + .header("Lambda-Runtime-Deadline-Ms") + .unwrap() + .parse::() + .unwrap() + > 0 + ); + let path = format!( + "/2018-06-01/runtime/invocation/{}/response", + ticket.attempt() + ); + let result = request( + api.addr(), + "POST", + &path, + &format!("Lambda-Runtime-Invocation-Id: {}\r\n", ticket.attempt()), + b"secret-result\0\xff", + ) + .await; + assert_eq!(result.status, 202); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::Success(payload) if payload.as_ref() == b"secret-result\0\xff") + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_preserve_current_attempt_after_unknown_duplicate_or_wrong_header_completions() +{ + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + let path = format!("/2018-06-01/runtime/invocation/{}/error", ticket.attempt()); + let mismatch = request( + api.addr(), + "POST", + &path, + &format!("Lambda-Runtime-Invocation-Id: {}\r\n", AttemptId::default()), + b"{}", + ) + .await; + assert_eq!(mismatch.status, 400); + assert!( + String::from_utf8(mismatch.body) + .unwrap() + .contains("InvalidInvocationId") + ); + let duplicate_headers = request( + api.addr(), + "POST", + &path, + &format!( + "Lambda-Runtime-Invocation-Id: {}\r\nLambda-Runtime-Invocation-Id: {}\r\n", + ticket.attempt(), + ticket.attempt() + ), + b"{}", + ) + .await; + assert_eq!(duplicate_headers.status, 400); + let unknown = request( + api.addr(), + "POST", + &format!( + "/2018-06-01/runtime/invocation/{}/response", + AttemptId::default() + ), + "", + b"{}", + ) + .await; + assert_eq!(unknown.status, 400); + assert_eq!( + request(api.addr(), "POST", &path, "", b"function-error") + .await + .status, + 202 + ); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::FunctionError(payload) if payload.as_ref() == b"function-error") + ); + assert_eq!( + request(api.addr(), "POST", &path, "", b"duplicate") + .await + .status, + 400 + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_acknowledge_initialization_error_before_joining_the_closed_session() { + let mut api = start().await.unwrap(); + let receiver = api.take_initialization_error().unwrap(); + assert!(api.take_initialization_error().is_none()); + assert_eq!( + request( + api.addr(), + "POST", + "/2018-06-01/runtime/init/error", + "", + b"secret-init-error" + ) + .await + .status, + 202 + ); + assert_eq!(receiver.await.unwrap().as_ref(), b"secret-init-error"); + api.shutdown().await.unwrap(); +} + +#[rstest] +#[case("GET", "/2018-06-01/runtime/invocation/next?secret-query=1", 400)] +#[case("GET", "http://localhost/2018-06-01/runtime/invocation/next", 400)] +#[case("POST", "/2018-06-01/runtime/invocation/next", 405)] +#[case("GET", "/2018-06-01/runtime/init/error", 405)] +#[case("POST", "/2018-06-01/runtime/invocation/not-an-id/response", 400)] +#[case( + "POST", + "/2018-06-01/runtime/invocation/00112233-4455-6677-8899-aabbccddeeff/extra/response", + 404 +)] +#[tokio::test] +async fn test_should_reject_invalid_routes_before_bootstrap_readiness( + #[case] method: &str, + #[case] path: &str, + #[case] expected: u16, +) { + let mut api = start().await.unwrap(); + assert_eq!( + request(api.addr(), method, path, "", b"").await.status, + expected + ); + assert_eq!( + api.session().bootstrap_ready(Instant::now()).await, + Err(SessionError::Deadline) + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_bound_headers_get_bodies_and_declared_post_bodies_before_session_commands() { + let mut api = start().await.unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"x" + ) + .await + .status, + 413 + ); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + &format!("X-Secret: {}\r\n", "s".repeat(257)), + b"" + ) + .await + .status, + 431 + ); + let request = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: \ + close\r\nContent-Length: {}\r\n\r\n", + MAX_PAYLOAD_BYTES + 1 + ) + .into_bytes(); + assert_eq!(raw(api.addr(), request).await.status, 413); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_enforce_streamed_chunked_body_limit_without_accepting_init_error() { + let mut api = start().await.unwrap(); + let mut notification = api.take_initialization_error().unwrap(); + let stream = TcpStream::connect(api.addr()).await.unwrap(); + let (mut reader, mut writer) = stream.into_split(); + let send = tokio::spawn(async move { + writer.write_all(b"POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n").await.unwrap(); + let header = format!("{:x}\r\n", MAX_PAYLOAD_BYTES + 1); + writer.write_all(header.as_bytes()).await.unwrap(); + let _ = writer.write_all(&vec![0xff; MAX_PAYLOAD_BYTES + 1]).await; + let _ = writer.write_all(b"\r\n0\r\n\r\n").await; + }); + let mut reply = Vec::new(); + timeout(Duration::from_secs(3), reader.read_to_end(&mut reply)) + .await + .unwrap() + .unwrap(); + send.await.unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 413")); + assert!(notification.try_recv().is_err()); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_keep_shutdown_owned_when_its_waiter_is_dropped_and_close_the_listener() { + let mut api = start().await.unwrap(); + let addr = api.addr(); + let mut waiting = Box::pin(api.shutdown()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(api.server.task.is_some()); + api.shutdown().await.unwrap(); + api.shutdown().await.unwrap(); + assert!(TcpStream::connect(addr).await.is_err()); + assert!(matches!( + start_at("0.0.0.0:0".parse().unwrap(), Duration::from_secs(60)).await, + Err(ApiError::NonLoopback) + )); +} + +#[tokio::test] +async fn test_should_retain_failed_supervision_on_repeated_shutdown() { + let mut api = start().await.unwrap(); + api.server.task.as_ref().unwrap().abort(); + assert!(matches!( + api.shutdown().await, + Err(ApiError::SupervisorFailed) + )); + assert!(matches!( + api.shutdown().await, + Err(ApiError::SupervisorFailed) + )); +} + +async fn observe_private_error_diagnostics(api: &RuntimeApiHandle) { + let error_call = invocation(); + let error_ticket = api + .session() + .submit(error_call.clone(), error_call.parts().1) + .await + .unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"" + ) + .await + .status, + 200 + ); + let error_path = format!( + "/2018-06-01/runtime/invocation/{}/error", + error_ticket.attempt() + ); + assert_eq!( + request( + api.addr(), + "POST", + &error_path, + &format!( + "{XRAY_ERROR_CAUSE}: {}\r\n", + "secret-xray-cause".repeat(100) + ), + b"secret-function-error" + ) + .await + .status, + 202 + ); + error_ticket.wait().await.unwrap(); +} + +#[tokio::test] +async fn test_should_omit_payloads_headers_and_targets_from_trace_level_runtime_diagnostics() { + let logs = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = Arc::clone(&logs); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::TRACE) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + .with_writer(move || LogWriter(Arc::clone(&captured))) + .finish(); + // This current-thread runtime keeps its tasks under one thread-local + // dispatcher, without changing process-wide logging or environment settings. + let _subscriber = tracing::subscriber::set_default(subscriber); + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "X-Secret: secret-header\r\n", + b"" + ) + .await + .status, + 200 + ); + let path = format!( + "/2018-06-01/runtime/invocation/{}/response", + ticket.attempt() + ); + assert_eq!( + request(api.addr(), "POST", &path, "", b"secret-result") + .await + .status, + 202 + ); + ticket.wait().await.unwrap(); + observe_private_error_diagnostics(&api).await; + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next?secret-target", + "", + b"" + ) + .await + .status, + 400 + ); + let mut stream = TcpStream::connect(api.addr()).await.unwrap(); + stream.write_all(b"POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX-Secret: secret-fragmented-trailer").await.unwrap(); + // Force another decoder poll with a partially buffered trailer. + tokio::time::sleep(Duration::from_millis(30)).await; + stream.write_all(b"\r\n\r\n").await.unwrap(); + let mut reply = Vec::new(); + timeout(Duration::from_secs(3), stream.read_to_end(&mut reply)) + .await + .unwrap() + .unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 400")); + api.shutdown().await.unwrap(); + let output = String::from_utf8(logs.lock().unwrap().clone()).unwrap(); + assert!( + !output.is_empty(), + "trace capture must observe runtime spans" + ); + for secret in [ + "secret-event", + "secret-header", + "secret-result", + "secret-target", + "secret-fragmented-trailer", + "secret-xray-cause", + "secret-function-error", + ] { + assert!( + !output.contains(secret), + "runtime diagnostics leaked {secret}" + ); + } +} + +#[tokio::test] +async fn test_should_reject_all_trailers_before_initialization_or_invocation_mutation() { + let mut api = start().await.unwrap(); + let mut notification = api.take_initialization_error().unwrap(); + for trailer in [ + "X-Secret: secret-trailer".to_owned(), + format!("X-Long: {}", "s".repeat(300)), + format!("Lambda-Runtime-Invocation-Id: {}", AttemptId::default()), + ] { + let bytes = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: \ + close\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n{trailer}\r\n\r\n" + ) + .into_bytes(); + assert_eq!(raw(api.addr(), bytes).await.status, 400); + assert!(notification.try_recv().is_err()); + assert!(!api.session().is_closed()); + } + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_publish_effective_delivery_deadline_and_join_api_after_handler_timeout() { + use crate::{metadata::deadline_unix_ms, protocol::HandlerTimeout}; + let mut api = start().await.unwrap(); + let handle = api.session().clone(); + let mut call = invocation(); + call = Invocation::new( + call.parts().0, + Instant::now() + Duration::from_secs(30), + call.parts().2.clone(), + call.parts().3.clone(), + call.parts().4.clone(), + ) + .unwrap() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let outer = deadline_unix_ms(call.parts().1).unwrap().get(); + let ticket = handle + .submit(call, Instant::now() + Duration::from_secs(3)) + .await + .unwrap(); + let reply = request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + assert_eq!(reply.status, 200); + let effective = reply + .header("Lambda-Runtime-Deadline-Ms") + .unwrap() + .parse::() + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + assert!(u128::from(effective) <= now + 1000 && u128::from(effective) > now); + assert!(effective + 20_000 < outer); + assert_eq!(ticket.wait().await, Err(SessionError::HandlerTimeout)); + api.shutdown().await.unwrap(); + assert!(TcpStream::connect(api.addr()).await.is_err()); +} diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs new file mode 100644 index 00000000..5a18232d --- /dev/null +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -0,0 +1,17 @@ +//! Shared, host-portable Lambda runtime contracts. +//! +//! The control protocol validates untrusted CBOR before deserialization and +//! preserves raw payload bytes without logging their contents. This crate has +//! no dependency on a function provider, VM implementation or Docker client. + +#![forbid(unsafe_code)] + +pub mod client; +pub mod diagnostics; +pub mod http; +pub mod metadata; +pub mod probe; +pub mod protocol; +pub mod relay; +pub mod session; +mod transport; diff --git a/crates/rustack-lambda-runtime/src/main.rs b/crates/rustack-lambda-runtime/src/main.rs new file mode 100644 index 00000000..3c30eccb --- /dev/null +++ b/crates/rustack-lambda-runtime/src/main.rs @@ -0,0 +1,39 @@ +//! Linux-only Lambda relay process. Stdout is reserved for binary control frames. + +#![forbid(unsafe_code)] + +#[cfg(not(target_os = "linux"))] +use std::future::{Ready, ready}; +use std::process::ExitCode; + +#[cfg(not(target_os = "linux"))] +use rustack_lambda_runtime::relay::RelayError; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> ExitCode { + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_target(false) + .with_writer(std::io::stderr) + .finish(); + if tracing::subscriber::set_global_default(subscriber).is_err() { + return ExitCode::FAILURE; + } + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + tracing::error!(%error, "Lambda runtime relay exited"); + ExitCode::FAILURE + } + } +} + +#[cfg(not(target_os = "linux"))] +fn run() -> Ready> { + ready(Err(RelayError::UnsupportedPlatform)) +} + +#[cfg(target_os = "linux")] +mod process; +#[cfg(target_os = "linux")] +use process::run; diff --git a/crates/rustack-lambda-runtime/src/metadata.rs b/crates/rustack-lambda-runtime/src/metadata.rs new file mode 100644 index 00000000..1ee6d34b --- /dev/null +++ b/crates/rustack-lambda-runtime/src/metadata.rs @@ -0,0 +1,134 @@ +//! Checked wall/monotonic deadline conversion and local tracing metadata. + +use std::{ + num::NonZeroU64, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use thiserror::Error; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::protocol::TraceId; + +/// Clock/metadata failures contain no invocation contents. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum MetadataError { + /// Supplied deadline has already elapsed. + #[error("runtime deadline expired")] + Expired, + /// Clock/range/conversion cannot represent the metadata. + #[error("invalid runtime clock metadata")] + Invalid, +} + +/// Convert a future monotonic deadline into checked Unix milliseconds. +/// +/// # Errors +/// Rejects expired deadlines, a pre-epoch wall clock or arithmetic overflow. +/// +/// ``` +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// assert!(rustack_lambda_runtime::metadata::deadline_unix_ms(deadline).is_ok()); +/// ``` +pub fn deadline_unix_ms(deadline: Instant) -> Result { + to_wall(deadline, Instant::now(), SystemTime::now()) +} + +fn to_wall(deadline: Instant, now: Instant, wall: SystemTime) -> Result { + let remaining = deadline + .checked_duration_since(now) + .filter(|remaining| !remaining.is_zero()) + .ok_or(MetadataError::Expired)?; + let epoch = wall + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)?; + let total = epoch.checked_add(remaining).ok_or(MetadataError::Invalid)?; + NonZeroU64::new(u64::try_from(total.as_millis()).map_err(|_| MetadataError::Invalid)?) + .ok_or(MetadataError::Invalid) +} + +/// Translate an incoming wall deadline into a bounded monotonic deadline. +/// +/// # Errors +/// Rejects expired deadlines, invalid clocks and more than one hour remaining. +/// Lambda's≤900s handler plus bounded startup fits within this local ceiling. +/// +/// ``` +/// # use rustack_lambda_runtime::metadata::{deadline_unix_ms, deadline_from_unix_ms}; +/// let value = deadline_unix_ms(tokio::time::Instant::now() + std::time::Duration::from_secs(5))?; +/// assert!(deadline_from_unix_ms(value).is_ok()); +/// # Ok::<(), rustack_lambda_runtime::metadata::MetadataError>(()) +/// ``` +pub fn deadline_from_unix_ms(value: NonZeroU64) -> Result { + from_wall(value, Instant::now(), SystemTime::now()) +} + +fn from_wall(value: NonZeroU64, now: Instant, wall: SystemTime) -> Result { + let epoch = wall + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)?; + let remaining = Duration::from_millis(value.get()) + .checked_sub(epoch) + .filter(|remaining| !remaining.is_zero()) + .ok_or(MetadataError::Expired)?; + if remaining > Duration::from_hours(1) { + return Err(MetadataError::Invalid); + } + now.checked_add(remaining).ok_or(MetadataError::Invalid) +} + +/// Generate an X-Ray-shaped local trace, without claiming upstream sampling. +/// +/// # Errors +/// Returns Invalid if the wall clock/validated header cannot be represented. +/// +/// ``` +/// assert!(rustack_lambda_runtime::metadata::local_trace_id()?.as_ref().ends_with("Sampled=0")); +/// # Ok::<(), rustack_lambda_runtime::metadata::MetadataError>(()) +/// ``` +pub fn local_trace_id() -> Result { + let epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)? + .as_secs() + % (1_u64 << 32); + let root = Uuid::new_v4().simple().to_string(); + let parent = Uuid::new_v4().simple().to_string(); + let root = root.get(..24).ok_or(MetadataError::Invalid)?; + let parent = parent.get(..16).ok_or(MetadataError::Invalid)?; + format!("Root=1-{epoch:08x}-{root};Parent={parent};Sampled=0") + .parse() + .map_err(|_| MetadataError::Invalid) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_convert_checked_fractional_deadlines_and_reject_invalid_clocks() { + let now = Instant::now(); + let wall = UNIX_EPOCH + Duration::from_millis(1000); + let deadline = now + Duration::from_micros(2500); + assert_eq!(to_wall(deadline, now, wall).unwrap().get(), 1002); + assert_eq!( + from_wall(NonZeroU64::new(1002).unwrap(), now, wall).unwrap(), + now + Duration::from_millis(2) + ); + assert_eq!(to_wall(now, now, wall), Err(MetadataError::Expired)); + assert_eq!( + to_wall(deadline, now, UNIX_EPOCH - Duration::from_secs(1)), + Err(MetadataError::Invalid) + ); + assert_eq!( + from_wall(NonZeroU64::new(1000).unwrap(), now, wall), + Err(MetadataError::Expired) + ); + assert_eq!( + from_wall(NonZeroU64::new(u64::MAX).unwrap(), now, wall), + Err(MetadataError::Invalid) + ); + } +} diff --git a/crates/rustack-lambda-runtime/src/probe/client.rs b/crates/rustack-lambda-runtime/src/probe/client.rs new file mode 100644 index 00000000..66dc388a --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/client.rs @@ -0,0 +1,180 @@ +//! One owned HTTP connection per request; never spawn a detached connection task. + +use std::{env, future::pending, num::NonZeroU64, sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::{HeaderMap, Method, Request, StatusCode}; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::client::conn::http1; +use hyper_util::rt::TokioIo; +use tokio::{ + net::TcpStream, + time::{Instant, sleep_until, timeout_at}, +}; + +use super::{EndpointChallenge, ProbeError, REPORT_BYTES, collect, endpoint}; +use crate::{metadata::deadline_from_unix_ms, protocol::AttemptId}; + +struct Reply { + status: StatusCode, + headers: HeaderMap, + body: Bytes, +} +async fn request( + method: Method, + path: &str, + body: Bytes, + deadline: Instant, +) -> Result { + let stream = timeout_at(deadline, TcpStream::connect("127.0.0.1:9001")) + .await + .map_err(|_| ProbeError::Deadline)??; + let mut builder = http1::Builder::new(); + builder.max_buf_size(16384).max_headers(32); + let (mut sender, connection) = timeout_at(deadline, builder.handshake(TokioIo::new(stream))) + .await + .map_err(|_| ProbeError::Deadline)??; + tokio::pin!(connection); + let operation = async { + let request = Request::builder() + .method(method) + .uri(path) + .header("host", "127.0.0.1:9001") + .header("connection", "close") + .header("content-type", "application/json") + .body(Full::new(body)) + .map_err(|_| ProbeError::Protocol)?; + let response = sender.send_request(request).await?; + let (parts, body) = response.into_parts(); + let body = Limited::new(body, REPORT_BYTES) + .collect() + .await + .map_err(|_| ProbeError::Protocol)? + .to_bytes(); + Ok(Reply { + status: parts.status, + headers: parts.headers, + body, + }) + }; + tokio::pin!(operation); + let mut finished = false; + loop { + tokio::select! {biased; + ()=sleep_until(deadline)=>return Err(ProbeError::Deadline), + result=&mut operation=>return result, + result=&mut connection,if !finished=>{result?;finished=true;}, + } + } +} +fn header<'a>(headers: &'a HeaderMap, key: &str, cap: usize) -> Result<&'a str, ProbeError> { + let mut values = headers.get_all(key).iter(); + let value = values.next().ok_or(ProbeError::Protocol)?; + if values.next().is_some() || value.as_bytes().len() > cap { + return Err(ProbeError::Protocol); + } + value.to_str().map_err(|_| ProbeError::Protocol) +} +#[derive(Debug, Clone, Copy)] +enum Mode { + Resource, + Endpoint, +} +pub(super) async fn run() -> Result<(), ProbeError> { + run_probe(Mode::Resource).await +} +pub(super) async fn run_endpoint() -> Result<(), ProbeError> { + run_probe(Mode::Endpoint).await +} +async fn run_probe(mode: Mode) -> Result<(), ProbeError> { + if !cfg!(target_os = "linux") + || env::var("AWS_LAMBDA_RUNTIME_API").as_deref() != Ok("127.0.0.1:9001") + { + return Err(ProbeError::Configuration); + } + let challenge = request( + Method::GET, + "/2018-06-01/runtime/invocation/next", + Bytes::new(), + Instant::now() + Duration::from_secs(60), + ) + .await?; + if challenge.status != StatusCode::OK + || (!matches!(mode, Mode::Endpoint) && challenge.body.as_ref() != b"{}") + { + return Err(ProbeError::Protocol); + } + let attempt: AttemptId = header(&challenge.headers, "lambda-runtime-aws-request-id", 36)? + .parse() + .map_err(|_| ProbeError::Protocol)?; + let wall = header(&challenge.headers, "lambda-runtime-deadline-ms", 20)? + .parse::() + .map_err(|_| ProbeError::Protocol)?; + let deadline = deadline_from_unix_ms(wall).map_err(|_| ProbeError::Deadline)?; + let observation = if matches!(mode, Mode::Endpoint) { + match EndpointChallenge::decode(&challenge.body) { + Ok(challenge) => endpoint::observe(&challenge, attempt, deadline) + .await + .and_then(|report| { + serde_json::to_vec(&report).map_err(|error| ProbeError::Report(Arc::new(error))) + }), + Err(error) => Err(error), + } + } else { + collect::collect(attempt, deadline) + .await + .and_then(|report| { + serde_json::to_vec(&report).map_err(|error| ProbeError::Report(Arc::new(error))) + }) + }; + let (suffix, body) = match observation { + Ok(bytes) if bytes.len() <= REPORT_BYTES => ("response", Bytes::from(bytes)), + Ok(_) => return Err(ProbeError::Protocol), + Err(_) => ( + "error", + Bytes::from_static( + br#"{"errorType":"Rustack.ProbeFailed","errorMessage":"runtime probe failed"}"#, + ), + ), + }; + let reply = request( + Method::POST, + &format!("/2018-06-01/runtime/invocation/{attempt}/{suffix}"), + body, + deadline.min(Instant::now() + Duration::from_secs(5)), + ) + .await?; + if reply.status != StatusCode::ACCEPTED || !reply.body.is_empty() { + return Err(ProbeError::Protocol); + } + // The host owns this one-shot function's retirement, including on rejection. + pending().await +} + +#[cfg(test)] +mod tests { + use http::HeaderValue; + + use super::*; + + #[test] + fn test_should_require_exact_single_bounded_ascii_probe_metadata() { + let mut values = HeaderMap::new(); + assert!(header(&values, "probe", 3).is_err()); + values.insert("probe", HeaderValue::from_static("123")); + assert_eq!(header(&values, "probe", 3).unwrap(), "123"); + assert!(header(&values, "probe", 2).is_err()); + values.append("probe", HeaderValue::from_static("456")); + assert!(header(&values, "probe", 3).is_err()); + values.clear(); + values.insert("probe", HeaderValue::from_bytes(b"\xff").unwrap()); + assert!(header(&values, "probe", 3).is_err()); + } + #[test] + fn test_should_redact_probe_error_sources() { + use std::error::Error as _; + let error = ProbeError::from(std::io::Error::other("private-kernel-path")); + assert!(error.source().is_some()); + assert!(!format!("{error:?} {error}").contains("private-kernel-path")); + } +} diff --git a/crates/rustack-lambda-runtime/src/probe/collect.rs b/crates/rustack-lambda-runtime/src/probe/collect.rs new file mode 100644 index 00000000..3b76e814 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/collect.rs @@ -0,0 +1,102 @@ +//! Fixed-path bounded async reads; process/container retirement owns cancellation. + +use std::{env::consts::ARCH, io, os::unix::fs::MetadataExt}; + +use tokio::{ + fs::{File, metadata}, + io::AsyncReadExt, + time::{Instant, timeout_at}, +}; + +use super::{ProbeError, ResourceReport, parse}; +use crate::protocol::{Architecture, AttemptId}; + +async fn read(path: &'static str, cap: usize) -> Result { + let mut bytes = Vec::new(); + File::open(path) + .await? + .take( + u64::try_from(cap) + .map_err(|_| ProbeError::Observation)? + .saturating_add(1), + ) + .read_to_end(&mut bytes) + .await?; + if bytes.len() > cap { + return Err(ProbeError::Observation); + } + String::from_utf8(bytes).map_err(|_| ProbeError::Observation) +} +pub(super) async fn collect( + attempt: AttemptId, + deadline: Instant, +) -> Result { + timeout_at(deadline, async { + let memory = read("/sys/fs/cgroup/memory.max", 128).await; + let (memory_max, swap_max, pids_max, cpu_quota, cpu_period, unified) = match memory { + Ok(memory) => { + let (quota, period) = parse::cpu(&read("/sys/fs/cgroup/cpu.max", 128).await?)?; + ( + parse::limit(&memory)?, + parse::limit(&read("/sys/fs/cgroup/memory.swap.max", 128).await?)?, + parse::limit(&read("/sys/fs/cgroup/pids.max", 128).await?)?, + quota, + period, + true, + ) + } + Err(ProbeError::Io(error)) if error.kind() == io::ErrorKind::NotFound => { + let memory = + parse::limit(&read("/sys/fs/cgroup/memory/memory.limit_in_bytes", 128).await?)?; + let total = parse::limit( + &read("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes", 128).await?, + )?; + let swap = match (memory, total) { + (Some(memory), Some(total)) => { + Some(total.checked_sub(memory).ok_or(ProbeError::Observation)?) + } + _ => None, + }; + ( + memory, + swap, + parse::limit(&read("/sys/fs/cgroup/pids/pids.max", 128).await?)?, + parse::limit(&read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", 128).await?)?, + parse::number(&read("/sys/fs/cgroup/cpu/cpu.cfs_period_us", 128).await?)?, + false, + ) + } + Err(error) => return Err(error), + }; + let status = parse::status(&read("/proc/self/status", 65536).await?)?; + let cgroup_root = parse::membership(&read("/proc/self/cgroup", 65536).await?, unified)?; + let mode = metadata("/tmp").await?.mode() & 0o7777; + let (root_readonly, temporary) = + parse::mounts(&read("/proc/self/mountinfo", 65536).await?, mode)?; + let architecture = match ARCH { + "aarch64" => Architecture::Arm64, + "x86_64" => Architecture::X86_64, + _ => return Err(ProbeError::Configuration), + }; + Ok(ResourceReport::builder() + .version(1) + .attempt(attempt) + .architecture(architecture) + .cgroup_root(cgroup_root) + .memory_max(memory_max) + .swap_max(swap_max) + .pids_max(pids_max) + .cpu_quota(cpu_quota) + .cpu_period(cpu_period) + .euid(status.uid) + .egid(status.gid) + .cap_effective(status.capabilities) + .no_new_privileges(status.no_new_privileges) + .seccomp_mode(status.seccomp) + .root_readonly(root_readonly) + .temporary(temporary) + .build()) + }) + .await + .map_err(|_| ProbeError::Deadline)? +} diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint.rs b/crates/rustack-lambda-runtime/src/probe/endpoint.rs new file mode 100644 index 00000000..d7a3c208 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint.rs @@ -0,0 +1,192 @@ +//! One container-origin HTTP(S) observation; all IO futures stay locally owned. + +pub(super) mod entropy; +pub(super) mod types; + +use std::{env::consts::ARCH, sync::Arc}; + +use bytes::{Bytes, BytesMut}; +use http::{ + Request, StatusCode, + header::{CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH, HOST}, +}; +use http_body_util::{BodyExt, Full}; +use hyper::client::conn::http1; +use hyper_util::rt::TokioIo; +use rustls::{ + ClientConfig, RootCertStore, crypto::aws_lc_rs::default_provider, pki_types::ServerName, +}; +use serde_json::from_slice; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::TcpStream, + time::{Instant, timeout_at}, +}; +use tokio_rustls::TlsConnector; +use tracing::instrument::WithSubscriber; +use url::{Host, Url}; +use webpki_roots::TLS_SERVER_ROOTS; + +use self::types::{EndpointChallenge, EndpointReport, GATEWAY_PROBE_PATH, GatewayReply}; +use super::{ProbeError, REPORT_BYTES}; +use crate::{ + diagnostics::runtime_dispatch, + protocol::{Architecture, AttemptId}, +}; + +pub(super) async fn observe( + challenge: &EndpointChallenge, + attempt: AttemptId, + deadline: Instant, +) -> Result { + let roots = RootCertStore { + roots: TLS_SERVER_ROOTS.to_vec(), + }; + let observed = observe_with_roots(challenge, deadline, roots) + .with_subscriber(runtime_dispatch()) + .await?; + let architecture = match ARCH { + "aarch64" => Architecture::Arm64, + "x86_64" => Architecture::X86_64, + _ => return Err(ProbeError::Configuration), + }; + if Instant::now() >= deadline { + return Err(ProbeError::Deadline); + } + Ok(EndpointReport::observed( + attempt, + architecture, + observed.instance_id(), + )) +} +async fn observe_with_roots( + challenge: &EndpointChallenge, + deadline: Instant, + roots: RootCertStore, +) -> Result { + let reply = timeout_at(deadline, async { + let url = + Url::parse(challenge.endpoint().as_str()).map_err(|_| ProbeError::Configuration)?; + let host = match url.host().ok_or(ProbeError::Configuration)? { + Host::Domain(name) => name.to_owned(), + Host::Ipv4(ip) => ip.to_string(), + Host::Ipv6(ip) => ip.to_string(), + }; + let port = url + .port_or_known_default() + .ok_or(ProbeError::Configuration)?; + let wire_host = url.host_str().ok_or(ProbeError::Configuration)?; + let authority = match url.port() { + Some(port) => format!("{wire_host}:{port}"), + None => wire_host.to_owned(), + }; + let stream = TcpStream::connect((host.as_str(), port)).await?; + if url.scheme() == "https" { + let name = ServerName::try_from(host).map_err(|_| ProbeError::Configuration)?; + let config = ClientConfig::builder_with_provider(Arc::new(default_provider())) + .with_safe_default_protocol_versions() + .map_err(|_| ProbeError::Tls)? + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = TlsConnector::from(Arc::new(config)); + let stream = connector + .connect(name, stream) + .await + .map_err(|_| ProbeError::Tls)?; + exchange(stream, &authority).await + } else { + exchange(stream, &authority).await + } + }) + .await + .map_err(|_| ProbeError::Deadline)??; + if Instant::now() >= deadline { + return Err(ProbeError::Deadline); + } + if reply.instance_id() != challenge.expected() { + return Err(ProbeError::Protocol); + } + Ok(reply) +} +async fn exchange( + stream: S, + authority: &str, +) -> Result { + let (mut sender, connection) = http1::Builder::new() + .max_headers(32) + .max_buf_size(16 * 1024) + .handshake(TokioIo::new(stream)) + .await?; + let connection = connection.without_shutdown(); + tokio::pin!(connection); + let mut ended = false; + let outcome = { + let operation = async { + let request = Request::builder() + .method("GET") + .uri(GATEWAY_PROBE_PATH) + .header(HOST, authority) + .header(CONNECTION, "close") + .header(CONTENT_LENGTH, 0) + .body(Full::new(Bytes::new())) + .map_err(|_| ProbeError::Protocol)?; + let response = sender.send_request(request).await?; + if response.status() != StatusCode::OK + || response.headers().contains_key(CONTENT_ENCODING) + { + return Err(ProbeError::Protocol); + } + let mut header_bytes = 0_usize; + for (name, value) in response.headers() { + header_bytes = header_bytes + .checked_add(name.as_str().len()) + .and_then(|count| count.checked_add(value.as_bytes().len())) + .ok_or(ProbeError::Protocol)?; + } + if header_bytes > 16 * 1024 { + return Err(ProbeError::Protocol); + } + let mut lengths = response.headers().get_all(CONTENT_LENGTH).iter(); + if let Some(length) = lengths.next() { + if lengths.next().is_some() + || length + .to_str() + .ok() + .and_then(|value| value.parse::().ok()) + .is_none_or(|length| length > REPORT_BYTES) + { + return Err(ProbeError::Protocol); + } + } + let mut body = response.into_body(); + let mut bytes = BytesMut::new(); + while let Some(frame) = body.frame().await { + let data = frame?.into_data().map_err(|_| ProbeError::Protocol)?; + if bytes + .len() + .checked_add(data.len()) + .is_none_or(|size| size > REPORT_BYTES) + { + return Err(ProbeError::Protocol); + } + bytes.extend_from_slice(&data); + } + from_slice::(&bytes).map_err(|_| ProbeError::Protocol) + }; + tokio::pin!(operation); + loop { + tokio::select! {biased; + result=&mut operation=>break result, + result=&mut connection,if !ended=>{drop(result?);ended=true;}, + } + } + }; + drop(sender); + if outcome.is_ok() && !ended { + drop(connection.await?); + } + outcome +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint/entropy.rs b/crates/rustack-lambda-runtime/src/probe/endpoint/entropy.rs new file mode 100644 index 00000000..b3978e3d --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint/entropy.rs @@ -0,0 +1,152 @@ +//! Read-only OS entropy jobs retain bounded permits through observer loss/results. + +use std::time::Duration; + +use tokio::{ + runtime::Handle, + sync::{Semaphore, SemaphorePermit}, + task::{JoinHandle, spawn_blocking}, + time::{Instant, timeout_at}, +}; + +use super::types::GatewayInstanceId; +use crate::probe::ProbeError; + +static WORKERS: Semaphore = Semaphore::const_new(4); +type IdentityResult = ( + Result, + SemaphorePermit<'static>, +); + +pub(super) async fn generate(deadline: Instant) -> Result { + owned(deadline, &WORKERS, GatewayInstanceId::generate).await +} +async fn owned( + deadline: Instant, + pool: &'static Semaphore, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|value| !value.is_zero()) + .ok_or(ProbeError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(ProbeError::Configuration); + } + Handle::try_current().map_err(|_| ProbeError::Worker)?; + let permit = timeout_at(deadline, pool.acquire()) + .await + .map_err(|_| ProbeError::Deadline)? + .map_err(|_| ProbeError::Worker)?; + let job = spawn_identity(permit, deadline, work); + // Expired/dropped observers discard their read-only result; the blocking job + // retains its physical permit until OS return and ready-result disposal. + let (result, _permit) = timeout_at(deadline, job) + .await + .map_err(|_| ProbeError::Deadline)? + .map_err(|_| ProbeError::Worker)?; + if Instant::now() >= deadline { + return Err(ProbeError::Deadline); + } + result +} +fn spawn_identity( + permit: SemaphorePermit<'static>, + deadline: Instant, + work: impl FnOnce() -> Result + Send + 'static, +) -> JoinHandle { + spawn_blocking(move || { + let result = if Instant::now() >= deadline { + Err(ProbeError::Deadline) + } else { + work() + }; + let result = if Instant::now() >= deadline { + Err(ProbeError::Deadline) + } else { + result + }; + (result, permit) + }) +} +#[cfg(test)] +mod tests { + use std::{sync::mpsc, time::Duration}; + + use tokio::{ + spawn, + sync::{Semaphore, oneshot}, + task::yield_now, + time::{Instant, timeout}, + }; + + use super::{owned, spawn_identity}; + use crate::probe::{GatewayInstanceId, ProbeError}; + fn id() -> GatewayInstanceId { + "aaaaaaaa-1111-4111-8111-111111111111".parse().unwrap() + } + #[tokio::test] + async fn test_should_keep_entropy_work_charged_after_waiter_abandonment_and_queue_expiry() { + static POOL: Semaphore = Semaphore::const_new(1); + let (release, held) = mpsc::sync_channel(1); + let (entered, saw) = oneshot::channel(); + let job = spawn(owned( + Instant::now() + Duration::from_secs(5), + &POOL, + move || { + entered.send(()).unwrap(); + held.recv().unwrap(); + Ok(id()) + }, + )); + saw.await.unwrap(); + job.abort(); + assert!(job.await.unwrap_err().is_cancelled()); + assert_eq!(POOL.available_permits(), 0); + assert!(matches!( + owned(Instant::now() + Duration::from_millis(25), &POOL, || Ok( + id() + )) + .await, + Err(ProbeError::Deadline) + )); + assert_eq!(POOL.available_permits(), 0); + release.send(()).unwrap(); + timeout(Duration::from_secs(2), async { + while POOL.available_permits() != 1 { + yield_now().await; + } + }) + .await + .unwrap(); + } + #[tokio::test] + async fn test_should_hold_ready_entropy_result_until_observed_and_reject_expired_budget() { + static POOL: Semaphore = Semaphore::const_new(1); + let job = spawn_identity( + POOL.acquire().await.unwrap(), + Instant::now() + Duration::from_secs(5), + || Ok(id()), + ); + timeout(Duration::from_secs(2), async { + while !job.is_finished() { + yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(POOL.available_permits(), 0); + let (result, permit) = job.await.unwrap(); + assert_eq!(result.unwrap(), id()); + drop(permit); + assert_eq!(POOL.available_permits(), 1); + assert!(matches!( + owned(Instant::now(), &POOL, || Ok(id())).await, + Err(ProbeError::Deadline) + )); + assert!(matches!( + owned(Instant::now() + Duration::from_secs(61), &POOL, || Ok(id())).await, + Err(ProbeError::Configuration) + )); + } +} diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint/tests.rs b/crates/rustack-lambda-runtime/src/probe/endpoint/tests.rs new file mode 100644 index 00000000..3460df92 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint/tests.rs @@ -0,0 +1,190 @@ +use std::{io, time::Duration}; + +use rstest::rstest; +use rustls::RootCertStore; +use serde_json::to_vec; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + spawn, + sync::oneshot, + task::JoinHandle, + time::{Instant, timeout}, +}; + +use super::{ + observe_with_roots, + types::{EndpointChallenge, GatewayInstanceId, GatewayReply}, +}; +use crate::probe::ProbeError; + +mod tls; + +async fn read_head(stream: &mut S) -> io::Result> { + let mut bytes = Vec::new(); + while bytes.len() < 8192 { + let byte = stream.read_u8().await?; + bytes.push(byte); + if bytes.ends_with(b"\r\n\r\n") { + return Ok(bytes); + } + } + Err(io::Error::other("fixture request exceeded bound")) +} +fn response(status: u16, body: &[u8], headers: &str) -> Vec { + let mut bytes = format!( + "HTTP/1.1 {status} Fixture\r\nContent-Length: {}\r\nConnection: close\r\n{headers}\r\n", + body.len() + ) + .into_bytes(); + bytes.extend_from_slice(body); + bytes +} +async fn fixture( + build: impl FnOnce(GatewayInstanceId) -> Vec, +) -> (EndpointChallenge, JoinHandle>) { + let id = GatewayInstanceId::generate().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let reply = build(id); + let server = spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_head(&mut stream).await.unwrap(); + let _ = stream.write_all(&reply).await; + let _ = stream.shutdown().await; + request + }); + (EndpointChallenge::new(endpoint, id), server) +} +async fn join(server: JoinHandle>) -> Vec { + timeout(Duration::from_secs(5), server) + .await + .unwrap() + .unwrap() +} +#[tokio::test] +async fn test_should_observe_independent_instance_without_sending_expected_identity() { + let (challenge, server) = + fixture(|id| response(200, &to_vec(&GatewayReply::new(id)).unwrap(), "")).await; + let reply = observe_with_roots( + &challenge, + Instant::now() + Duration::from_secs(5), + RootCertStore::empty(), + ) + .await + .unwrap(); + assert_eq!(reply.instance_id(), challenge.expected()); + let request = String::from_utf8(join(server).await).unwrap(); + assert!(request.starts_with("GET /_rustack/runtime-probe HTTP/1.1\r\n")); + let id: String = challenge.expected().into(); + assert!(!request.contains(&id)); + assert!(request.to_ascii_lowercase().contains("connection: close")); +} +#[rstest] +#[case::redirect(0)] +#[case::wrong_instance(1)] +#[case::unknown_field(2)] +#[case::duplicate_field(3)] +#[case::encoding(4)] +#[case::unterminated(5)] +#[case::wrong_version(6)] +#[case::empty(7)] +#[case::overlong_declared_body(8)] +#[case::overlong_actual_body(9)] +#[case::headers(10)] +#[case::header_bytes(11)] +#[case::trailers(12)] +#[tokio::test] +async fn test_should_reject_untrusted_gateway_responses_without_peer_text(#[case] case: u8) { + let (challenge,server)=fixture(|id|{ + let id:String=id.into(); + match case { + 0=>response(302,b"PRIVATE-PEER","Location: http://127.0.0.1:1/\r\n"), + 1=>response(200,&to_vec(&GatewayReply::new(GatewayInstanceId::generate().unwrap())).unwrap(),""), + 2=>response(200,format!("{{\"version\":1,\"instanceId\":\"{id}\",\"PRIVATE-PEER\":1}}").as_bytes(),""), + 3=>response(200,format!("{{\"version\":1,\"version\":1,\"instanceId\":\"{id}\"}}").as_bytes(),""), + 4=>response(200,&to_vec(&GatewayReply::new(id.parse().unwrap())).unwrap(),"Content-Encoding: gzip\r\n"), + 5=>b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\n{}".to_vec(), + 6=>response(200,format!("{{\"version\":2,\"instanceId\":\"{id}\"}}").as_bytes(),""), + 7=>response(200,b"",""), + 8=>response(200,&vec![b'x';4097],""), + 9=>{let mut bytes=b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_vec();bytes.extend_from_slice(&vec![b'x';4097]);bytes}, + 10=>response(200,b"{}",&"X-Fixture: x\r\n".repeat(33)), + 11=>response(200,b"{}",&format!("X-Fixture: {}\r\n","x".repeat(16_385))), + _=>b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n2\r\n{}\r\n0\r\nPRIVATE-PEER: trailer\r\n\r\n".to_vec(), + } + }).await; + let error = observe_with_roots( + &challenge, + Instant::now() + Duration::from_secs(5), + RootCertStore::empty(), + ) + .await + .unwrap_err(); + assert!(!format!("{error:?}:{error}").contains("PRIVATE-PEER")); + join(server).await; +} +#[rstest] +#[case::deadline(false)] +#[case::abandonment(true)] +#[tokio::test] +async fn test_should_close_owned_http_io_on_expiry_or_abandoned_caller(#[case] abandon: bool) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let challenge = EndpointChallenge::new( + format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(), + GatewayInstanceId::generate().unwrap(), + ); + let (entered, saw) = oneshot::channel(); + let server = spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + read_head(&mut stream).await.unwrap(); + entered.send(()).unwrap(); + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 0); + }); + let client = spawn(async move { + observe_with_roots( + &challenge, + Instant::now() + Duration::from_secs(1), + RootCertStore::empty(), + ) + .await + }); + saw.await.unwrap(); + if abandon { + client.abort(); + assert!(client.await.unwrap_err().is_cancelled()); + } else { + assert!(matches!(client.await.unwrap(), Err(ProbeError::Deadline))); + } + timeout(Duration::from_secs(3), server) + .await + .unwrap() + .unwrap(); +} +#[tokio::test] +async fn test_should_not_follow_a_redirect_to_another_listener() { + let other = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let target = other.local_addr().unwrap(); + let (challenge, server) = + fixture(|_| response(302, b"", &format!("Location: http://{target}/\r\n"))).await; + assert!( + observe_with_roots( + &challenge, + Instant::now() + Duration::from_secs(5), + RootCertStore::empty() + ) + .await + .is_err() + ); + join(server).await; + assert!( + timeout(Duration::from_millis(50), other.accept()) + .await + .is_err() + ); +} diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint/tests/tls.rs b/crates/rustack-lambda-runtime/src/probe/endpoint/tests/tls.rs new file mode 100644 index 00000000..145ae3e0 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint/tests/tls.rs @@ -0,0 +1,121 @@ +use std::{sync::Arc, time::Duration}; + +use rcgen::generate_simple_self_signed; +use rstest::rstest; +use rustls::{ + RootCertStore, ServerConfig, + crypto::aws_lc_rs::default_provider, + pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}, +}; +use serde_json::to_vec; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + spawn, + sync::oneshot, + task::spawn_blocking, + time::{Instant, timeout}, +}; +use tokio_rustls::TlsAcceptor; + +use super::{read_head, response}; +use crate::probe::{ + ProbeError, + endpoint::{ + observe_with_roots, + types::{EndpointChallenge, GatewayInstanceId, GatewayReply}, + }, +}; + +#[rstest] +#[case::valid(true, true)] +#[case::untrusted(false, true)] +#[case::wrong_name(true, false)] +#[tokio::test] +async fn test_should_verify_tls_ca_and_server_name_with_explicit_aws_lc( + #[case] trusted: bool, + #[case] named: bool, +) { + let certificate = spawn_blocking(|| generate_simple_self_signed(vec!["localhost".into()])) + .await + .unwrap() + .unwrap(); + let mut roots = RootCertStore::empty(); + if trusted { + roots.add(certificate.cert.der().clone()).unwrap(); + } + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + certificate.signing_key.serialize_der(), + )); + let server_config = ServerConfig::builder_with_provider(Arc::new(default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![certificate.cert.der().clone()], key) + .unwrap(); + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let host = if named { "localhost" } else { "127.0.0.1" }; + let id = GatewayInstanceId::generate().unwrap(); + let challenge = EndpointChallenge::new(format!("https://{host}:{port}").parse().unwrap(), id); + let server = spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + match acceptor.accept(stream).await { + Ok(mut stream) => { + let request = read_head(&mut stream).await.unwrap(); + let _ = stream + .write_all(&response(200, &to_vec(&GatewayReply::new(id)).unwrap(), "")) + .await; + let _ = stream.shutdown().await; + Some(request) + } + Err(_) => None, + } + }); + let result = + observe_with_roots(&challenge, Instant::now() + Duration::from_secs(5), roots).await; + let observed = timeout(Duration::from_secs(5), server) + .await + .unwrap() + .unwrap(); + if trusted && named { + assert_eq!(result.unwrap().instance_id(), id); + assert!(observed.is_some()); + } else { + assert!(matches!(result, Err(ProbeError::Tls))); + assert!(observed.is_none()); + } +} +#[tokio::test] +async fn test_should_close_owned_tls_handshake_on_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let challenge = EndpointChallenge::new( + format!("https://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(), + GatewayInstanceId::generate().unwrap(), + ); + let (entered, saw) = oneshot::channel(); + let server = spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + entered.send(()).unwrap(); + let mut bytes = Vec::new(); + stream.take(65_537).read_to_end(&mut bytes).await.unwrap(); + assert!(bytes.len() <= 65_536); + }); + let client = spawn(async move { + observe_with_roots( + &challenge, + Instant::now() + Duration::from_secs(1), + RootCertStore::empty(), + ) + .await + }); + saw.await.unwrap(); + assert!(matches!(client.await.unwrap(), Err(ProbeError::Deadline))); + timeout(Duration::from_secs(3), server) + .await + .unwrap() + .unwrap(); +} diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint/types.rs b/crates/rustack-lambda-runtime/src/probe/endpoint/types.rs new file mode 100644 index 00000000..8317301a --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint/types.rs @@ -0,0 +1,285 @@ +//! Shared bounded data, deliberately separate from host-owned readiness receipts. + +use std::{fmt, str::FromStr}; + +use getrandom::fill; +use serde::{Deserialize, Deserializer, Serialize, de::Error}; +use serde_json::from_slice; +use tokio::time::Instant; +use url::Url; +use uuid::{Builder, Uuid}; + +use crate::{ + probe::{ProbeError, REPORT_BYTES}, + protocol::{Architecture, AttemptId}, +}; + +/// Fixed admission-independent diagnostic route; no caller-selected path. +pub const GATEWAY_PROBE_PATH: &str = "/_rustack/runtime-probe"; + +/// Non-secret process identity; not authentication or a readiness capability. +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct GatewayInstanceId(Uuid); +impl fmt::Debug for GatewayInstanceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("GatewayInstanceId()") + } +} +impl GatewayInstanceId { + /// Generate a fresh identity with OS randomness, without a panic/fallback. + /// + /// This can block on OS entropy. Use a blocking owner or call + /// [`Self::generate_owned`] from async startup. + /// # Errors + /// Returns a static entropy failure if the OS source is unavailable. + /// ``` + /// let id=rustack_lambda_runtime::probe::GatewayInstanceId::generate()?; + /// # Ok::<(),rustack_lambda_runtime::probe::ProbeError>(()) + /// ``` + pub fn generate() -> Result { + let mut bytes = [0; 16]; + fill(&mut bytes).map_err(|_| ProbeError::Entropy)?; + Ok(Self(Builder::from_random_bytes(bytes).into_uuid())) + } + /// Generate identity in bounded retained blocking work under one deadline. + /// # Errors + /// Rejects nonpositive/over60s budgets, queue expiry, entropy or worker failure. + /// Abandoned observers never publish a late identity or release a running slot. + /// ```no_run + /// # async fn id()->Result<(),rustack_lambda_runtime::probe::ProbeError>{ + /// let id=rustack_lambda_runtime::probe::GatewayInstanceId::generate_owned(tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn generate_owned(deadline: Instant) -> Result { + super::entropy::generate(deadline).await + } +} +impl FromStr for GatewayInstanceId { + type Err = ProbeError; + fn from_str(raw: &str) -> Result { + if raw.len() != 36 { + return Err(ProbeError::Configuration); + } + let id = Uuid::parse_str(raw).map_err(|_| ProbeError::Configuration)?; + if id.is_nil() || id.to_string() != raw { + return Err(ProbeError::Configuration); + } + Ok(Self(id)) + } +} +impl TryFrom for GatewayInstanceId { + type Error = ProbeError; + fn try_from(raw: String) -> Result { + raw.parse() + } +} +impl From for String { + fn from(id: GatewayInstanceId) -> Self { + id.0.to_string() + } +} +/// Validated literal-root operator endpoint; parsing performs no network IO. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct GatewayEndpoint(String); +impl fmt::Debug for GatewayEndpoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("GatewayEndpoint()") + } +} +impl FromStr for GatewayEndpoint { + type Err = ProbeError; + fn from_str(raw: &str) -> Result { + if raw.is_empty() + || raw.len() > 2048 + || raw.chars().any(char::is_control) + || raw.bytes().any(|byte| byte.is_ascii_whitespace()) + || raw.contains('\\') + { + return Err(ProbeError::Configuration); + } + let rest = raw + .split_once("://") + .map(|(_, rest)| rest) + .ok_or(ProbeError::Configuration)?; + let authority = rest + .split(['/', '?', '#']) + .next() + .ok_or(ProbeError::Configuration)?; + let tail = rest + .strip_prefix(authority) + .ok_or(ProbeError::Configuration)?; + if authority.contains('@') || !matches!(tail, "" | "/") { + return Err(ProbeError::Configuration); + } + let url = Url::parse(raw).map_err(|_| ProbeError::Configuration)?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || url.port() == Some(0) + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.path() != "/" + || url.as_str().len() > 2048 + { + return Err(ProbeError::Configuration); + } + Ok(Self(url.into())) + } +} +impl TryFrom for GatewayEndpoint { + type Error = ProbeError; + fn try_from(raw: String) -> Result { + raw.parse() + } +} +impl From for String { + fn from(endpoint: GatewayEndpoint) -> Self { + endpoint.0 + } +} +impl GatewayEndpoint { + /// Canonical operator origin, excluding any caller request path. + /// ``` + /// let endpoint:rustack_lambda_runtime::probe::GatewayEndpoint="http://localhost:4566".parse()?; + /// assert_eq!(endpoint.as_str(),"http://localhost:4566/"); + /// # Ok::<(),rustack_lambda_runtime::probe::ProbeError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} +fn version<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let version = u8::deserialize(deserializer)?; + if version != 1 { + return Err(D::Error::custom("invalid probe schema")); + } + Ok(version) +} +/// Intentional public diagnostic data, never a readiness receipt. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct GatewayReply { + #[serde(deserialize_with = "version")] + version: u8, + instance_id: GatewayInstanceId, +} +impl GatewayReply { + /// Reply for the exact installed process identity. + /// ``` + /// let reply=rustack_lambda_runtime::probe::GatewayReply::new(rustack_lambda_runtime::probe::GatewayInstanceId::generate()?); + /// # Ok::<(),rustack_lambda_runtime::probe::ProbeError>(()) + /// ``` + #[must_use] + pub fn new(instance_id: GatewayInstanceId) -> Self { + Self { + version: 1, + instance_id, + } + } + /// Observed independent process identity. + /// ``` + /// let id=rustack_lambda_runtime::probe::GatewayInstanceId::generate()?; + /// assert_eq!(rustack_lambda_runtime::probe::GatewayReply::new(id).instance_id(),id); + /// # Ok::<(),rustack_lambda_runtime::probe::ProbeError>(()) + /// ``` + #[must_use] + pub fn instance_id(&self) -> GatewayInstanceId { + self.instance_id + } +} +/// Private Runtime API challenge; expected identity is never sent in gateway GET. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct EndpointChallenge { + #[serde(deserialize_with = "version")] + version: u8, + endpoint: GatewayEndpoint, + instance_id: GatewayInstanceId, +} +impl EndpointChallenge { + /// Capture already validated origin and expected instance. + /// ``` + /// let challenge=rustack_lambda_runtime::probe::EndpointChallenge::new("http://localhost:4566".parse()?,rustack_lambda_runtime::probe::GatewayInstanceId::generate()?); + /// # Ok::<(),rustack_lambda_runtime::probe::ProbeError>(()) + /// ``` + #[must_use] + pub fn new(endpoint: GatewayEndpoint, instance_id: GatewayInstanceId) -> Self { + Self { + version: 1, + endpoint, + instance_id, + } + } + pub(in crate::probe) fn decode(bytes: &[u8]) -> Result { + if bytes.len() > REPORT_BYTES { + return Err(ProbeError::Protocol); + } + from_slice(bytes).map_err(|_| ProbeError::Protocol) + } + pub(in crate::probe) fn endpoint(&self) -> &GatewayEndpoint { + &self.endpoint + } + pub(in crate::probe) fn expected(&self) -> GatewayInstanceId { + self.instance_id + } +} +/// Untrusted report data; only the retained host owner can issue endpoint proof. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct EndpointReport { + #[serde(deserialize_with = "version")] + version: u8, + attempt: AttemptId, + architecture: Architecture, + instance_id: GatewayInstanceId, +} +impl EndpointReport { + pub(in crate::probe) fn observed( + attempt: AttemptId, + architecture: Architecture, + instance_id: GatewayInstanceId, + ) -> Self { + Self { + version: 1, + attempt, + architecture, + instance_id, + } + } + /// Consumed attempt to match against the host's actual invocation. + /// ```no_run + /// # fn inspect(r:&rustack_lambda_runtime::probe::EndpointReport){let attempt=r.attempt();} + /// ``` + #[must_use] + pub fn attempt(&self) -> AttemptId { + self.attempt + } + /// Actual compiled probe architecture. + /// ```no_run + /// # fn inspect(r:&rustack_lambda_runtime::probe::EndpointReport){let architecture=r.architecture();} + /// ``` + #[must_use] + pub fn architecture(&self) -> Architecture { + self.architecture + } + /// Independently observed gateway instance identity. + /// ```no_run + /// # fn inspect(r:&rustack_lambda_runtime::probe::EndpointReport){let id=r.instance_id();} + /// ``` + #[must_use] + pub fn instance_id(&self) -> GatewayInstanceId { + self.instance_id + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/probe/endpoint/types/tests.rs b/crates/rustack-lambda-runtime/src/probe/endpoint/types/tests.rs new file mode 100644 index 00000000..a6545d8f --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/endpoint/types/tests.rs @@ -0,0 +1,68 @@ +use rstest::rstest; +use serde_json::{from_str, json, to_string, to_vec}; + +use super::{EndpointChallenge, EndpointReport, GatewayEndpoint, GatewayInstanceId, GatewayReply}; + +#[rstest] +#[case("")] +#[case("00000000-0000-0000-0000-000000000000")] +#[case("AAAAAAAA-1111-4111-8111-111111111111")] +#[case("aaaaaaaa111141118111111111111111")] +fn test_should_reject_noncanonical_or_nil_gateway_identity(#[case] raw: &str) { + assert!(raw.parse::().is_err()); +} +#[test] +fn test_should_roundtrip_validated_identity_reply_and_challenge_without_debug_values() { + let id = GatewayInstanceId::generate().unwrap(); + let raw: String = id.into(); + assert_eq!(raw.parse::().unwrap(), id); + let reply = GatewayReply::new(id); + let text = to_string(&reply).unwrap(); + assert_eq!(from_str::(&text).unwrap(), reply); + assert!(!format!("{id:?}").contains(&raw)); + let endpoint: GatewayEndpoint = "http://private-endpoint.example:4566".parse().unwrap(); + let challenge = EndpointChallenge::new(endpoint, id); + let bytes = to_vec(&challenge).unwrap(); + let decoded = EndpointChallenge::decode(&bytes).unwrap(); + assert_eq!(decoded.expected(), id); + assert!(!format!("{decoded:?}").contains("private-endpoint")); +} +#[rstest] +#[case(" http://localhost")] +#[case("http://localhost ")] +#[case("http://@localhost")] +#[case("http://user:password@localhost")] +#[case("http://localhost/a/..")] +#[case("http://localhost/%2e")] +#[case("http://localhost/?x")] +#[case("http://localhost/#x")] +#[case("http://localhost\\")] +#[case("http://localhost:0")] +#[case("ssh://localhost")] +#[case("http:localhost")] +fn test_should_reject_endpoint_rewrites_before_url_normalization(#[case] raw: &str) { + assert!(raw.parse::().is_err()); +} +#[test] +fn test_should_reject_report_schema_identity_and_challenge_bounds() { + let id = GatewayInstanceId::generate().unwrap(); + let id: String = id.into(); + for text in [ + format!("{{\"version\":0,\"instanceId\":\"{id}\"}}"), + format!("{{\"version\":1,\"instanceId\":\"{id}\",\"extra\":1}}"), + format!("{{\"version\":1,\"instanceId\":\"{id}\",\"instanceId\":\"{id}\"}}"), + format!("{{\"version\":1,\"instanceId\":\"{id}\"}} trailing"), + ] { + assert!(from_str::(&text).is_err()); + } + assert!(EndpointChallenge::decode(&vec![b' '; 4097]).is_err()); + assert!( + EndpointChallenge::decode( + to_vec(&json!({"version":1,"endpoint":"http://localhost/path","instanceId":id})) + .unwrap() + .as_slice() + ) + .is_err() + ); + assert!(from_str::(r#"{"version":2,"attempt":"00000000-0000-0000-0000-000000000000","architecture":"arm64","instanceId":"00000000-0000-0000-0000-000000000000"}"#).is_err()); +} diff --git a/crates/rustack-lambda-runtime/src/probe/mod.rs b/crates/rustack-lambda-runtime/src/probe/mod.rs new file mode 100644 index 00000000..3ecad1fa --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/mod.rs @@ -0,0 +1,168 @@ +//! Typed bounded kernel observations; qualification belongs to the host owner. + +#[cfg(unix)] +mod client; +#[cfg(unix)] +mod collect; +mod endpoint; +#[cfg(any(unix, test))] +mod parse; + +use std::{fmt, io, sync::Arc}; + +pub use endpoint::types::{ + EndpointChallenge, EndpointReport, GATEWAY_PROBE_PATH, GatewayEndpoint, GatewayInstanceId, + GatewayReply, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use typed_builder::TypedBuilder; + +use crate::protocol::{Architecture, AttemptId}; + +/// Maximum encoded resource report/challenge response size. +pub const REPORT_BYTES: usize = 4096; + +/// Static probe errors never display kernel text or filesystem paths. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum ProbeError { + /// No runtime or retained probe worker failed to join. + #[error("runtime probe worker failed")] + Worker, + /// The OS randomness source failed; no identity fallback is generated. + #[error("runtime probe entropy failed")] + Entropy, + /// TLS configuration or certificate/name verification failed; peer text is discarded. + #[error("runtime probe TLS failed")] + Tls, + /// Unsupported platform, arguments or private API configuration. + #[error("invalid runtime probe configuration")] + Configuration, + /// Incomplete, duplicate, malformed or unsupported kernel observations. + #[error("invalid runtime probe observation")] + Observation, + /// The private API protocol/challenge did not match. + #[error("invalid runtime probe protocol")] + Protocol, + /// Absolute collection/request budget expired. + #[error("runtime probe deadline exceeded")] + Deadline, + /// Fixed local kernel/API IO failed. + #[error("runtime probe IO failed")] + Io(#[source] Arc), + /// Private Runtime API HTTP failed. + #[error("runtime probe HTTP failed")] + Http(#[source] Arc), + /// Typed report serialization failed. + #[error("runtime probe report failed")] + Report(#[source] Arc), +} +impl fmt::Debug for ProbeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl From for ProbeError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} +impl From for ProbeError { + fn from(error: hyper::Error) -> Self { + Self::Http(Arc::new(error)) + } +} + +/// Observed container-local temporary filesystem, without mount source text. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct TemporaryReport { + /// Kernel-reported tmpfs size in bytes. + pub bytes: u64, + /// Actual permission/special mode bits. + pub mode: u32, + /// The mount filesystem is tmpfs. + pub tmpfs: bool, + /// The effective mount disallows setuid transitions. + pub nosuid: bool, + /// The effective mount disallows device interpretation. + pub nodev: bool, +} + +/// Finite/unlimited normalized controller and process observations. +/// +/// A deserialized report is untrusted data, never a readiness capability. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ResourceReport { + /// Report schema, currently1. + pub version: u8, + /// Consumed Runtime API attempt identity. + pub attempt: AttemptId, + /// Actual compiled Linux architecture. + pub architecture: Architecture, + /// Required controller memberships are rooted in this private namespace. + pub cgroup_root: bool, + /// Observed memory controller value; textual unlimited becomes None. Numeric + /// v1 unlimited sentinels retain their numeric value and cannot qualify. + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_max: Option, + /// Observed extra swap cap; textual unlimited becomes None. + #[serde(skip_serializing_if = "Option::is_none")] + pub swap_max: Option, + /// Process/thread hard cap; None means unlimited. + #[serde(skip_serializing_if = "Option::is_none")] + pub pids_max: Option, + /// CPU time quota in microseconds; None means unlimited. + #[serde(skip_serializing_if = "Option::is_none")] + pub cpu_quota: Option, + /// CPU scheduling period in microseconds. + pub cpu_period: u64, + /// Effective UID observed from proc status. + pub euid: u32, + /// Effective GID observed from proc status. + pub egid: u32, + /// Effective Linux capability bitmask. + pub cap_effective: u64, + /// Actual `no_new_privs` flag. + pub no_new_privileges: bool, + /// Actual seccomp mode (2 means filter mode). + pub seccomp_mode: u8, + /// Root's effective mount flags include read-only. + pub root_readonly: bool, + /// Actual /tmp mount observation. + pub temporary: TemporaryReport, +} + +/// Run one private Linux probe challenge, then wait for host-owned retirement. +/// +/// # Errors +/// Rejects non-Linux execution, any endpoint other than fixed loopback9001, +/// malformed challenges, expired budgets and failed kernel/API observations. +/// +/// ```no_run +/// # #[cfg(unix)] async fn probe() { +/// let outcome = rustack_lambda_runtime::probe::run().await; +/// # } +/// ``` +#[cfg(unix)] +pub async fn run() -> Result<(), ProbeError> { + client::run().await +} + +/// Run one pinned gateway probe from the actual function network. +/// # Errors +/// Rejects invalid private challenges, unreachable/wrong endpoints, TLS/protocol +/// failure or expiry. The host owns physical function retirement. +/// ```no_run +/// # #[cfg(unix)] async fn probe(){ +/// let result=rustack_lambda_runtime::probe::run_endpoint().await; +/// # } +/// ``` +#[cfg(unix)] +pub async fn run_endpoint() -> Result<(), ProbeError> { + client::run_endpoint().await +} diff --git a/crates/rustack-lambda-runtime/src/probe/parse.rs b/crates/rustack-lambda-runtime/src/probe/parse.rs new file mode 100644 index 00000000..747d2424 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/parse.rs @@ -0,0 +1,214 @@ +//! Checked parsers over fixed, bounded kernel text; no paths are derived from it. + +use super::{ProbeError, TemporaryReport}; + +pub(super) fn number(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() || value.len() > 20 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProbeError::Observation); + } + value.parse().map_err(|_| ProbeError::Observation) +} +pub(super) fn limit(value: &str) -> Result, ProbeError> { + if matches!(value.trim(), "max" | "-1") { + Ok(None) + } else { + number(value).map(Some) + } +} +pub(super) fn cpu(value: &str) -> Result<(Option, u64), ProbeError> { + let mut fields = value.split_ascii_whitespace(); + let quota = limit(fields.next().ok_or(ProbeError::Observation)?)?; + let period = number(fields.next().ok_or(ProbeError::Observation)?)?; + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + Ok((quota, period)) +} +pub(super) struct Status { + pub(super) uid: u32, + pub(super) gid: u32, + pub(super) capabilities: u64, + pub(super) no_new_privileges: bool, + pub(super) seccomp: u8, +} +pub(super) fn status(text: &str) -> Result { + bounded(text)?; + let field = |key: &str| -> Result<&str, ProbeError> { + let mut matching = text + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| *name == key) + .map(|(_, value)| value.trim()); + let value = matching.next().ok_or(ProbeError::Observation)?; + if matching.next().is_some() { + return Err(ProbeError::Observation); + } + Ok(value) + }; + let effective = |text: &str| -> Result { + let mut fields = text.split_ascii_whitespace(); + let mut ids = [0_u32; 4]; + for id in &mut ids { + *id = u32::try_from(number(fields.next().ok_or(ProbeError::Observation)?)?) + .map_err(|_| ProbeError::Observation)?; + } + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + let [_, effective, _, _] = ids; + Ok(effective) + }; + let cap = field("CapEff")?; + if cap.len() != 16 || !cap.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(ProbeError::Observation); + } + let no_new_privileges = match field("NoNewPrivs")? { + "0" => false, + "1" => true, + _ => return Err(ProbeError::Observation), + }; + Ok(Status { + uid: effective(field("Uid")?)?, + gid: effective(field("Gid")?)?, + capabilities: u64::from_str_radix(cap, 16).map_err(|_| ProbeError::Observation)?, + no_new_privileges, + seccomp: u8::try_from(number(field("Seccomp")?)?).map_err(|_| ProbeError::Observation)?, + }) +} +fn bounded(text: &str) -> Result<(), ProbeError> { + if text.len() > 65536 + || text.lines().count() > 512 + || text.lines().any(|line| line.len() > 4096) + { + Err(ProbeError::Observation) + } else { + Ok(()) + } +} +pub(super) fn membership(text: &str, unified: bool) -> Result { + bounded(text)?; + let mut v2 = None; + let mut required = [None; 3]; + for line in text.lines() { + let mut fields = line.splitn(3, ':'); + let id = number(fields.next().ok_or(ProbeError::Observation)?)?; + u32::try_from(id).map_err(|_| ProbeError::Observation)?; + let controllers = fields.next().ok_or(ProbeError::Observation)?; + let path = fields.next().ok_or(ProbeError::Observation)?; + if controllers.len() > 256 || controllers.split(',').count() > 32 || path.is_empty() { + return Err(ProbeError::Observation); + } + if controllers.is_empty() { + if id != 0 { + return Err(ProbeError::Observation); + } + if v2.replace(path == "/").is_some() { + return Err(ProbeError::Observation); + } + } else if !unified { + if id == 0 { + return Err(ProbeError::Observation); + } + for controller in controllers.split(',') { + let slot = match controller { + "memory" => required.get_mut(0), + "pids" => required.get_mut(1), + "cpu" => required.get_mut(2), + _ => None, + }; + if let Some(slot) = slot { + if slot.replace(path == "/").is_some() { + return Err(ProbeError::Observation); + } + } + } + } + } + if unified { + v2.ok_or(ProbeError::Observation) + } else { + if required.iter().any(Option::is_none) { + return Err(ProbeError::Observation); + } + Ok(required.iter().all(|value| *value == Some(true))) + } +} +fn size(value: &str) -> Result { + let (digits, scale) = match value.as_bytes().last() { + Some(b'k') => ( + value.strip_suffix('k').ok_or(ProbeError::Observation)?, + 1024, + ), + Some(b'm') => ( + value.strip_suffix('m').ok_or(ProbeError::Observation)?, + 1024 * 1024, + ), + Some(b'g') => ( + value.strip_suffix('g').ok_or(ProbeError::Observation)?, + 1024 * 1024 * 1024, + ), + _ => (value, 1), + }; + number(digits)? + .checked_mul(scale) + .ok_or(ProbeError::Observation) +} +pub(super) fn mounts(text: &str, mode: u32) -> Result<(bool, TemporaryReport), ProbeError> { + bounded(text)?; + let mut root = None; + let mut temporary = None; + for line in text.lines() { + let mut fields = line.split_ascii_whitespace(); + for _ in 0..4 { + fields.next().ok_or(ProbeError::Observation)?; + } + let target = fields.next().ok_or(ProbeError::Observation)?; + let options = fields.next().ok_or(ProbeError::Observation)?; + if !matches!(target, "/" | "/tmp") { + continue; + } + if !fields.by_ref().any(|field| field == "-") { + return Err(ProbeError::Observation); + } + let filesystem = fields.next().ok_or(ProbeError::Observation)?; + fields.next().ok_or(ProbeError::Observation)?; + let super_options = fields.next().ok_or(ProbeError::Observation)?; + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + if target == "/" { + if root + .replace(options.split(',').any(|v| v == "ro")) + .is_some() + { + return Err(ProbeError::Observation); + } + } else { + let mut sizes = super_options + .split(',') + .filter_map(|v| v.strip_prefix("size=")); + let bytes = size(sizes.next().ok_or(ProbeError::Observation)?)?; + if sizes.next().is_some() { + return Err(ProbeError::Observation); + } + let observed = TemporaryReport::builder() + .bytes(bytes) + .mode(mode) + .tmpfs(filesystem == "tmpfs") + .nosuid(options.split(',').any(|v| v == "nosuid")) + .nodev(options.split(',').any(|v| v == "nodev")) + .build(); + if temporary.replace(observed).is_some() { + return Err(ProbeError::Observation); + } + } + } + Ok(( + root.ok_or(ProbeError::Observation)?, + temporary.ok_or(ProbeError::Observation)?, + )) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/probe/parse/tests.rs b/crates/rustack-lambda-runtime/src/probe/parse/tests.rs new file mode 100644 index 00000000..93969eeb --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/parse/tests.rs @@ -0,0 +1,83 @@ +use super::*; + +const STATUS: &str = "Uid:\t1000 1001 1000 1000\nGid:\t1000 1002 1000 \ + 1000\nCapEff:\t0000000000000000\nNoNewPrivs:\t1\nSeccomp:\t2\n"; +const MOUNTS: &str = "1 0 0:1 / / ro,relatime - overlay overlay rw\n2 1 0:2 / /tmp \ + rw,nosuid,nodev,relatime - tmpfs tmpfs rw,size=524288k,inode64\n"; +#[test] +fn test_should_parse_independent_kernel_observation_fixtures() { + let report = status(STATUS).unwrap(); + assert_eq!((report.uid, report.gid), (1001, 1002)); + assert_eq!(report.capabilities, 0); + assert!(report.no_new_privileges); + assert_eq!(report.seccomp, 2); + let (root, tmp) = mounts(MOUNTS, 0o1777).unwrap(); + assert!(root); + assert_eq!((tmp.bytes, tmp.mode), (512 * 1024 * 1024, 0o1777)); + assert!(tmp.tmpfs && tmp.nosuid && tmp.nodev); + assert_eq!(cpu("100000 100000\n").unwrap(), (Some(100_000), 100_000)); + assert_eq!(cpu("max 100000").unwrap(), (None, 100_000)); + assert_eq!(limit("-1").unwrap(), None); + assert_eq!( + limit("9223372036854771712").unwrap(), + Some(9_223_372_036_854_771_712) + ); +} +#[rstest::rstest] +#[case::empty("")] +#[case::overflow("18446744073709551616")] +#[case::negative("-2")] +#[case::fraction("1.0")] +#[case::nonascii("١")] +#[case::multiple("1 2")] +#[test] +fn test_should_reject_invalid_kernel_numbers(#[case] value: &str) { + assert!(number(value).is_err()); +} +#[rstest::rstest] +#[case::missing("Seccomp:\t2\n", "")] +#[case::duplicate("Seccomp:\t2\n", "Seccomp:\t2\nSeccomp:\t2\n")] +#[case::short_uid("1000 1001 1000 1000", "1000 1001 1000")] +#[case::long_uid("1000 1001 1000 1000", "1000 1001 1000 1000 1000")] +#[case::uid_overflow("1000 1001 1000 1000", "1000 4294967296 1000 1000")] +#[case::bad_privilege("NoNewPrivs:\t1", "NoNewPrivs:\t2")] +#[case::bad_cap("0000000000000000", "00000000000000zz")] +#[test] +fn test_should_reject_missing_duplicate_or_invalid_process_status( + #[case] before: &str, + #[case] after: &str, +) { + assert!(status(&STATUS.replace(before, after)).is_err()); +} +#[rstest::rstest] +#[case::missing( + "2 1 0:2 / /tmp rw,nosuid,nodev,relatime - tmpfs tmpfs rw,size=524288k,inode64\n", + "" +)] +#[case::size("size=524288k", "size=18446744073709551615g")] +#[case::duplicate_size("size=524288k", "size=524288k,size=1")] +#[case::missing_separator(" - tmpfs", " tmpfs")] +#[test] +fn test_should_reject_incomplete_or_overflowing_mounts(#[case] before: &str, #[case] after: &str) { + assert!(mounts(&MOUNTS.replace(before, after), 0o1777).is_err()); +} +#[test] +fn test_should_bound_lines_and_reject_duplicate_root_or_temporary_mounts() { + assert!(status(&"x".repeat(65537)).is_err()); + assert!(status(&format!("{STATUS}{}", "x\n".repeat(513))).is_err()); + assert!(mounts(&format!("{MOUNTS}{MOUNTS}"), 0o1777).is_err()); + assert!(cpu("1 2 3").is_err()); +} + +#[test] +fn test_should_require_current_controller_membership_at_namespace_root() { + assert!(membership("0::/\n", true).unwrap()); + assert!(!membership("0::/docker/foreign\n", true).unwrap()); + let v1 = "2:memory:/\n3:pids:/\n4:cpu,cpuacct:/\n5:name=systemd:/unrelated\n"; + assert!(membership(v1, false).unwrap()); + assert!(!membership(&v1.replace("memory:/", "memory:/docker/current"), false).unwrap()); + assert!(membership(&v1.replace("2:memory:/\n", ""), false).is_err()); + assert!(membership(&format!("{v1}9:memory:/\n"), false).is_err()); + assert!(membership("0::/\n0::/\n", true).is_err()); + assert!(membership("1::/\n", true).is_err()); +} diff --git a/crates/rustack-lambda-runtime/src/process.rs b/crates/rustack-lambda-runtime/src/process.rs new file mode 100644 index 00000000..964c801e --- /dev/null +++ b/crates/rustack-lambda-runtime/src/process.rs @@ -0,0 +1,84 @@ +//! Linux process wiring with nonblocking owned pipe descriptors. + +use std::{ + env, + io::{stdin, stdout}, + os::fd::AsFd, + time::Duration, +}; + +use rustack_lambda_runtime::{ + http::start_at, + protocol::{Architecture, EnvironmentId}, + relay::{RelayConfig, RelayError, start}, +}; +use tokio::net::unix::pipe::{Receiver, Sender}; + +fn seconds(key: &str, default: u64, maximum: u64) -> Result { + let value = match env::var(key) { + Ok(value) => { + if value.len() > 5 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(RelayError::Configuration); + } + let parsed = value + .parse::() + .map_err(|_| RelayError::Configuration)?; + if parsed.to_string() != value { + return Err(RelayError::Configuration); + } + parsed + } + Err(env::VarError::NotPresent) => default, + Err(_) => return Err(RelayError::Configuration), + }; + if value == 0 || value > maximum { + return Err(RelayError::Configuration); + } + Ok(Duration::from_secs(value)) +} + +/// Run the private Linux relay process. +pub(crate) async fn run() -> Result<(), RelayError> { + let mut arguments = env::args_os(); + arguments.next(); + match arguments.next() { + None => {} + Some(command) if command == "probe" && arguments.next().is_none() => { + return rustack_lambda_runtime::probe::run() + .await + .map_err(|_| RelayError::Configuration); + } + Some(command) if command == "endpoint-probe" && arguments.next().is_none() => { + return rustack_lambda_runtime::probe::run_endpoint() + .await + .map_err(|_| RelayError::Configuration); + } + Some(_) => return Err(RelayError::Configuration), + } + let environment = + env::var("RUSTACK_LAMBDA_ENVIRONMENT_ID").map_err(|_| RelayError::Configuration)?; + if environment.len() != 36 { + return Err(RelayError::Configuration); + } + let environment: EnvironmentId = environment.parse().map_err(|_| RelayError::Configuration)?; + let architecture = match env::consts::ARCH { + "x86_64" => Architecture::X86_64, + "aarch64" => Architecture::Arm64, + _ => return Err(RelayError::UnsupportedPlatform), + }; + let idle = seconds("RUSTACK_LAMBDA_RELAY_IDLE_SECONDS", 600, 86_400)?; + let io = seconds("RUSTACK_LAMBDA_RELAY_IO_SECONDS", 5, 60)?; + let retirement = seconds("RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS", 10, 60)?; + let config = RelayConfig::new(environment, architecture, idle, io, retirement)?; + let input = Receiver::from_owned_fd(stdin().as_fd().try_clone_to_owned()?)?; + let output = Sender::from_owned_fd(stdout().as_fd().try_clone_to_owned()?)?; + let api = start_at( + "127.0.0.1:9001" + .parse() + .map_err(|_| RelayError::Configuration)?, + idle, + ) + .await?; + let mut relay = start(api, config, input, output)?; + relay.wait().await +} diff --git a/crates/rustack-lambda-runtime/src/protocol/mod.rs b/crates/rustack-lambda-runtime/src/protocol/mod.rs new file mode 100644 index 00000000..45f772e3 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/mod.rs @@ -0,0 +1,273 @@ +//! Length-prefixed, bounded CBOR for one environment's attach connection. +//! +//! A partial read/write failure invalidates the connection. Callers own the IO +//! task and must retire that transport rather than retrying a partially read +//! frame or replaying an invocation. + +mod preflight; +#[cfg(test)] +mod tests; +mod types; + +use std::{io, num::TryFromIntError}; + +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + time::{Instant, timeout_at}, +}; +pub use types::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, FunctionArn, HandlerTimeout, Message, + Outcome, Payload, TraceId, +}; + +/// Maximum event or result payload, in bytes. +pub const MAX_PAYLOAD_BYTES: usize = 6 * 1024 * 1024; +/// Maximum CBOR envelope metadata, excluding raw payload bytes. +pub const MAX_METADATA_BYTES: usize = 64 * 1024; +/// Maximum frame body, excluding its four-byte prefix. +pub const MAX_FRAME_BYTES: usize = MAX_PAYLOAD_BYTES + MAX_METADATA_BYTES; +/// Host/relay contract version; older relays cannot enforce delivery-based timeouts. +pub const PROTOCOL_VERSION: u32 = 2; + +/// Protocol failures contain classes and IO sources, never invalid peer values. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ProtocolError { + /// A declared/allotted length exceeds a protocol budget. + #[error("control protocol size budget exceeded")] + TooLarge, + /// The byte structure is invalid, unsupported, duplicated or truncated. + #[error("invalid control CBOR structure")] + InvalidStructure, + /// The valid CBOR does not match the typed message schema. + #[error("invalid control message schema")] + InvalidSchema, + /// An incompatible host/relay version was supplied. + #[error("unsupported control protocol version")] + UnsupportedVersion, + /// A validated primitive has invalid syntax or range. + #[error("invalid control metadata")] + InvalidMetadata, + /// IO failed, including truncated transport input. + #[error("control transport IO failed")] + Io(#[from] io::Error), + /// An absolute IO deadline expired; the transport must be retired. + #[error("control transport deadline exceeded")] + Deadline, +} + +impl From for ProtocolError { + fn from(_: TryFromIntError) -> Self { + Self::TooLarge + } +} + +/// A validated, environment-scoped control envelope. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", try_from = "RawFrame")] +#[non_exhaustive] +pub struct Frame { + version: u32, + environment: EnvironmentId, + message: Message, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RawFrame { + version: u32, + environment: EnvironmentId, + message: Message, +} + +impl TryFrom for Frame { + type Error = ProtocolError; + + fn try_from(raw: RawFrame) -> Result { + if raw.version != PROTOCOL_VERSION { + return Err(ProtocolError::UnsupportedVersion); + } + Ok(Self::new(raw.environment, raw.message)) + } +} + +impl Frame { + /// Wrap a typed message in the current protocol version. + /// + /// ``` + /// use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Ready); + /// assert_eq!(frame.message(), &Message::Ready); + /// ``` + #[must_use] + pub fn new(environment: EnvironmentId, message: Message) -> Self { + Self { + version: PROTOCOL_VERSION, + environment, + message, + } + } + + /// Identity that must match the connection's owned environment. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let id = EnvironmentId::default(); + /// assert_eq!(Frame::new(id, Message::Close).environment(), id); + /// ``` + #[must_use] + pub fn environment(&self) -> EnvironmentId { + self.environment + } + + /// Borrow the typed message without cloning its payload. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Close); + /// assert!(matches!(frame.message(), Message::Close)); + /// ``` + #[must_use] + pub fn message(&self) -> &Message { + &self.message + } + + /// Move the message into the owning session or transport actor. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Close); + /// assert_eq!(frame.into_message(), Message::Close); + /// ``` + #[must_use] + pub fn into_message(self) -> Message { + self.message + } +} + +/// Encode one complete frame, including the big-endian length prefix. +/// +/// # Errors +/// Rejects excessive metadata or an encoding/size failure. +/// +/// ``` +/// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message, encode}; +/// let bytes = encode(&Frame::new(EnvironmentId::default(), Message::Ready))?; +/// assert!(bytes.len() > 4); +/// # Ok::<(), rustack_lambda_runtime::protocol::ProtocolError>(()) +/// ``` +pub fn encode(frame: &Frame) -> Result { + let mut output = vec![0; 4]; + ciborium::into_writer(frame, &mut output).map_err(|_| ProtocolError::InvalidSchema)?; + let body = output.get(4..).ok_or(ProtocolError::InvalidStructure)?; + preflight::validate(body)?; + let length = u32::try_from(body.len())?.to_be_bytes(); + output + .get_mut(..4) + .ok_or(ProtocolError::InvalidStructure)? + .copy_from_slice(&length); + Ok(Bytes::from(output)) +} + +/// Decode one bounded frame body, without its length prefix. +/// +/// # Errors +/// Rejects malformed CBOR, unsupported schema/version or invalid metadata. +/// No untrusted text is included in the returned error. +/// +/// ``` +/// use rustack_lambda_runtime::protocol::decode; +/// assert!(decode(&[]).is_err()); +/// ``` +pub fn decode(body: &[u8]) -> Result { + preflight::validate(body)?; + let raw: RawFrame = ciborium::from_reader(body).map_err(|_| ProtocolError::InvalidSchema)?; + Frame::try_from(raw) +} + +/// Read a frame under one absolute deadline; None denotes clean boundary EOF. +/// +/// # Errors +/// Returns IO/truncation, deadline or protocol validation errors. A failed or +/// cancelled read invalidates the connection, including any consumed prefix. +/// +/// ```no_run +/// # async fn read(mut input: impl tokio::io::AsyncRead + Unpin) -> Result<(), rustack_lambda_runtime::protocol::ProtocolError> { +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// let frame = rustack_lambda_runtime::protocol::read_frame(&mut input, deadline).await?; +/// # Ok(()) } +/// ``` +pub async fn read_frame( + reader: &mut R, + deadline: Instant, +) -> Result, ProtocolError> { + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + let frame = timeout_at(deadline, async { + let mut prefix = [0; 4]; + let count = reader.read(&mut prefix).await?; + if count == 0 { + return Ok(None); + } + reader + .read_exact( + prefix + .get_mut(count..) + .ok_or(ProtocolError::InvalidStructure)?, + ) + .await?; + let length = usize::try_from(u32::from_be_bytes(prefix))?; + if length > MAX_FRAME_BYTES { + return Err(ProtocolError::TooLarge); + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).await?; + decode(&body).map(Some) + }) + .await + .map_err(|_| ProtocolError::Deadline)??; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + Ok(frame) +} + +/// Encode, write and flush a frame under one absolute deadline. +/// +/// # Errors +/// Returns encoding, IO or deadline failure. Partial writes require retiring +/// the connection; retrying a business invocation is never implicit. +/// +/// ```no_run +/// # async fn write(mut output: impl tokio::io::AsyncWrite + Unpin, frame: rustack_lambda_runtime::protocol::Frame) -> Result<(), rustack_lambda_runtime::protocol::ProtocolError> { +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// rustack_lambda_runtime::protocol::write_frame(&mut output, &frame, deadline).await?; +/// # Ok(()) } +/// ``` +pub async fn write_frame( + writer: &mut W, + frame: &Frame, + deadline: Instant, +) -> Result<(), ProtocolError> { + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + let encoded = encode(frame)?; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + timeout_at(deadline, async { + writer.write_all(&encoded).await?; + writer.flush().await + }) + .await + .map_err(|_| ProtocolError::Deadline)??; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + Ok(()) +} diff --git a/crates/rustack-lambda-runtime/src/protocol/preflight.rs b/crates/rustack-lambda-runtime/src/protocol/preflight.rs new file mode 100644 index 00000000..e387550e --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/preflight.rs @@ -0,0 +1,140 @@ +//! Allocation-bounded CBOR structural validation before Serde materialization. + +use std::{collections::HashSet, str}; + +use super::{MAX_FRAME_BYTES, MAX_METADATA_BYTES, MAX_PAYLOAD_BYTES, ProtocolError}; + +#[derive(Debug)] +struct Scanner<'a> { + remaining: &'a [u8], + members: usize, + payload: Option, +} + +pub(super) fn validate(body: &[u8]) -> Result<(), ProtocolError> { + if body.len() > MAX_FRAME_BYTES { + return Err(ProtocolError::TooLarge); + } + let mut scanner = Scanner { + remaining: body, + members: 0, + payload: None, + }; + scanner.item(1, false)?; + if !scanner.remaining.is_empty() { + return Err(ProtocolError::InvalidStructure); + } + if body.len().saturating_sub(scanner.payload.unwrap_or(0)) > MAX_METADATA_BYTES { + return Err(ProtocolError::TooLarge); + } + Ok(()) +} + +impl<'a> Scanner<'a> { + fn take(&mut self, length: usize) -> Result<&'a [u8], ProtocolError> { + let (value, rest) = self + .remaining + .split_at_checked(length) + .ok_or(ProtocolError::InvalidStructure)?; + self.remaining = rest; + Ok(value) + } + + fn header(&mut self) -> Result<(u8, u64), ProtocolError> { + let (&initial, rest) = self + .remaining + .split_first() + .ok_or(ProtocolError::InvalidStructure)?; + self.remaining = rest; + let major = initial >> 5; + if major == 7 && initial != 0xf4 && initial != 0xf5 { + return Err(ProtocolError::InvalidStructure); + } + let additional = initial & 31; + let value = match additional { + 0..=23 => u64::from(additional), + 24 => u64::from(u8::from_be_bytes( + self.take(1)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 25 => u64::from(u16::from_be_bytes( + self.take(2)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 26 => u64::from(u32::from_be_bytes( + self.take(4)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 27 => u64::from_be_bytes( + self.take(8)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + ), + _ => return Err(ProtocolError::InvalidStructure), + }; + Ok((major, value)) + } + + fn text(&mut self, key: bool) -> Result<&'a str, ProtocolError> { + let (major, length) = self.header()?; + if major != 3 { + return Err(ProtocolError::InvalidStructure); + } + let length = usize::try_from(length)?; + if length > if key { 64 } else { 2048 } { + return Err(ProtocolError::TooLarge); + } + str::from_utf8(self.take(length)?).map_err(|_| ProtocolError::InvalidStructure) + } + + fn item(&mut self, depth: usize, payload: bool) -> Result<(), ProtocolError> { + if depth > 16 { + return Err(ProtocolError::TooLarge); + } + let (major, value) = self.header()?; + match major { + 0 => Ok(()), + 2 if payload && self.payload.is_none() => { + let length = usize::try_from(value)?; + if length > MAX_PAYLOAD_BYTES { + return Err(ProtocolError::TooLarge); + } + self.take(length)?; + self.payload = Some(length); + Ok(()) + } + 3 => { + let length = usize::try_from(value)?; + if length > 2048 { + return Err(ProtocolError::TooLarge); + } + str::from_utf8(self.take(length)?).map_err(|_| ProtocolError::InvalidStructure)?; + Ok(()) + } + 5 => { + let count = usize::try_from(value)?; + self.members = self + .members + .checked_add(count) + .ok_or(ProtocolError::TooLarge)?; + if self.members > 128 { + return Err(ProtocolError::TooLarge); + } + let mut keys = HashSet::with_capacity(count); + for _ in 0..count { + let key = self.text(true)?; + if !keys.insert(key) { + return Err(ProtocolError::InvalidStructure); + } + self.item(depth + 1, key == "payload")?; + } + Ok(()) + } + 7 if value == 20 || value == 21 => Ok(()), + _ => Err(ProtocolError::InvalidStructure), + } + } +} diff --git a/crates/rustack-lambda-runtime/src/protocol/tests.rs b/crates/rustack-lambda-runtime/src/protocol/tests.rs new file mode 100644 index 00000000..1081b287 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/tests.rs @@ -0,0 +1,506 @@ +use std::{num::NonZeroU64, time::Duration}; + +use ciborium::Value; +use proptest::{collection, prelude::*}; +use rstest::rstest; +use tokio::io::{duplex, empty}; + +use super::*; + +fn payload() -> Payload { + Bytes::from_static(b"secret-payload\0\xff") + .try_into() + .unwrap() +} + +fn messages() -> Vec { + let attempt = AttemptId::default(); + vec![ + Message::Hello { + architecture: Architecture::Arm64, + }, + Message::Hello { + architecture: Architecture::X86_64, + }, + Message::Ready, + Message::BootstrapReady, + Message::Invoke { + attempt, + deadline_unix_ms: NonZeroU64::new(1_800_000_000_000).unwrap(), + handler_timeout_ms: None, + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + trace_id: "Root=1-00000000-000000000000000000000000;Sampled=0" + .parse() + .unwrap(), + payload: payload(), + }, + Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: payload(), + }, + Message::InvocationResult { + attempt, + outcome: Outcome::FunctionError, + payload: payload(), + }, + Message::InitError { payload: payload() }, + Message::Cancel { attempt }, + Message::Close, + Message::Ack(Acknowledgement::Close), + Message::Ack(Acknowledgement::Cancel(attempt)), + Message::InvocationTimedOut { attempt }, + ] +} + +fn body(value: &Value) -> Vec { + let mut bytes = Vec::new(); + ciborium::into_writer(value, &mut bytes).unwrap(); + bytes +} + +fn value(message: Message) -> Value { + let encoded = encode(&Frame::new(EnvironmentId::default(), message)).unwrap(); + ciborium::from_reader(encoded.get(4..).unwrap()).unwrap() +} + +fn field<'a>(value: &'a mut Value, key: &str) -> &'a mut Value { + value + .as_map_mut() + .unwrap() + .iter_mut() + .find_map(|(k, v)| (k.as_text() == Some(key)).then_some(v)) + .unwrap() +} + +#[test] +fn test_should_round_trip_every_variant_and_redact_payload_debug() { + for message in messages() { + let frame = Frame::new(EnvironmentId::default(), message); + let bytes = encode(&frame).unwrap(); + let length = u32::from_be_bytes(bytes.get(..4).unwrap().try_into().unwrap()); + assert_eq!(usize::try_from(length).unwrap(), bytes.len() - 4); + assert_eq!(decode(bytes.get(4..).unwrap()).unwrap(), frame); + assert!(!format!("{frame:?}").contains("secret-payload")); + } + let encoded = value(Message::InitError { payload: payload() }); + let mut encoded = encoded; + let data = field(field(&mut encoded, "message"), "data"); + assert!(matches!(field(data, "payload"), Value::Bytes(_))); +} + +#[rstest] +#[case("00000000-0000-0000-0000-000000000000")] +#[case("00112233-4455-6677-8899-AABBCCDDEEFF")] +#[case("00112233445566778899aabbccddeeff")] +#[case("00112233-4455-6677-8899-aabbccddeeff\n")] +#[case("not-an-id")] +fn test_should_reject_noncanonical_or_nil_identities(#[case] value: &str) { + assert!(value.parse::().is_err()); + assert!(value.parse::().is_err()); +} + +#[rstest] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:$LATEST")] +#[case("arn:aws-cn:lambda:cn-north-1:123456789012:function:echo:1")] +#[case("arn:aws-us-gov:lambda:us-gov-west-1:123456789012:function:echo:prod")] +fn test_should_accept_scoped_function_metadata(#[case] value: &str) { + assert_eq!(value.parse::().unwrap().as_ref(), value); +} + +#[rstest] +#[case("")] +#[case("arn:aws:s3:us-east-1:000000000000:function:echo")] +#[case("arn:aws:lambda::000000000000:function:echo")] +#[case("arn:aws:lambda:us-east-1:0000:function:echo")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:0")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:18446744073709551616")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:prod:extra")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo\r\nsecret")] +fn test_should_reject_invalid_header_metadata_without_echoing_it(#[case] value: &str) { + let error = value.parse::().unwrap_err(); + assert_eq!(error.to_string(), "invalid control metadata"); + assert!(!format!("{error:?}").contains("secret")); + assert!("trace\nsecret".parse::().is_err()); + assert!("é".parse::().is_err()); + assert!("x".repeat(257).parse::().is_err()); + + let mut packet = self::value(messages().into_iter().nth(4).unwrap()); + *field(field(field(&mut packet, "message"), "data"), "functionArn") = Value::Text(value.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn test_should_never_panic_on_bounded_malformed_control_input(bytes in collection::vec(any::(), 0..8192)) { + if let Ok(frame) = decode(&bytes) { + let encoded = encode(&frame).unwrap(); + prop_assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + } + } + + #[test] + fn test_should_preserve_arbitrary_binary_payloads(bytes in collection::vec(any::(), 0..16384)) { + let frame = Frame::new(EnvironmentId::default(), Message::InitError { + payload: Payload::try_from(Bytes::from(bytes)).unwrap(), + }); + let encoded = encode(&frame).unwrap(); + prop_assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + } + + #[test] + fn test_should_handle_mutated_valid_envelopes_without_panicking(position in any::(), mask in any::()) { + let frame = Frame::new(EnvironmentId::default(), Message::InitError { payload: payload() }); + let mut encoded = encode(&frame).unwrap().get(4..).unwrap().to_vec(); + let position = position % encoded.len(); + *encoded.get_mut(position).unwrap() ^= mask; + if let Ok(frame) = decode(&encoded) { + let bytes = encode(&frame).unwrap(); + prop_assert_eq!(decode(bytes.get(4..).unwrap()).unwrap(), frame); + } + } +} + +#[test] +fn test_should_reject_unknown_duplicate_and_mismatched_variant_fields() { + let frame = value(Message::Ready); + let mut unknown = frame.clone(); + unknown + .as_map_mut() + .unwrap() + .push((Value::Text("secret-unrecognized".into()), Value::Bool(true))); + let error = decode(&body(&unknown)).unwrap_err(); + assert!(matches!(error, ProtocolError::InvalidSchema)); + assert!(!format!("{error:?} {error}").contains("secret-unrecognized")); + let mut duplicate = frame.clone(); + let first = duplicate.as_map().unwrap().first().unwrap().clone(); + duplicate.as_map_mut().unwrap().push(first); + assert!(matches!( + decode(&body(&duplicate)), + Err(ProtocolError::InvalidStructure) + )); + let mut unknown_kind = frame.clone(); + *field(field(&mut unknown_kind, "message"), "kind") = Value::Text("secret-variant".into()); + assert!(matches!( + decode(&body(&unknown_kind)), + Err(ProtocolError::InvalidSchema) + )); + let mut version = frame; + *field(&mut version, "version") = Value::Integer(1.into()); + assert!(matches!( + decode(&body(&version)), + Err(ProtocolError::UnsupportedVersion) + )); + + let mut ack = value(Message::Ack(Acknowledgement::Cancel(AttemptId::default()))); + *field(field(field(&mut ack, "message"), "data"), "operation") = Value::Text("close".into()); + assert!(decode(&body(&ack)).is_err()); + let data = field(field(&mut ack, "message"), "data"); + data.as_map_mut() + .unwrap() + .retain(|(k, _)| k.as_text() != Some("attempt")); + *field(data, "operation") = Value::Text("cancel".into()); + assert!(decode(&body(&ack)).is_err()); + + for mut message in [ + value(Message::InitError { payload: payload() }), + value(Message::InvocationResult { + attempt: AttemptId::default(), + outcome: Outcome::Success, + payload: payload(), + }), + ] { + let data = field(field(&mut message, "message"), "data"); + let duplicate = data.as_map().unwrap().first().unwrap().clone(); + data.as_map_mut().unwrap().push(duplicate); + assert!(matches!( + decode(&body(&message)), + Err(ProtocolError::InvalidStructure) + )); + } +} + +#[test] +fn test_should_reject_extra_data_fields_for_every_message_variant_and_zero_deadlines() { + for message in messages() { + let mut packet = value(message); + let message = field(&mut packet, "message"); + if let Some((_, data)) = message + .as_map_mut() + .unwrap() + .iter_mut() + .find(|(key, _)| key.as_text() == Some("data")) + { + data.as_map_mut() + .unwrap() + .push((Value::Text("secret-extra".into()), Value::Bool(true))); + } else { + message + .as_map_mut() + .unwrap() + .push((Value::Text("data".into()), Value::Bool(true))); + } + let error = decode(&body(&packet)).unwrap_err(); + assert!(!format!("{error} {error:?}").contains("secret-extra")); + } + let mut packet = value(messages().into_iter().nth(4).unwrap()); + *field( + field(field(&mut packet, "message"), "data"), + "deadlineUnixMs", + ) = Value::Integer(0.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} + +#[test] +fn test_should_bound_payload_metadata_nesting_and_collection_counts_before_deserialization() { + let maximum = Payload::try_from(Bytes::from(vec![0xff; MAX_PAYLOAD_BYTES])).unwrap(); + let frame = Frame::new( + EnvironmentId::default(), + Message::InitError { payload: maximum }, + ); + let encoded = encode(&frame).unwrap(); + assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + assert!(Payload::try_from(Bytes::from(vec![0; MAX_PAYLOAD_BYTES + 1])).is_err()); + let mut too_large = value(Message::InitError { payload: payload() }); + *field(field(field(&mut too_large, "message"), "data"), "payload") = + Value::Bytes(vec![0; MAX_PAYLOAD_BYTES + 1]); + assert!(matches!( + decode(&body(&too_large)), + Err(ProtocolError::TooLarge) + )); + let metadata = Value::Map( + (0..65) + .map(|i| (Value::Text(format!("k{i}")), Value::Text("x".repeat(1024)))) + .collect(), + ); + assert!(matches!( + preflight::validate(&body(&metadata)), + Err(ProtocolError::TooLarge) + )); + let members = Value::Map( + (0..129) + .map(|i| (Value::Text(format!("k{i}")), Value::Bool(true))) + .collect(), + ); + assert!(matches!( + preflight::validate(&body(&members)), + Err(ProtocolError::TooLarge) + )); + let mut nested = Value::Bool(true); + for _ in 0..17 { + nested = Value::Map(vec![(Value::Text("x".into()), nested)]); + } + assert!(matches!( + preflight::validate(&body(&nested)), + Err(ProtocolError::TooLarge) + )); + assert!(matches!( + preflight::validate(&body(&Value::Map(vec![( + Value::Text("k".repeat(65)), + Value::Bool(true) + )]))), + Err(ProtocolError::TooLarge) + )); +} + +#[rstest] +#[case(&[0x9f, 0xff])] +#[case(&[0xbf, 0xff])] +#[case(&[0x5f, 0x40, 0xff])] +#[case(&[0xc0, 0x00])] +#[case(&[0x20])] +#[case(&[0x80])] +#[case(&[0xf9, 0, 20])] +#[case(&[0xfa, 0, 0, 0, 20])] +#[case(&[0xfb, 0, 0, 0, 0, 0, 0, 0, 20])] +#[case(&[0xf6])] +#[case(&[0xa1, 0x61, b'x', 0x7b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])] +fn test_should_reject_forbidden_cbor_shapes_and_huge_declared_lengths(#[case] bytes: &[u8]) { + assert!(preflight::validate(bytes).is_err()); +} + +#[test] +fn test_should_reject_trailing_data_and_payload_arrays_or_text() { + let frame = Frame::new(EnvironmentId::default(), Message::Ready); + let mut bytes = encode(&frame).unwrap().get(4..).unwrap().to_vec(); + bytes.push(0x00); + assert!(matches!( + decode(&bytes), + Err(ProtocolError::InvalidStructure) + )); + for invalid in [Value::Array(vec![]), Value::Text("base64-secret".into())] { + let mut message = value(Message::InitError { payload: payload() }); + *field(field(field(&mut message, "message"), "data"), "payload") = invalid; + assert!(decode(&body(&message)).is_err()); + } +} + +#[tokio::test] +async fn test_should_round_trip_fragmented_transport_and_distinguish_clean_and_truncated_eof() { + let frame = Frame::new( + EnvironmentId::default(), + Message::InitError { payload: payload() }, + ); + let encoded = encode(&frame).unwrap(); + let (mut writer, mut reader) = duplex(3); + let expected = frame.clone(); + let send = tokio::spawn(async move { + write_frame(&mut writer, &frame, Instant::now() + Duration::from_secs(2)) + .await + .unwrap(); + }); + assert_eq!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(2)) + .await + .unwrap() + .unwrap(), + expected + ); + send.await.unwrap(); + assert!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(2)) + .await + .unwrap() + .is_none() + ); + assert!( + read_frame(&mut empty(), Instant::now() + Duration::from_secs(1)) + .await + .unwrap() + .is_none() + ); + for length in [1, 2, 3, 4, encoded.len() - 1] { + let mut truncated = encoded.get(..length).unwrap(); + assert!( + matches!(read_frame(&mut truncated, Instant::now() + Duration::from_secs(1)).await, Err(ProtocolError::Io(error)) if error.kind() == io::ErrorKind::UnexpectedEof) + ); + } + let oversized = u32::MAX.to_be_bytes(); + assert!(matches!( + read_frame( + &mut oversized.as_slice(), + Instant::now() + Duration::from_secs(1) + ) + .await, + Err(ProtocolError::TooLarge) + )); +} + +#[tokio::test(start_paused = true)] +async fn test_should_enforce_absolute_deadlines_for_partial_reads_and_stalled_writes() { + let (mut writer, mut reader) = duplex(2); + writer.write_all(&[0, 0]).await.unwrap(); + assert!(matches!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(1)).await, + Err(ProtocolError::Deadline) + )); + let frame = Frame::new(EnvironmentId::default(), Message::Ready); + assert!(matches!( + write_frame(&mut writer, &frame, Instant::now() + Duration::from_secs(1)).await, + Err(ProtocolError::Deadline) + )); + let encoded = encode(&frame).unwrap(); + let mut unread = encoded.as_ref(); + assert!(matches!( + read_frame(&mut unread, Instant::now()).await, + Err(ProtocolError::Deadline) + )); + assert_eq!( + unread, + encoded.as_ref(), + "expired reads must not consume a ready frame" + ); +} + +#[rstest] +#[case(0)] +#[case(999)] +#[case(1001)] +#[case(900_001)] +#[case(u32::MAX)] +fn test_should_reject_out_of_profile_handler_timeouts_before_session_admission( + #[case] milliseconds: u32, +) { + assert!(HandlerTimeout::try_from(milliseconds).is_err()); + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(HandlerTimeout::try_from(1000).unwrap()); + } + let mut packet = value(invoke); + *field( + field(field(&mut packet, "message"), "data"), + "handlerTimeoutMs", + ) = Value::Integer(milliseconds.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} +#[test] +fn test_should_round_trip_handler_budget_timeout_event_and_reject_older_relays() { + for seconds in [1, 900] { + let timeout = HandlerTimeout::try_from(Duration::from_secs(seconds)).unwrap(); + assert_eq!(timeout.duration(), Duration::from_secs(seconds)); + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(timeout); + } + let packet = value(invoke.clone()); + assert_eq!(decode(&body(&packet)).unwrap().into_message(), invoke); + } + for timeout in [ + Duration::ZERO, + Duration::from_millis(1001), + Duration::from_secs(901), + Duration::MAX, + ] { + assert!(HandlerTimeout::try_from(timeout).is_err()); + } + let message = Message::InvocationTimedOut { + attempt: AttemptId::default(), + }; + let packet = value(message.clone()); + assert_eq!(decode(&body(&packet)).unwrap().into_message(), message); + for version in [0, 1, 3, u32::MAX] { + let mut packet = value(Message::Ready); + *field(&mut packet, "version") = Value::Integer(version.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::UnsupportedVersion) + )); + } + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(HandlerTimeout::try_from(1000).unwrap()); + } + let mut packet = value(invoke); + *field( + field(field(&mut packet, "message"), "data"), + "handlerTimeoutMs", + ) = Value::Null; + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidStructure) + )); +} diff --git a/crates/rustack-lambda-runtime/src/protocol/types.rs b/crates/rustack-lambda-runtime/src/protocol/types.rs new file mode 100644 index 00000000..af75f76a --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/types.rs @@ -0,0 +1,401 @@ +//! Validated metadata and closed control-message variants. + +use std::{fmt, num::NonZeroU64, str::FromStr, time::Duration}; + +use bytes::Bytes; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use uuid::Uuid; + +use super::{MAX_PAYLOAD_BYTES, ProtocolError}; + +macro_rules! identity { + ($name:ident, $doc:literal) => { + #[doc = $doc] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(try_from = "String", into = "String")] + pub struct $name(Uuid); + + impl Default for $name { + fn default() -> Self { + Self(Uuid::new_v4()) + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + impl FromStr for $name { + type Err = ProtocolError; + fn from_str(value: &str) -> Result { + if value.len() != 36 { + return Err(ProtocolError::InvalidMetadata); + } + let id = Uuid::parse_str(value).map_err(|_| ProtocolError::InvalidMetadata)?; + if id.is_nil() || id.to_string() != value { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(id)) + } + } + impl TryFrom for $name { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl From<$name> for String { + fn from(value: $name) -> Self { + value.to_string() + } + } + }; +} + +identity!( + EnvironmentId, + "Canonical nonnil environment UUID; parse with `FromStr`, generate with `Default`." +); + +/// Validated whole-second handler timeout, encoded as milliseconds on the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "u32", into = "u32")] +pub struct HandlerTimeout(u32); + +impl TryFrom for HandlerTimeout { + type Error = ProtocolError; + fn try_from(milliseconds: u32) -> Result { + if !(1000..=900_000).contains(&milliseconds) || !milliseconds.is_multiple_of(1000) { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(milliseconds)) + } +} +impl TryFrom for HandlerTimeout { + type Error = ProtocolError; + fn try_from(timeout: Duration) -> Result { + if timeout.subsec_nanos() != 0 { + return Err(ProtocolError::InvalidMetadata); + } + Self::try_from( + u32::try_from(timeout.as_millis()).map_err(|_| ProtocolError::InvalidMetadata)?, + ) + } +} +impl From for u32 { + fn from(timeout: HandlerTimeout) -> Self { + timeout.0 + } +} +impl HandlerTimeout { + /// Return the validated relative budget without starting a clock. + /// + /// ``` + /// use std::time::Duration; + /// let timeout = rustack_lambda_runtime::protocol::HandlerTimeout::try_from(Duration::from_secs(3))?; + /// assert_eq!(timeout.duration(), Duration::from_secs(3)); + /// # Ok::<(), rustack_lambda_runtime::protocol::ProtocolError>(()) + /// ``` + #[must_use] + pub fn duration(self) -> Duration { + Duration::from_millis(u64::from(self.0)) + } +} +identity!( + AttemptId, + "Canonical nonnil invocation attempt UUID; parse with `FromStr`, generate with `Default`." +); + +/// Architecture advertised by the environment-local relay. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Architecture { + /// Linux x86-64. + #[serde(rename = "x86_64")] + X86_64, + /// Linux `AArch64`. + #[serde(rename = "arm64")] + Arm64, +} + +/// Validated Lambda invocation ARN, including an optional version/alias. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct FunctionArn(String); + +impl FromStr for FunctionArn { + type Err = ProtocolError; + + fn from_str(value: &str) -> Result { + if value.len() > 2048 { + return Err(ProtocolError::InvalidMetadata); + } + let mut parts = value.split(':'); + let valid = matches!(parts.next(), Some("arn")) + && matches!(parts.next(), Some("aws" | "aws-cn" | "aws-us-gov")) + && matches!(parts.next(), Some("lambda")) + && parts + .next() + .is_some_and(|region| identifier(region, 64, false)) + && parts.next().is_some_and(|account| { + account.len() == 12 && account.bytes().all(|b| b.is_ascii_digit()) + }) + && matches!(parts.next(), Some("function")) + && parts.next().is_some_and(|name| identifier(name, 64, true)); + if !valid { + return Err(ProtocolError::InvalidMetadata); + } + if let Some(qualifier) = parts.next() { + if qualifier != "$LATEST" && !identifier(qualifier, 128, true) { + return Err(ProtocolError::InvalidMetadata); + } + if qualifier.bytes().all(|byte| byte.is_ascii_digit()) + && !qualifier.parse::().is_ok_and(|version| version != 0) + { + return Err(ProtocolError::InvalidMetadata); + } + } + if parts.next().is_some() { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(value.into())) + } +} + +fn identifier(value: &str, maximum: usize, underscore: bool) -> bool { + !value.is_empty() + && value.len() <= maximum + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || (underscore && b == b'_')) +} + +impl TryFrom for FunctionArn { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: FunctionArn) -> Self { + value.0 + } +} +impl AsRef for FunctionArn { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// Nonempty printable ASCII trace metadata, at most256 bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct TraceId(String); + +impl FromStr for TraceId { + type Err = ProtocolError; + fn from_str(value: &str) -> Result { + if value.is_empty() || value.len() > 256 || !value.bytes().all(|b| (32..=126).contains(&b)) + { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(value.into())) + } +} +impl TryFrom for TraceId { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: TraceId) -> Self { + value.0 + } +} +impl AsRef for TraceId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// A size-validated raw event/result whose Debug never displays contents. +#[derive(Clone, PartialEq, Eq)] +pub struct Payload(Bytes); + +impl TryFrom for Payload { + type Error = ProtocolError; + fn try_from(value: Bytes) -> Result { + if value.len() > MAX_PAYLOAD_BYTES { + return Err(ProtocolError::TooLarge); + } + Ok(Self(value)) + } +} +impl From for Bytes { + fn from(value: Payload) -> Self { + value.0 + } +} +impl AsRef<[u8]> for Payload { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} +impl fmt::Debug for Payload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Payload") + .field("bytes", &self.0.len()) + .finish_non_exhaustive() + } +} +impl Serialize for Payload { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.0) + } +} +impl<'de> Deserialize<'de> for Payload { + fn deserialize>(deserializer: D) -> Result { + struct PayloadVisitor; + impl de::Visitor<'_> for PayloadVisitor { + type Value = Payload; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded CBOR byte string") + } + fn visit_bytes(self, bytes: &[u8]) -> Result { + if bytes.len() > MAX_PAYLOAD_BYTES { + return Err(E::custom("payload size budget exceeded")); + } + Ok(Payload(Bytes::copy_from_slice(bytes))) + } + fn visit_byte_buf(self, bytes: Vec) -> Result { + Payload::try_from(Bytes::from(bytes)).map_err(E::custom) + } + } + deserializer.deserialize_byte_buf(PayloadVisitor) + } +} + +/// Acknowledgement scope; cancellation always names exactly one attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "AckFields", into = "AckFields")] +pub enum Acknowledgement { + /// The session invalidated this attempt; user-code exit is not proven. + Cancel(AttemptId), + /// The control session accepted closure. + Close, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +enum AckOperation { + Cancel, + Close, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct AckFields { + operation: AckOperation, + #[serde(skip_serializing_if = "Option::is_none")] + attempt: Option, +} +impl TryFrom for Acknowledgement { + type Error = ProtocolError; + fn try_from(value: AckFields) -> Result { + match (value.operation, value.attempt) { + (AckOperation::Cancel, Some(attempt)) => Ok(Self::Cancel(attempt)), + (AckOperation::Close, None) => Ok(Self::Close), + _ => Err(ProtocolError::InvalidMetadata), + } + } +} +impl From for AckFields { + fn from(value: Acknowledgement) -> Self { + match value { + Acknowledgement::Cancel(attempt) => Self { + operation: AckOperation::Cancel, + attempt: Some(attempt), + }, + Acknowledgement::Close => Self { + operation: AckOperation::Close, + attempt: None, + }, + } + } +} + +/// Whether the function posted a success response or a handler error. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Outcome { + /// Runtime /response. + Success, + /// Runtime /error, distinct from a transport or initialization failure. + FunctionError, +} + +/// Closed control schema; payload and invocation metadata are validated types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + content = "data", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +#[non_exhaustive] +pub enum Message { + /// First relay frame advertising the compiled platform. + Hello { + /// Linux image architecture. + architecture: Architecture, + }, + /// Relay listener is bound; does not prove bootstrap readiness. + Ready, + /// The function issued its first valid GET /next. + BootstrapReady, + /// One host-owned invocation; never replay automatically. + Invoke { + /// Shared caller/backend attempt UUID. + attempt: AttemptId, + /// Absolute wall deadline, checked and converted by the receiver. + deadline_unix_ms: NonZeroU64, + /// Optional captured handler budget, armed only at consumed delivery. + #[serde(default, skip_serializing_if = "Option::is_none")] + handler_timeout_ms: Option, + /// Requested ARN, preserving an invoked alias. + function_arn: FunctionArn, + /// Runtime trace header. + trace_id: TraceId, + /// Raw event bytes. + payload: Payload, + }, + /// Function completion for one delivered attempt. + InvocationResult { + /// Completed attempt UUID. + attempt: AttemptId, + /// /response versus /error. + outcome: Outcome, + /// Raw result or error bytes. + payload: Payload, + }, + /// The delivered handler clock expired before the caller's absolute deadline. + InvocationTimedOut { + /// Timed-out active attempt; this environment cannot be reused. + attempt: AttemptId, + }, + /// Fatal initialization failure before event delivery. + InitError { + /// Raw initialization error document. + payload: Payload, + }, + /// Invalidate an attempt; the host still terminates its function. + Cancel { + /// Invalidated attempt UUID. + attempt: AttemptId, + }, + /// Close the environment's sole control session. + Close, + /// Cancellation or closure acknowledgement, scoped by operation. + Ack(Acknowledgement), +} diff --git a/crates/rustack-lambda-runtime/src/relay.rs b/crates/rustack-lambda-runtime/src/relay.rs new file mode 100644 index 00000000..a43d2c3d --- /dev/null +++ b/crates/rustack-lambda-runtime/src/relay.rs @@ -0,0 +1,454 @@ +//! Supervised control transport for an environment-local Lambda Runtime API. +//! +//! This owner invalidates the API session; the host environment supervisor +//! remains responsible for stopping/removing the separate function container. + +use std::{future::pending, io, sync::Arc, time::Duration}; + +use thiserror::Error; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + runtime::Handle, + sync::{oneshot, watch}, + task::JoinHandle, + time::{Instant, sleep_until, timeout_at}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; + +use crate::{ + http::{ApiError, RuntimeApiHandle}, + metadata::deadline_from_unix_ms, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + ProtocolError, + }, + session::{Invocation, InvocationTicket, RuntimeResult, SessionError}, + transport::{InputEnd, Settings, Transport, after}, +}; + +const CONTROL_SILENCE: Duration = Duration::from_hours(1); + +/// Relay errors contain classes and redacted protocol/IO sources. +#[derive(Debug, Clone, Error)] +#[non_exhaustive] +pub enum RelayError { + /// The process entry point needs Linux and supported CPU architecture. + #[error("Lambda relay requires Linux x86_64 or arm64")] + UnsupportedPlatform, + /// Invalid operator-owned worker configuration. + #[error("invalid Lambda relay configuration")] + Configuration, + /// Unexpected EOF before a normal Close exchange. + #[error("Lambda relay control EOF")] + Eof, + /// Protocol validation/transport failure. + #[error("Lambda relay protocol failed")] + Protocol(#[source] Arc), + /// IO while reading the frame's first byte failed. + #[error("Lambda relay control IO failed")] + Io(#[source] Arc), + /// A frame belongs to another environment. + #[error("Lambda relay environment mismatch")] + Scope, + /// Message direction, identity or session phase is invalid. + #[error("unexpected Lambda relay control message")] + UnexpectedMessage, + /// An IO/retirement lease expired. + #[error("Lambda relay deadline exceeded")] + Deadline, + /// Output could not be confirmed flushed. + #[error("Lambda relay output failed")] + Output, + /// An owned worker terminated unexpectedly. + #[error("Lambda relay worker failed")] + Worker, + /// Shared session failed. + #[error(transparent)] + Session(#[from] SessionError), + /// The API could not be cleaned up. + #[error(transparent)] + Api(#[from] ApiError), +} + +impl From for RelayError { + fn from(error: ProtocolError) -> Self { + Self::Protocol(Arc::new(error)) + } +} +impl From for RelayError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} + +/// Validated environment identity and forwarded worker leases. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct RelayConfig { + environment: EnvironmentId, + architecture: Architecture, + idle_lease: Duration, + io_budget: Duration, + retirement_budget: Duration, +} + +impl RelayConfig { + /// Validate worker leases without reading/mutating process environment. + /// + /// # Errors + /// Rejects zero leases, idle over24h or IO/retirement over60s. + /// + /// ``` + /// # use rustack_lambda_runtime::{protocol::{EnvironmentId, Architecture}, relay::RelayConfig}; + /// # use std::time::Duration; + /// let config = RelayConfig::new(EnvironmentId::default(), Architecture::Arm64, + /// Duration::from_secs(600), Duration::from_secs(5), Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_runtime::relay::RelayError>(()) + /// ``` + pub fn new( + environment: EnvironmentId, + architecture: Architecture, + idle_lease: Duration, + io_budget: Duration, + retirement_budget: Duration, + ) -> Result { + if idle_lease.is_zero() + || idle_lease > Duration::from_hours(24) + || io_budget.is_zero() + || io_budget > Duration::from_secs(60) + || retirement_budget.is_zero() + || retirement_budget > Duration::from_secs(60) + { + return Err(RelayError::Configuration); + } + Ok(Self { + environment, + architecture, + idle_lease, + io_budget, + retirement_budget, + }) + } +} + +/// Owner of the relay supervisor; Drop signals stop without abandoning cleanup. +#[derive(Debug)] +#[non_exhaustive] +pub struct RelayHandle { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +impl RelayHandle { + /// Await natural Close/EOF/failure without requesting shutdown. + /// + /// # Errors + /// Returns cached relay/protocol/supervision failures. + /// + /// ```no_run + /// # async fn wait(relay: &mut rustack_lambda_runtime::relay::RelayHandle) { + /// let outcome = relay.wait().await; + /// # } + /// ``` + pub async fn wait(&mut self) -> Result<(), RelayError> { + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| RelayError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settlement = Some(result.clone()); + result + } + + /// Signal priority stop and join the owned API/control workers. + /// + /// # Errors + /// Returns protocol, API cleanup or supervision failure. + /// + /// ```no_run + /// # async fn stop(relay: &mut rustack_lambda_runtime::relay::RelayHandle) { + /// let clean = relay.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), RelayError> { + self.stop.send_replace(true); + self.wait().await + } +} + +impl Drop for RelayHandle { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +/// Transfer the API and cancel-safe owned streams into one relay supervisor. +/// +/// IO must be asynchronously cancellable; the Linux binary supplies nonblocking +/// Unix pipes. A blocking stdin adapter cannot satisfy the shutdown contract. +/// +/// # Errors +/// Returns unavailable supervision when no Tokio runtime is entered. +/// +/// ```no_run +/// # async fn example(api: rustack_lambda_runtime::http::RuntimeApiHandle, config: rustack_lambda_runtime::relay::RelayConfig) -> Result<(), rustack_lambda_runtime::relay::RelayError> { +/// let (reader, _) = tokio::io::duplex(1024); +/// let (writer, _) = tokio::io::duplex(1024); +/// let mut relay = rustack_lambda_runtime::relay::start(api, config, reader, writer)?; +/// relay.shutdown().await?; +/// # Ok(()) } +/// ``` +pub fn start( + mut api: RuntimeApiHandle, + config: RelayConfig, + reader: R, + writer: W, +) -> Result +where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, +{ + let runtime = Handle::try_current().map_err(|_| RelayError::Worker)?; + let (stop, stopped) = watch::channel(false); + let transport = Transport::start( + reader, + writer, + Settings { + silence: config.idle_lease.max(CONTROL_SILENCE), + io_budget: config.io_budget, + input_end: InputEnd::Close, + }, + )?; + let initialization_error = api.take_initialization_error(); + let actor = RelayActor::builder() + .api(api) + .config(config) + .transport(transport) + .stop(stopped) + .initialization_error(initialization_error) + .build(); + let task = runtime.spawn(actor.run().with_current_subscriber()); + Ok(RelayHandle { + stop, + task: Some(task), + settlement: None, + }) +} + +#[derive(Debug, Default)] +enum Phase { + #[default] + Initializing, + Idle, + Busy(InvocationTicket), + Retiring(Instant), +} + +impl Phase { + async fn result(&mut self) -> Result { + match self { + Self::Busy(ticket) => ticket.result().await, + _ => pending().await, + } + } +} + +#[derive(Debug, TypedBuilder)] +struct RelayActor { + api: RuntimeApiHandle, + config: RelayConfig, + transport: Transport, + stop: watch::Receiver, + initialization_error: Option>, + #[builder(default)] + phase: Phase, + #[builder(default)] + bootstrap_announced: bool, + #[builder(default)] + last_attempt: Option, +} + +impl RelayActor { + async fn run(mut self) -> Result<(), RelayError> { + let outcome = self.control().await; + self.api.session().close(); + let api = self.api.shutdown().await.map_err(RelayError::from); + let transport = self.transport.shutdown().await; + outcome.and(api).and(transport) + } + + async fn emit(&self, message: Message) -> Result<(), RelayError> { + let deadline = match self.phase { + Phase::Retiring(until) => after(self.config.io_budget)?.min(until), + _ => after(self.config.io_budget)?, + }; + self.transport + .send(Frame::new(self.config.environment, message), deadline) + .await + } + + async fn control(&mut self) -> Result<(), RelayError> { + if *self.stop.borrow() { + return Ok(()); + } + self.emit(Message::Hello { + architecture: self.config.architecture, + }) + .await?; + self.emit(Message::Ready).await?; + let session = self.api.session().clone(); + loop { + let retirement = match self.phase { + Phase::Retiring(deadline) => Some(deadline), + _ => None, + }; + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => return Ok(()), + () = retirement_deadline(retirement) => return Err(RelayError::Deadline), + frame = self.transport.recv() => { + let frame = frame?; + if frame.environment() != self.config.environment { return Err(RelayError::Scope); } + if self.command(frame.into_message()).await? { return Ok(()); } + } + error = initialization_error(&mut self.initialization_error), if self.initialization_error.is_some() && retirement.is_none() => { + self.initialization_error = None; + if let Some(payload) = error { + self.emit(Message::InitError { payload }).await?; + self.retire().await?; + } + } + ready = session.bootstrap_ready(after(CONTROL_SILENCE)?), if !self.bootstrap_announced && !matches!(self.phase, Phase::Retiring(_)) => { + ready?; + self.bootstrap_announced = true; + self.phase = Phase::Idle; + self.emit(Message::BootstrapReady).await?; + } + result = self.phase.result() => { + match result { + Ok(result) => self.completed(result).await?, + Err(SessionError::HandlerTimeout) => { + let attempt=self.last_attempt.ok_or(RelayError::UnexpectedMessage)?; + self.emit(Message::InvocationTimedOut { attempt }).await?; + self.retire().await?; + } + Err(error) => return Err(error.into()), + } + } + () = session.closed(), if !matches!(self.phase, Phase::Retiring(_)) => return Err(RelayError::Session(SessionError::Unavailable)), + } + } + } + + async fn command(&mut self, message: Message) -> Result { + match message { + Message::Invoke { + attempt, + deadline_unix_ms, + handler_timeout_ms, + function_arn, + trace_id, + payload, + } if matches!(self.phase, Phase::Idle) && self.last_attempt != Some(attempt) => { + let deadline = + deadline_from_unix_ms(deadline_unix_ms).map_err(|_| RelayError::Deadline)?; + let mut invocation = + Invocation::new(attempt, deadline, function_arn, trace_id, payload)?; + if let Some(timeout) = handler_timeout_ms { + invocation = invocation.with_handler_timeout(timeout); + } + let ticket = self + .api + .session() + .submit(invocation, after(self.config.io_budget)?.min(deadline)) + .await?; + self.last_attempt = Some(attempt); + self.phase = Phase::Busy(ticket); + } + Message::Cancel { attempt } if self.last_attempt == Some(attempt) => { + self.retire().await?; + self.emit(Message::Ack(Acknowledgement::Cancel(attempt))) + .await?; + } + Message::Close => { + self.api.session().close(); + self.api.shutdown().await?; + self.emit(Message::Ack(Acknowledgement::Close)).await?; + return Ok(true); + } + _ => return Err(RelayError::UnexpectedMessage), + } + Ok(false) + } + + async fn completed(&mut self, result: RuntimeResult) -> Result<(), RelayError> { + let attempt = self.last_attempt.ok_or(RelayError::UnexpectedMessage)?; + self.phase = Phase::Idle; + match result { + RuntimeResult::Success(payload) => { + self.emit(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload, + }) + .await?; + } + RuntimeResult::FunctionError(payload) => { + self.emit(Message::InvocationResult { + attempt, + outcome: Outcome::FunctionError, + payload, + }) + .await?; + self.retire().await?; + } + RuntimeResult::InitError(payload) => { + self.initialization_error = None; + self.emit(Message::InitError { payload }).await?; + self.retire().await?; + } + } + Ok(()) + } + + async fn retire(&mut self) -> Result<(), RelayError> { + if !matches!(self.phase, Phase::Retiring(_)) { + if let Phase::Busy(ticket) = &self.phase { + ticket.cancel(); + } + self.api.session().close(); + timeout_at(after(self.config.io_budget)?, self.api.session().closed()) + .await + .map_err(|_| RelayError::Deadline)?; + self.phase = Phase::Retiring(after(self.config.retirement_budget)?); + } + Ok(()) + } +} + +async fn initialization_error( + receiver: &mut Option>, +) -> Option { + match receiver { + Some(receiver) => receiver.await.ok(), + None => pending().await, + } +} + +async fn retirement_deadline(deadline: Option) { + match deadline { + Some(deadline) => sleep_until(deadline).await, + None => pending().await, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/relay/tests.rs b/crates/rustack-lambda-runtime/src/relay/tests.rs new file mode 100644 index 00000000..9ebaa570 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/relay/tests.rs @@ -0,0 +1,461 @@ +use std::{ + future::Future, + num::NonZeroU64, + task::{Context, Waker}, +}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncWriteExt, DuplexStream, duplex}, + net::TcpStream, +}; + +use super::*; +use crate::{ + http::tests::{Reply, request}, + metadata::{deadline_unix_ms, local_trace_id}, + protocol::{FunctionArn, Payload, read_frame, write_frame}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +struct Harness { + owner: RelayHandle, + environment: EnvironmentId, + input: DuplexStream, + output: DuplexStream, + api: std::net::SocketAddr, +} + +impl Harness { + async fn new() -> Self { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let environment = EnvironmentId::default(); + let config = RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(2), + ) + .unwrap(); + let (input, reader) = duplex(128); + let (writer, output) = duplex(128); + let owner = start(api, config, reader, writer).unwrap(); + let mut harness = Self { + owner, + environment, + input, + output, + api: address, + }; + assert!(matches!( + harness.next().await, + Message::Hello { + architecture: Architecture::Arm64 + } + )); + assert_eq!(harness.next().await, Message::Ready); + harness + } + + async fn next(&mut self) -> Message { + let frame = read_frame(&mut self.output, deadline()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.environment(), self.environment); + frame.into_message() + } + + async fn send(&mut self, message: Message) { + write_frame( + &mut self.input, + &Frame::new(self.environment, message), + deadline(), + ) + .await + .unwrap(); + } + + async fn close(mut self) { + self.send(Message::Close).await; + assert_eq!(self.next().await, Message::Ack(Acknowledgement::Close)); + timeout_at(deadline(), self.owner.wait()) + .await + .unwrap() + .unwrap(); + assert!( + read_frame(&mut self.output, deadline()) + .await + .unwrap() + .is_none() + ); + assert!(TcpStream::connect(self.api).await.is_err()); + } + + async fn bootstrap(&mut self) -> JoinHandle { + let api = self.api; + let poll = tokio::spawn(async move { + request(api, "GET", "/2018-06-01/runtime/invocation/next", "", b"").await + }); + assert_eq!(self.next().await, Message::BootstrapReady); + poll + } +} + +fn invoke() -> (AttemptId, Message) { + let attempt = AttemptId::default(); + ( + attempt, + Message::Invoke { + attempt, + deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(10)).unwrap(), + handler_timeout_ms: None, + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse::() + .unwrap(), + trace_id: local_trace_id().unwrap(), + payload: Payload::try_from(Bytes::from_static(b"secret-event")).unwrap(), + }, + ) +} + +#[tokio::test] +async fn test_should_handshake_forward_http_results_and_reuse_one_session() { + let mut harness = Harness::new().await; + let first = harness.bootstrap().await; + for (index, poll) in [(0, Some(first)), (1, None)] { + let poll = poll.unwrap_or_else(|| { + let api = harness.api; + tokio::spawn(async move { + request(api, "GET", "/2018-06-01/runtime/invocation/next", "", b"").await + }) + }); + let (attempt, message) = invoke(); + harness.send(message).await; + let event = poll.await.unwrap(); + assert_eq!(event.status, 200); + assert_eq!(event.body, b"secret-event"); + let computed = format!("{{\"computed\":{},\"iteration\":{index}}}", 6 * 7); + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/response"), + "", + computed.as_bytes() + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InvocationResult { attempt: id, outcome: Outcome::Success, payload } if id == attempt && payload.as_ref() == computed.as_bytes()) + ); + } + harness.close().await; +} + +#[tokio::test] +async fn test_should_ack_cancel_while_busy_and_after_completion_without_replaying_a_result() { + for completed in [false, true] { + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, message) = invoke(); + harness.send(message).await; + assert_eq!(poll.await.unwrap().status, 200); + if completed { + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/response"), + "", + b"done" + ) + .await + .status, + 202 + ); + assert!(matches!( + harness.next().await, + Message::InvocationResult { + outcome: Outcome::Success, + .. + } + )); + } + harness.send(Message::Cancel { attempt }).await; + assert_eq!( + harness.next().await, + Message::Ack(Acknowledgement::Cancel(attempt)) + ); + harness.send(Message::Cancel { attempt }).await; + assert_eq!( + harness.next().await, + Message::Ack(Acknowledgement::Cancel(attempt)) + ); + harness.close().await; + } +} + +#[tokio::test] +async fn test_should_distinguish_function_and_initialization_errors_and_allow_close_ack() { + let mut harness = Harness::new().await; + assert_eq!( + request( + harness.api, + "POST", + "/2018-06-01/runtime/init/error", + "", + b"secret-init-error" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InitError { payload } if payload.as_ref() == b"secret-init-error") + ); + harness.close().await; + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, message) = invoke(); + harness.send(message).await; + poll.await.unwrap(); + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/error"), + "", + b"secret-function-error" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InvocationResult { outcome: Outcome::FunctionError, payload, .. } if payload.as_ref() == b"secret-function-error") + ); + harness.close().await; +} + +#[tokio::test] +async fn test_should_reject_foreign_scope_wrong_direction_and_invoke_before_bootstrap() { + for variant in 0..4 { + let mut harness = Harness::new().await; + match variant { + 0 => write_frame( + &mut harness.input, + &Frame::new(EnvironmentId::default(), Message::Close), + deadline(), + ) + .await + .unwrap(), + 1 => harness.send(Message::Ready).await, + 2 => { + harness + .send(Message::InvocationTimedOut { + attempt: AttemptId::default(), + }) + .await; + } + _ => harness.send(invoke().1).await, + } + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + assert!(harness.owner.wait().await.is_err()); + assert!(TcpStream::connect(harness.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_retain_shutdown_after_dropping_its_waiter_and_interrupt_idle_input() { + let mut harness = Harness::new().await; + let mut waiting = Box::pin(harness.owner.shutdown()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(harness.owner.task.is_some()); + timeout_at(deadline(), harness.owner.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(TcpStream::connect(harness.api).await.is_err()); +} + +#[tokio::test] +async fn test_should_cleanup_api_after_eof_and_truncated_control_frames() { + for truncated in [false, true] { + let mut harness = Harness::new().await; + if truncated { + harness.input.write_all(&[0, 0]).await.unwrap(); + } + harness.input.shutdown().await.unwrap(); + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + assert!(TcpStream::connect(harness.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_reject_bad_clock_and_overlapping_invocations_without_success() { + for overlapping in [false, true] { + let mut harness = Harness::new().await; + let mut poll = Some(harness.bootstrap().await); + let (_, mut message) = invoke(); + if overlapping { + harness.send(message).await; + poll.take().unwrap().await.unwrap(); + message = invoke().1; + } else if let Message::Invoke { + deadline_unix_ms, .. + } = &mut message + { + *deadline_unix_ms = NonZeroU64::new(1).unwrap(); + } + harness.send(message).await; + if let Some(poll) = poll { + let _ = poll.await; + } + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + } +} + +#[tokio::test] +async fn test_should_close_a_probe_before_any_function_has_polled() { + Harness::new().await.close().await; +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_partial_frame_and_blocked_output_io_and_join_the_api() { + let mut harness = Harness::new().await; + harness.input.write_all(&[0]).await.unwrap(); + assert!( + matches!(harness.owner.wait().await, Err(RelayError::Protocol(error)) if matches!(&*error, ProtocolError::Deadline)) + ); + assert!(TcpStream::connect(harness.api).await.is_err()); + + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let config = RelayConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(1), + ) + .unwrap(); + let (_host_input, reader) = duplex(1); + let (writer, _host_output) = duplex(1); + let mut owner = start(api, config, reader, writer).unwrap(); + assert!(matches!(owner.wait().await, Err(RelayError::Output))); + assert!(TcpStream::connect(address).await.is_err()); +} + +#[tokio::test] +async fn test_should_preserve_a_completed_io_failure_when_priority_stop_wins() { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let config = RelayConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(1), + ) + .unwrap(); + let (input, reader) = duplex(16); + let (writer, _output) = duplex(16); + drop(input); + let mut transport = Transport::start( + reader, + writer, + Settings { + silence: CONTROL_SILENCE, + io_budget: config.io_budget, + input_end: InputEnd::Close, + }, + ) + .unwrap(); + assert!(matches!(transport.recv().await, Err(RelayError::Eof))); + let (_stop, stopped) = watch::channel(true); + let actor = RelayActor::builder() + .api(api) + .config(config) + .transport(transport) + .stop(stopped) + .initialization_error(None) + .build(); + assert!(matches!(actor.run().await, Err(RelayError::Eof))); + assert!(TcpStream::connect(address).await.is_err()); +} + +#[test] +fn test_should_bound_relay_config() { + let environment = EnvironmentId::default(); + assert!( + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::ZERO, + Duration::from_secs(1), + Duration::from_secs(1) + ) + .is_err() + ); + assert!( + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(61), + Duration::from_secs(1) + ) + .is_err() + ); +} + +#[tokio::test] +async fn test_should_forward_delivered_handler_timeout_and_preserve_close_ack_retirement() { + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, mut message) = invoke(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut message + { + *handler_timeout_ms = + Some(crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + } + harness.send(message).await; + let response = poll.await.unwrap(); + assert_eq!(response.status, 200); + assert_eq!( + harness.next().await, + Message::InvocationTimedOut { attempt } + ); + harness.close().await; +} diff --git a/crates/rustack-lambda-runtime/src/session.rs b/crates/rustack-lambda-runtime/src/session.rs new file mode 100644 index 00000000..7d490ba8 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/session.rs @@ -0,0 +1,1102 @@ +//! A single actor owns each environment's invocation and long-poll state. +//! +//! This module owns only in-process state/channels. Transport adapters own HTTP +//! connections and the environment supervisor owns its process/container/VM. +//! Session cancellation invalidates an attempt; it never proves user-code exit. + +use std::{fmt, future::pending, mem::replace, time::Duration}; + +use thiserror::Error; +use tokio::{ + runtime::Handle, + sync::{ + OwnedSemaphorePermit, mpsc, + oneshot::{self, error::TryRecvError}, + watch, + }, + task::JoinHandle, + time::{Instant, sleep_until, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use crate::protocol::{AttemptId, FunctionArn, HandlerTimeout, Outcome, Payload, TraceId}; + +const MAILBOX_CAPACITY: usize = 8; +const MAX_IDLE_LEASE: Duration = Duration::from_hours(24); + +/// Bounded session errors never include function inputs or peer-supplied values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum SessionError { + /// Actor stopped or an owner disappeared. + #[error("runtime session unavailable")] + Unavailable, + /// One invocation/poll is already active, or the mailbox is full. + #[error("runtime session busy")] + Busy, + /// Completion does not identify the delivered active attempt. + #[error("unknown runtime invocation attempt")] + UnknownAttempt, + /// Command, idle lease or attempt deadline expired. + #[error("runtime session deadline exceeded")] + Deadline, + /// The delivered handler's separate function timeout expired first. + #[error("function handler timeout exceeded")] + HandlerTimeout, + /// Caller or environment owner cancelled the session. + #[error("runtime session cancelled")] + Cancelled, + /// A lease/deadline cannot be represented or is outside its range. + #[error("invalid runtime session metadata")] + InvalidMetadata, + /// The owned actor panicked or failed to join. + #[error("runtime session supervisor failed")] + SupervisorFailed, +} + +/// Captured, validated event and runtime metadata for a single attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Invocation { + attempt: AttemptId, + deadline: Instant, + handler_timeout: Option, + function_arn: FunctionArn, + trace_id: TraceId, + payload: Payload, +} + +impl Invocation { + /// Capture one event and its absolute monotonic deadline. + /// + /// # Errors + /// Returns Deadline if the attempt has already expired. + /// + /// ``` + /// # use rustack_lambda_runtime::{protocol::{AttemptId, Payload}, session::Invocation}; + /// # fn example() -> Result<(), Box> { + /// let call = Invocation::new( + /// AttemptId::default(), tokio::time::Instant::now() + std::time::Duration::from_secs(5), + /// "arn:aws:lambda:us-east-1:000000000000:function:echo".parse()?, + /// "Root=1-00000000-000000000000000000000000;Sampled=0".parse()?, + /// Payload::try_from(bytes::Bytes::from_static(b"{}"))?, + /// )?; + /// # Ok(()) } + /// ``` + pub fn new( + attempt: AttemptId, + deadline: Instant, + function_arn: FunctionArn, + trace_id: TraceId, + payload: Payload, + ) -> Result { + check_deadline(deadline)?; + Ok(Self { + attempt, + deadline, + handler_timeout: None, + function_arn, + trace_id, + payload, + }) + } + + /// Attach a validated handler budget; it starts only at consumed delivery. + /// + /// ```no_run + /// # fn timed(call: rustack_lambda_runtime::session::Invocation, timeout: rustack_lambda_runtime::protocol::HandlerTimeout) { + /// let timed = call.with_handler_timeout(timeout); + /// # } + /// ``` + #[must_use] + pub fn with_handler_timeout(mut self, timeout: HandlerTimeout) -> Self { + self.handler_timeout = Some(timeout); + self + } + + /// Read the relative handler budget without arming it. + /// + /// ```no_run + /// # fn inspect(call: &rustack_lambda_runtime::session::Invocation) { let timeout = call.handler_timeout(); } + /// ``` + #[must_use] + pub fn handler_timeout(&self) -> Option { + self.handler_timeout + } + + fn timing(&self, started: Instant) -> Result<(Instant, SessionError), SessionError> { + if let Some(timeout) = self.handler_timeout { + let handler = started + .checked_add(timeout.duration()) + .ok_or(SessionError::InvalidMetadata)?; + if handler < self.deadline { + return Ok((handler, SessionError::HandlerTimeout)); + } + } + Ok((self.deadline, SessionError::Deadline)) + } + + /// Borrow all event metadata and payload without copying bytes. + /// + /// ``` + /// # fn inspect(call: &rustack_lambda_runtime::session::Invocation) { + /// let (attempt, deadline, arn, trace, payload) = call.parts(); + /// # } + /// ``` + #[must_use] + pub fn parts(&self) -> (AttemptId, Instant, &FunctionArn, &TraceId, &Payload) { + ( + self.attempt, + self.deadline, + &self.function_arn, + &self.trace_id, + &self.payload, + ) + } +} + +/// One accepted runtime result; raw payload diagnostics are always redacted. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum RuntimeResult { + /// Function /response. + Success(Payload), + /// Function /error. + FunctionError(Payload), + /// Fatal initialization error before any event delivery. + InitError(Payload), +} + +/// Attached ownership of a session attempt; Drop invalidates only this attempt. +#[non_exhaustive] +pub struct InvocationTicket { + attempt: AttemptId, + deadline: Instant, + cancellation: watch::Sender, + response: oneshot::Receiver>, + attached: bool, +} + +impl fmt::Debug for InvocationTicket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvocationTicket") + .field("attempt", &self.attempt) + .field("deadline", &self.deadline) + .finish_non_exhaustive() + } +} + +impl InvocationTicket { + /// Identity used by the delivered Runtime API event. + /// + /// ``` + /// # fn inspect(ticket: &rustack_lambda_runtime::session::InvocationTicket) { + /// let attempt = ticket.attempt(); + /// # } + /// ``` + #[must_use] + pub fn attempt(&self) -> AttemptId { + self.attempt + } + + /// Request invalidation; the environment supervisor still owns termination. + /// + /// ``` + /// # fn stop(ticket: &rustack_lambda_runtime::session::InvocationTicket) { + /// ticket.cancel(); + /// # } + /// ``` + pub fn cancel(&self) { + self.cancellation.send_replace(true); + } + + /// Await a single result within the attempt's absolute deadline. + /// + /// # Errors + /// Returns cancellation/expiry, session failure or disconnected supervision. + /// + /// ```no_run + /// # async fn finish(ticket: rustack_lambda_runtime::session::InvocationTicket) { + /// let result = ticket.wait().await; + /// # } + /// ``` + pub async fn wait(mut self) -> Result { + self.result().await + } + + /// Await while retaining ticket ownership across a cancelled select branch. + /// + /// # Errors + /// Returns cancellation, expiry or session failure. Repeated calls after a + /// terminal result return Unavailable, without polling the consumed channel. + /// + /// ```no_run + /// # async fn inspect(ticket: &mut rustack_lambda_runtime::session::InvocationTicket) { + /// let result = ticket.result().await; + /// # } + /// ``` + pub async fn result(&mut self) -> Result { + if !self.attached { + return Err(SessionError::Unavailable); + } + if let Err(error) = check_deadline(self.deadline) { + self.cancel(); + self.attached = false; + return Err(error); + } + let cancelled = *self.cancellation.borrow(); + if cancelled { + self.attached = false; + return Err(SessionError::Cancelled); + } + let result = timeout_at(self.deadline, &mut self.response).await; + // The response channel has either settled or the attempt has expired. + // Keep a completed receiver from ever being polled a second time. + self.attached = false; + let result = match result { + Ok(Ok(result)) => result, + Ok(Err(_)) => { + self.cancel(); + return Err(SessionError::Unavailable); + } + Err(_) => { + self.cancel(); + return Err(SessionError::Deadline); + } + }; + if let Err(error) = check_deadline(self.deadline) { + self.cancel(); + return Err(error); + } + if *self.cancellation.borrow() { + return Err(SessionError::Cancelled); + } + result + } +} + +impl Drop for InvocationTicket { + fn drop(&mut self) { + if self.attached { + self.cancel(); + } + } +} + +/// Cloneable bounded command port; no mutable invocation state is shared. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct SessionHandle { + inbox: mpsc::Sender, + stop: watch::Sender, + bootstrap: watch::Receiver, +} + +/// Owner of the actor and take-once initialization-error notification. +#[derive(Debug)] +#[non_exhaustive] +pub struct RuntimeSession { + handle: SessionHandle, + task: Option>, + initialization_error: Option>, + settlement: Option>, +} + +impl RuntimeSession { + /// Start a single-invocation actor with a bounded idle lease. + /// + /// # Errors + /// Rejects a zero/over24h lease, unrepresentable deadline or missing Tokio runtime. + /// + /// ```no_run + /// # async fn example() -> Result<(), rustack_lambda_runtime::session::SessionError> { + /// let mut session = rustack_lambda_runtime::session::RuntimeSession::start(std::time::Duration::from_secs(60))?; + /// session.shutdown().await?; + /// # Ok(()) } + /// ``` + pub fn start(idle_lease: Duration) -> Result { + if idle_lease.is_zero() || idle_lease > MAX_IDLE_LEASE { + return Err(SessionError::InvalidMetadata); + } + let runtime = Handle::try_current().map_err(|_| SessionError::Unavailable)?; + let idle_deadline = Instant::now() + .checked_add(idle_lease) + .ok_or(SessionError::InvalidMetadata)?; + let (inbox, commands) = mpsc::channel(MAILBOX_CAPACITY); + let (stop, stopped) = watch::channel(false); + let (bootstrap, initialized) = watch::channel(false); + let (initialization_error, error) = oneshot::channel(); + let actor = SessionActor::builder() + .inbox(commands) + .stop(stopped) + .bootstrap(bootstrap) + .initialization_error(Some(initialization_error)) + .idle_lease(idle_lease) + .idle_deadline(idle_deadline) + .build(); + Ok(Self { + handle: SessionHandle { + inbox, + stop, + bootstrap: initialized, + }, + task: Some(runtime.spawn(actor.run())), + initialization_error: Some(error), + settlement: None, + }) + } + + /// Borrow the actor's command port. + /// + /// ``` + /// # fn inspect(session: &rustack_lambda_runtime::session::RuntimeSession) { + /// let handle = session.handle().clone(); + /// # } + /// ``` + #[must_use] + pub fn handle(&self) -> &SessionHandle { + &self.handle + } + + /// Transfer the initialization notification exactly once. + /// + /// ``` + /// # fn inspect(session: &mut rustack_lambda_runtime::session::RuntimeSession) { + /// let notification = session.take_initialization_error(); + /// assert!(session.take_initialization_error().is_none()); + /// # } + /// ``` + pub fn take_initialization_error(&mut self) -> Option> { + self.initialization_error.take() + } + + /// Signal stop and join the owned actor; cancelling a waiter retains its handle. + /// + /// # Errors + /// Returns `SupervisorFailed` if the actor panicked or failed to join. + /// + /// ```no_run + /// # async fn stop(session: &mut rustack_lambda_runtime::session::RuntimeSession) { + /// let clean = session.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), SessionError> { + self.handle.close(); + if let Some(result) = self.settlement { + return result; + } + let result = if let Some(task) = self.task.as_mut() { + task.await.map_err(|_| SessionError::SupervisorFailed) + } else { + Ok(()) + }; + self.task.take(); + self.settlement = Some(result); + result + } +} + +impl Drop for RuntimeSession { + fn drop(&mut self) { + self.handle.close(); + } +} + +impl SessionHandle { + fn send(&self, command: Command, deadline: Instant) -> Result<(), SessionError> { + self.send_with_body_permit(command, deadline, None) + } + + fn send_with_body_permit( + &self, + command: Command, + deadline: Instant, + body_permit: Option, + ) -> Result<(), SessionError> { + check_deadline(deadline)?; + if *self.stop.borrow() { + return Err(SessionError::Unavailable); + } + self.inbox + .try_send(Envelope { + deadline, + command, + _body_permit: body_permit, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => SessionError::Busy, + mpsc::error::TrySendError::Closed(_) => SessionError::Unavailable, + }) + } + + /// Admit one event; a dropped admission future cancels its own attempt. + /// + /// # Errors + /// Returns Busy, cancellation/expiry or unavailable supervision. + /// + /// ```no_run + /// # async fn submit(handle: &rustack_lambda_runtime::session::SessionHandle, call: rustack_lambda_runtime::session::Invocation) { + /// let deadline = call.parts().1; + /// let ticket = handle.submit(call, deadline).await; + /// # } + /// ``` + pub async fn submit( + &self, + invocation: Invocation, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + check_deadline(invocation.deadline)?; + let (cancellation, cancelled) = watch::channel(false); + let (response, result) = oneshot::channel(); + let (admitted, accepted) = oneshot::channel(); + let ticket = InvocationTicket { + attempt: invocation.attempt, + deadline: invocation.deadline, + cancellation, + response: result, + attached: true, + }; + self.send( + Command::Submit { + invocation, + cancellation: cancelled, + response, + admitted, + }, + deadline, + )?; + receive(accepted, deadline).await??; + Ok(ticket) + } + + /// Wait for the next event, allowing exactly one outstanding poll. + /// + /// # Errors + /// Returns Busy for a concurrent poll/active delivery, expiry or closed session. + /// + /// ```no_run + /// # async fn poll(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(60); + /// let event = handle.next(deadline).await; + /// # } + /// ``` + pub async fn next(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send(Command::Next { reply }, deadline)?; + let mut delivery = receive(response, deadline).await??; + let started = Instant::now(); + let (effective, _) = delivery.invocation.timing(started)?; + check_deadline(effective)?; + delivery.invocation.deadline = effective; + delivery + .receipt + .send(started) + .map_err(|_| SessionError::Unavailable)?; + Ok(delivery.invocation) + } + + /// Validate and settle exactly one delivered attempt. + /// + /// # Errors + /// Rejects stale/mismatched identities, deadline expiry, overload or closure. + /// + /// ```no_run + /// # async fn complete(handle: &rustack_lambda_runtime::session::SessionHandle, attempt: rustack_lambda_runtime::protocol::AttemptId, payload: rustack_lambda_runtime::protocol::Payload) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let result = handle.complete(attempt, None, rustack_lambda_runtime::protocol::Outcome::Success, payload, deadline).await; + /// # } + /// ``` + pub async fn complete( + &self, + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + deadline: Instant, + ) -> Result<(), SessionError> { + self.complete_with_body_permit(attempt, invocation_id, outcome, payload, deadline, None) + .await + } + + /// Retain HTTP body admission with the accepted envelope until actor settlement. + pub(crate) async fn complete_with_body_permit( + &self, + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + deadline: Instant, + body_permit: Option, + ) -> Result<(), SessionError> { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send_with_body_permit( + Command::Complete { + attempt, + invocation_id, + outcome, + payload, + reply, + }, + deadline, + body_permit, + )?; + receive(response, deadline).await? + } + + /// Report a fatal initialization failure before any event delivery. + /// + /// # Errors + /// Returns Busy after delivery, or command deadline/overload/session failure. + /// + /// ```no_run + /// # async fn fail(handle: &rustack_lambda_runtime::session::SessionHandle, payload: rustack_lambda_runtime::protocol::Payload) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let result = handle.initialization_error(payload, deadline).await; + /// # } + /// ``` + pub async fn initialization_error( + &self, + payload: Payload, + deadline: Instant, + ) -> Result<(), SessionError> { + self.initialization_error_with_body_permit(payload, deadline, None) + .await + } + + /// Retain an initialization error body even after its HTTP observer disappears. + pub(crate) async fn initialization_error_with_body_permit( + &self, + payload: Payload, + deadline: Instant, + body_permit: Option, + ) -> Result<(), SessionError> { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send_with_body_permit(Command::InitError { payload, reply }, deadline, body_permit)?; + receive(response, deadline).await? + } + + /// Await the first valid GET /next; listener readiness cannot satisfy this. + /// + /// # Errors + /// Returns deadline expiry or closure before the first poll. + /// + /// ```no_run + /// # async fn ready(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let ready = handle.bootstrap_ready(deadline).await; + /// # } + /// ``` + pub async fn bootstrap_ready(&self, deadline: Instant) -> Result<(), SessionError> { + check_deadline(deadline)?; + let mut bootstrap = self.bootstrap.clone(); + timeout_at(deadline, bootstrap.wait_for(|ready| *ready)) + .await + .map_err(|_| SessionError::Deadline)? + .map_err(|_| SessionError::Unavailable)?; + check_deadline(deadline) + } + + /// Signal priority shutdown without waiting for the ordinary command mailbox. + /// + /// ``` + /// # fn stop(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// handle.close(); + /// # } + /// ``` + pub fn close(&self) { + self.stop.send_replace(true); + } + + /// Wait until the actor closes its command receiver. + /// + /// ```no_run + /// # async fn closed(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// handle.closed().await; + /// # } + /// ``` + pub async fn closed(&self) { + self.inbox.closed().await; + } + + /// Whether admission is closed or its owner has requested stop. + /// + /// ``` + /// # fn inspect(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let closed = handle.is_closed(); + /// # } + /// ``` + #[must_use] + pub fn is_closed(&self) -> bool { + *self.stop.borrow() || self.inbox.is_closed() + } +} + +fn check_deadline(deadline: Instant) -> Result<(), SessionError> { + if Instant::now() >= deadline { + Err(SessionError::Deadline) + } else { + Ok(()) + } +} + +async fn receive(response: oneshot::Receiver, deadline: Instant) -> Result { + let result = timeout_at(deadline, response) + .await + .map_err(|_| SessionError::Deadline)? + .map_err(|_| SessionError::Unavailable)?; + check_deadline(deadline)?; + Ok(result) +} + +#[derive(Debug)] +struct Envelope { + deadline: Instant, + command: Command, + _body_permit: Option, +} + +#[derive(Debug)] +enum Command { + Submit { + invocation: Invocation, + cancellation: watch::Receiver, + response: oneshot::Sender>, + admitted: oneshot::Sender>, + }, + Next { + reply: oneshot::Sender>, + }, + Complete { + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + reply: oneshot::Sender>, + }, + InitError { + payload: Payload, + reply: oneshot::Sender>, + }, +} + +#[derive(Debug)] +struct Pending { + attempt: AttemptId, + deadline: Instant, + timeout_error: SessionError, + cancellation: watch::Receiver, + response: oneshot::Sender>, +} + +impl Pending { + fn check(&self) -> Result<(), SessionError> { + if Instant::now() >= self.deadline { + return Err(self.timeout_error); + } + if *self.cancellation.borrow() || self.cancellation.has_changed().is_err() { + Err(SessionError::Cancelled) + } else { + Ok(()) + } + } +} + +#[derive(Debug)] +struct PollDelivery { + invocation: Invocation, + receipt: oneshot::Sender, +} + +#[derive(Debug)] +enum PhaseEvent { + Cancelled, + Receipt(Option), +} + +#[derive(Debug, Default)] +enum Phase { + #[default] + Idle, + Queued { + invocation: Invocation, + pending: Pending, + }, + Offering { + invocation: Invocation, + pending: Pending, + receipt: oneshot::Receiver, + }, + Delivered(Pending), + Closed, +} + +impl Phase { + fn pending(&self) -> Option<&Pending> { + match self { + Self::Queued { pending, .. } + | Self::Offering { pending, .. } + | Self::Delivered(pending) => Some(pending), + _ => None, + } + } + + async fn event(&mut self) -> PhaseEvent { + match self { + Self::Offering { + pending, receipt, .. + } => { + tokio::select! { + biased; + () = async { let _ = pending.cancellation.wait_for(|cancelled| *cancelled).await; } => PhaseEvent::Cancelled, + result = receipt => PhaseEvent::Receipt(result.ok()), + } + } + Self::Queued { pending, .. } | Self::Delivered(pending) => { + let _ = pending.cancellation.wait_for(|cancelled| *cancelled).await; + PhaseEvent::Cancelled + } + _ => pending().await, + } + } +} + +#[derive(Debug, TypedBuilder)] +struct SessionActor { + inbox: mpsc::Receiver, + stop: watch::Receiver, + bootstrap: watch::Sender, + initialization_error: Option>, + idle_lease: Duration, + idle_deadline: Instant, + #[builder(default)] + phase: Phase, + #[builder(default)] + poll: Option>>, + #[builder(default)] + ever_delivered: bool, +} + +impl SessionActor { + async fn run(mut self) { + loop { + if *self.stop.borrow() { + self.close(SessionError::Cancelled); + break; + } + self.reconcile_receipt(); + if matches!(self.phase, Phase::Closed) { + break; + } + let deadline = self + .phase + .pending() + .map_or(self.idle_deadline, |pending| pending.deadline); + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => { self.close(SessionError::Cancelled); break; } + () = sleep_until(deadline) => { + self.reconcile_receipt(); + if matches!(self.phase, Phase::Closed) { break; } + let error = self.phase.pending().map_or_else( + || (Instant::now() >= self.idle_deadline).then_some(SessionError::Deadline), + |pending| pending.check().err(), + ); + if let Some(error) = error { self.close(error); break; } + } + event = self.phase.event() => { + match event { + PhaseEvent::Cancelled => { self.close(SessionError::Cancelled); break; } + PhaseEvent::Receipt(received) => self.received(received), + } + } + command = self.inbox.recv() => { + let Some(Envelope { command, deadline, _body_permit }) = command else { self.close(SessionError::Unavailable); break; }; + if let Err(error) = check_deadline(deadline) { reject(command, error); continue; } + if self.command(command) { + let Some(deadline) = Instant::now().checked_add(self.idle_lease) else { self.close(SessionError::InvalidMetadata); break; }; + self.idle_deadline = deadline; + } + if matches!(self.phase, Phase::Closed) { break; } + self.deliver(); + } + } + } + } + + fn command(&mut self, command: Command) -> bool { + self.reconcile_receipt(); + let error = if *self.stop.borrow() { + Some(SessionError::Cancelled) + } else if matches!(self.phase, Phase::Closed) { + Some(SessionError::Unavailable) + } else { + self.phase + .pending() + .and_then(|pending| pending.check().err()) + }; + if let Some(error) = error { + self.close(error); + reject(command, error); + return false; + } + match command { + Command::Submit { + invocation, + cancellation, + response, + admitted, + } => { + let error = if !matches!(self.phase, Phase::Idle) { + Some(SessionError::Busy) + } else if check_deadline(invocation.deadline).is_err() { + Some(SessionError::Deadline) + } else if *cancellation.borrow() || cancellation.has_changed().is_err() { + Some(SessionError::Cancelled) + } else { + None + }; + if let Some(error) = error { + let _ = admitted.send(Err(error)); + return false; + } + let pending = Pending { + attempt: invocation.attempt, + deadline: invocation.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }; + self.phase = Phase::Queued { + invocation, + pending, + }; + let _ = admitted.send(Ok(())); + true + } + Command::Next { reply } => { + if matches!(self.phase, Phase::Offering { .. } | Phase::Delivered(_)) + || self.poll.as_ref().is_some_and(|poll| !poll.is_closed()) + { + let _ = reply.send(Err(SessionError::Busy)); + return false; + } + if reply.is_closed() { + return false; + } + self.bootstrap.send_if_modified(|ready| { + let changed = !*ready; + *ready = true; + changed + }); + self.poll = Some(reply); + true + } + Command::Complete { + attempt, + invocation_id, + outcome, + payload, + reply, + } => self.complete(attempt, invocation_id, outcome, payload, reply), + Command::InitError { payload, reply } => { + if let Some(error) = self + .phase + .pending() + .and_then(|pending| pending.check().err()) + { + self.close(error); + let _ = reply.send(Err(error)); + return false; + } + if self.ever_delivered { + let _ = reply.send(Err(SessionError::Busy)); + return false; + } + if let Some(notification) = self.initialization_error.take() { + let _ = notification.send(payload.clone()); + } + if let Some(pending) = self.take_pending() { + let _ = pending.response.send(Ok(RuntimeResult::InitError(payload))); + } + self.close(SessionError::Unavailable); + let _ = reply.send(Ok(())); + true + } + } + } + + fn reconcile_receipt(&mut self) { + let received = match &mut self.phase { + Phase::Offering { receipt, .. } => match receipt.try_recv() { + Ok(started) => Some(Some(started)), + Err(TryRecvError::Closed) => Some(None), + Err(TryRecvError::Empty) => None, + }, + _ => None, + }; + if let Some(received) = received { + self.received(received); + } + } + + fn complete( + &mut self, + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + reply: oneshot::Sender>, + ) -> bool { + let valid = matches!(&self.phase, Phase::Delivered(pending) if pending.attempt == attempt) + && invocation_id.is_none_or(|id| id == attempt); + if !valid { + let _ = reply.send(Err(SessionError::UnknownAttempt)); + return false; + } + let Phase::Delivered(pending) = replace(&mut self.phase, Phase::Idle) else { + let _ = reply.send(Err(SessionError::Unavailable)); + return false; + }; + let result = if let Err(error) = pending.check() { + let _ = pending.response.send(Err(error)); + self.close(error); + Err(error) + } else { + let result = match outcome { + Outcome::Success => RuntimeResult::Success(payload), + Outcome::FunctionError => RuntimeResult::FunctionError(payload), + }; + if pending.response.send(Ok(result)).is_err() { + self.close(SessionError::Cancelled); + Err(SessionError::Cancelled) + } else { + Ok(()) + } + }; + let accepted = result.is_ok(); + let _ = reply.send(result); + accepted + } + + fn deliver(&mut self) { + if !matches!(self.phase, Phase::Queued { .. }) { + return; + } + let Some(poll) = self.poll.take() else { + return; + }; + let Phase::Queued { + invocation, + pending, + } = replace(&mut self.phase, Phase::Idle) + else { + return; + }; + if let Err(error) = pending.check() { + let _ = poll.send(Err(error)); + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + let (receipt, received) = oneshot::channel(); + match poll.send(Ok(PollDelivery { + invocation: invocation.clone(), + receipt, + })) { + Ok(()) => { + self.phase = Phase::Offering { + invocation, + pending, + receipt: received, + }; + } + Err(_) => { + self.phase = Phase::Queued { + invocation, + pending, + }; + } + } + } + + fn received(&mut self, received: Option) { + let Phase::Offering { + invocation, + mut pending, + .. + } = replace(&mut self.phase, Phase::Idle) + else { + return; + }; + if let Some(started) = received { + match invocation.timing(started) { + Ok((deadline, error)) => { + pending.deadline = deadline; + pending.timeout_error = error; + } + Err(error) => { + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + } + self.ever_delivered = true; + } + if let Err(error) = pending.check() { + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + self.phase = if received.is_some() { + Phase::Delivered(pending) + } else { + Phase::Queued { + invocation, + pending, + } + }; + } + + fn take_pending(&mut self) -> Option { + match replace(&mut self.phase, Phase::Closed) { + Phase::Queued { pending, .. } + | Phase::Offering { pending, .. } + | Phase::Delivered(pending) => Some(pending), + _ => None, + } + } + + fn close(&mut self, error: SessionError) { + if let Some(pending) = self.take_pending() { + let _ = pending.response.send(Err(error)); + } + if let Some(poll) = self.poll.take() { + let _ = poll.send(Err(error)); + } + self.inbox.close(); + } +} + +fn reject(command: Command, error: SessionError) { + match command { + Command::Submit { + admitted, response, .. + } => { + let _ = admitted.send(Err(error)); + let _ = response.send(Err(error)); + } + Command::Next { reply } => { + let _ = reply.send(Err(error)); + } + Command::Complete { reply, .. } | Command::InitError { reply, .. } => { + let _ = reply.send(Err(error)); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/session/tests.rs b/crates/rustack-lambda-runtime/src/session/tests.rs new file mode 100644 index 00000000..b75838db --- /dev/null +++ b/crates/rustack-lambda-runtime/src/session/tests.rs @@ -0,0 +1,958 @@ +use std::{ + future::Future, + sync::Arc, + task::{Context, Poll, Waker}, +}; + +use bytes::Bytes; +use tokio::sync::Semaphore; + +use super::*; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} + +fn payload() -> Payload { + Payload::try_from(Bytes::from_static(b"secret-payload")).unwrap() +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + "Root=1-00000000-000000000000000000000000;Sampled=0" + .parse() + .unwrap(), + payload(), + ) + .unwrap() +} + +fn poll_once(future: &mut F) -> Poll { + Future::poll( + std::pin::Pin::new(future), + &mut Context::from_waker(Waker::noop()), + ) +} + +#[tokio::test] +async fn test_should_deliver_once_validate_completion_and_reuse_after_success_and_function_error() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let first = invocation(); + let ticket = handle.submit(first.clone(), deadline()).await.unwrap(); + assert_eq!(ticket.attempt(), first.parts().0); + assert_eq!( + handle.submit(invocation(), deadline()).await.unwrap_err(), + SessionError::Busy + ); + assert_eq!(handle.next(deadline()).await.unwrap(), first); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Busy + ); + assert_eq!( + handle + .complete( + AttemptId::default(), + None, + Outcome::Success, + payload(), + deadline() + ) + .await, + Err(SessionError::UnknownAttempt) + ); + assert_eq!( + handle + .complete( + ticket.attempt(), + Some(AttemptId::default()), + Outcome::Success, + payload(), + deadline() + ) + .await, + Err(SessionError::UnknownAttempt) + ); + handle + .complete( + ticket.attempt(), + Some(ticket.attempt()), + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + let old = ticket.attempt(); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::Success(payload()) + ); + let second = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + assert_eq!( + handle + .complete(old, None, Outcome::Success, payload(), deadline()) + .await, + Err(SessionError::UnknownAttempt) + ); + handle + .complete( + second.attempt(), + None, + Outcome::FunctionError, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + second.wait().await.unwrap(), + RuntimeResult::FunctionError(payload()) + ); + session.shutdown().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_distinguish_listener_readiness_first_poll_and_cancelled_poll_replacement() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut readiness = handle.bootstrap.clone(); + assert_eq!( + handle + .bootstrap_ready(Instant::now() + Duration::from_millis(1)) + .await, + Err(SessionError::Deadline) + ); + let mut first = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut first).is_pending()); + handle.bootstrap_ready(deadline()).await.unwrap(); + assert!(*readiness.borrow_and_update()); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Busy + ); + drop(first); + let call = invocation(); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + assert_eq!(handle.next(deadline()).await.unwrap(), call); + assert!( + !readiness.has_changed().unwrap(), + "readiness must be emitted once" + ); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_report_initialization_failure_once_before_delivery_and_close_pending_work() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let notification = session.take_initialization_error().unwrap(); + assert!(session.take_initialization_error().is_none()); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle + .initialization_error(payload(), deadline()) + .await + .unwrap(); + assert_eq!(notification.await.unwrap(), payload()); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::InitError(payload()) + ); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_reject_initialization_error_after_any_delivery_without_consuming_active_result() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + assert_eq!( + handle.initialization_error(payload(), deadline()).await, + Err(SessionError::Busy) + ); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + assert_eq!( + handle.initialization_error(payload(), deadline()).await, + Err(SessionError::Busy) + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_invalidate_dropped_tickets_and_expired_attempts_without_reuse() { + for delivered in [false, true] { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + if delivered { + handle.next(deadline()).await.unwrap(); + } + drop(ticket); + tokio::task::yield_now().await; + assert_eq!( + handle.submit(invocation(), deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); + } + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(ticket.wait().await.unwrap_err(), SessionError::Deadline); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_cancel_attached_waiters_and_never_cancel_new_work_when_an_old_ticket_drops() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let old = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + handle + .complete(old.attempt(), None, Outcome::Success, payload(), deadline()) + .await + .unwrap(); + let current = handle.submit(invocation(), deadline()).await.unwrap(); + drop(old); + handle.next(deadline()).await.unwrap(); + handle + .complete( + current.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + current.wait().await.unwrap(); + + let cancelled = handle.submit(invocation(), deadline()).await.unwrap(); + let mut waiting = Box::pin(cancelled.wait()); + assert!(poll_once(&mut waiting).is_pending()); + drop(waiting); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_retain_actor_join_ownership_after_a_cancelled_shutdown_waiter() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut shutdown = Box::pin(session.shutdown()); + assert!(poll_once(&mut shutdown).is_pending()); + drop(shutdown); + assert!(session.task.is_some()); + session.shutdown().await.unwrap(); + assert!(session.task.is_none()); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); +} + +#[tokio::test] +async fn test_should_cancel_admitted_work_after_the_admission_waiter_is_dropped() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut admission = Box::pin(handle.submit(invocation(), deadline())); + assert!(poll_once(&mut admission).is_pending()); + tokio::task::yield_now().await; + drop(admission); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_idle_and_command_deadlines_and_prioritize_stop_over_a_full_mailbox() { + assert!(matches!( + RuntimeSession::start(Duration::ZERO), + Err(SessionError::InvalidMetadata) + )); + assert!(matches!( + RuntimeSession::start(Duration::from_hours(25)), + Err(SessionError::InvalidMetadata) + )); + let mut session = RuntimeSession::start(Duration::from_secs(1)).unwrap(); + let handle = session.handle().clone(); + assert_eq!( + handle.next(Instant::now()).await.unwrap_err(), + SessionError::Deadline + ); + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); + + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut receivers = Vec::new(); + for _ in 0..MAILBOX_CAPACITY { + let (reply, receiver) = oneshot::channel(); + handle.send(Command::Next { reply }, deadline()).unwrap(); + receivers.push(receiver); + } + let (reply, _) = oneshot::channel(); + assert_eq!( + handle.send(Command::Next { reply }, deadline()), + Err(SessionError::Busy) + ); + handle.close(); + session.shutdown().await.unwrap(); + assert!( + !*handle.bootstrap.borrow(), + "priority stop must beat queued polls" + ); + for receiver in receivers { + assert!(receiver.await.is_err()); + } +} + +#[tokio::test] +async fn test_should_never_report_success_after_actor_supervision_failure() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + session.task.as_ref().unwrap().abort(); + assert_eq!( + session.shutdown().await, + Err(SessionError::SupervisorFailed) + ); + assert_eq!( + session.shutdown().await, + Err(SessionError::SupervisorFailed) + ); +} + +#[test] +fn test_should_reject_start_without_a_tokio_runtime() { + assert!(matches!( + RuntimeSession::start(Duration::from_secs(60)), + Err(SessionError::Unavailable) + )); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_expired_mailbox_commands_without_delivering_or_settling_work() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let (reply, response) = oneshot::channel(); + handle + .send( + Command::Next { reply }, + Instant::now() + Duration::from_millis(1), + ) + .unwrap(); + tokio::time::advance(Duration::from_millis(1)).await; + assert!(matches!( + response.await.unwrap(), + Err(SessionError::Deadline) + )); + assert!(!*handle.bootstrap.borrow()); + + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + let (reply, response) = oneshot::channel(); + handle + .send( + Command::Complete { + attempt: ticket.attempt(), + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply, + }, + Instant::now() + Duration::from_millis(1), + ) + .unwrap(); + tokio::time::advance(Duration::from_millis(1)).await; + assert_eq!(response.await.unwrap(), Err(SessionError::Deadline)); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::Success(payload()) + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_signal_owned_actor_cleanup_when_the_session_owner_is_dropped() { + let session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation(); + assert!( + !format!("{call:?} {:?}", RuntimeResult::Success(payload())).contains("secret-payload") + ); + let ticket = handle.submit(call, deadline()).await.unwrap(); + assert!(!format!("{ticket:?}").contains("secret-payload")); + drop(session); + assert_eq!(ticket.wait().await, Err(SessionError::Cancelled)); + assert_eq!( + handle.next(deadline()).await, + Err(SessionError::Unavailable) + ); +} + +#[tokio::test] +async fn test_should_requeue_an_offered_event_if_the_poll_future_drops_before_receiving_it() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation(); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + let mut poll = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut poll).is_pending()); + // The actor sends the event, but this poll future never consumes the reply. + tokio::task::yield_now().await; + drop(poll); + assert_eq!(handle.next(deadline()).await.unwrap(), call); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_reconcile_ready_poll_receipts_before_completion_or_initialization_commands() { + for initialization in [false, true] { + let mut actor = transition_actor(); + let invocation = invocation(); + let attempt = invocation.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: invocation.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }; + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation, + pending, + receipt: acknowledged, + }; + receipt.send(Instant::now()).unwrap(); + let (reply, result) = oneshot::channel(); + if initialization { + assert!(!actor.command(Command::InitError { + payload: payload(), + reply + })); + assert_eq!(result.await.unwrap(), Err(SessionError::Busy)); + assert!(actor.ever_delivered); + assert!(matches!(actor.phase, Phase::Delivered(_))); + let (reply, _) = oneshot::channel(); + assert!(actor.complete(attempt, None, Outcome::Success, payload(), reply)); + } else { + assert!(actor.command(Command::Complete { + attempt, + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply + })); + assert_eq!(result.await.unwrap(), Ok(())); + } + assert_eq!( + received.await.unwrap(), + Ok(RuntimeResult::Success(payload())) + ); + } +} + +fn transition_actor() -> SessionActor { + let (_, inbox) = mpsc::channel(MAILBOX_CAPACITY); + let (_, stop) = watch::channel(false); + let (bootstrap, _) = watch::channel(false); + SessionActor::builder() + .inbox(inbox) + .stop(stop) + .bootstrap(bootstrap) + .initialization_error(None) + .idle_lease(Duration::from_secs(60)) + .idle_deadline(Instant::now() + Duration::from_secs(60)) + .build() +} + +#[tokio::test] +async fn test_should_resume_a_borrowed_ticket_result_after_select_cancellation_and_reject_repolling() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut ticket = handle.submit(invocation(), deadline()).await.unwrap(); + let mut result = Box::pin(ticket.result()); + assert!(poll_once(&mut result).is_pending()); + drop(result); + assert!(!*ticket.cancellation.borrow()); + handle.next(deadline()).await.unwrap(); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + ticket.result().await.unwrap(), + RuntimeResult::Success(payload()) + ); + assert_eq!(ticket.result().await, Err(SessionError::Unavailable)); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_delivery_transition() +{ + for transition in ["deliver", "complete", "receipt"] { + for condition in ["cancelled", "closed", "expired"] { + let mut actor = transition_actor(); + let invocation = invocation(); + let attempt = invocation.attempt; + let (cancel, cancellation) = watch::channel(false); + let mut cancel = Some(cancel); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: invocation.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }; + match condition { + "cancelled" => { + cancel.as_ref().unwrap().send_replace(true); + } + "closed" => { + cancel.take(); + } + "expired" => tokio::time::advance(Duration::from_secs(5)).await, + _ => panic!("unknown test case"), + } + let expected = if condition == "expired" { + SessionError::Deadline + } else { + SessionError::Cancelled + }; + match transition { + "deliver" => { + actor.phase = Phase::Queued { + invocation, + pending, + }; + let (poll, received) = oneshot::channel(); + actor.poll = Some(poll); + actor.deliver(); + assert!(matches!(received.await.unwrap(), Err(error) if error == expected)); + } + "complete" => { + actor.phase = Phase::Delivered(pending); + let (reply, received) = oneshot::channel(); + assert!(!actor.complete(attempt, None, Outcome::Success, payload(), reply)); + assert_eq!(received.await.unwrap(), Err(expected)); + } + "receipt" => { + let (_, receipt) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation, + pending, + receipt, + }; + actor.received(Some(Instant::now())); + } + _ => panic!("unknown test case"), + } + assert_eq!(received.await.unwrap(), Err(expected)); + assert!(matches!(actor.phase, Phase::Closed)); + assert!(actor.inbox.is_closed()); + } + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_arm_handler_timeout_only_when_the_runtime_consumes_the_offered_event() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation(); + call.deadline = Instant::now() + Duration::from_secs(30); + call = call.with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!handle.is_closed()); + let mut abandoned = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut abandoned).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!handle.is_closed()); + drop(abandoned); + let started = Instant::now(); + let delivered = handle.next(deadline()).await.unwrap(); + assert_eq!(delivered.parts().1, started + Duration::from_secs(1)); + assert_ne!(delivered.parts().1, call.parts().1); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(ticket.wait().await, Err(SessionError::HandlerTimeout)); + assert!(handle.is_closed()); + session.shutdown().await.unwrap(); +} +#[tokio::test(start_paused = true)] +async fn test_should_keep_outer_deadline_classification_when_shorter_or_tied_with_handler_timeout() +{ + for remaining in [Duration::from_millis(500), Duration::from_secs(1)] { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation(); + call.deadline = Instant::now() + remaining; + call = call.with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let expected = call.parts().1; + let ticket = handle.submit(call, deadline()).await.unwrap(); + assert_eq!(handle.next(deadline()).await.unwrap().parts().1, expected); + tokio::time::advance(remaining).await; + assert_eq!(ticket.wait().await, Err(SessionError::Deadline)); + session.shutdown().await.unwrap(); + } +} +#[tokio::test(start_paused = true)] +async fn test_should_reject_late_handler_completion_and_delayed_consumption_receipt_without_replay() +{ + for transition in ["receipt", "complete", "receipt_after_outer"] { + let mut actor = transition_actor(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let attempt = call.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: call.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }; + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation: call, + pending, + receipt: acknowledged, + }; + receipt.send(Instant::now()).unwrap(); + if transition == "complete" { + actor.reconcile_receipt(); + } + tokio::time::advance(Duration::from_secs( + if transition == "receipt_after_outer" { + 6 + } else { + 1 + }, + )) + .await; + let (reply, result) = oneshot::channel(); + assert!(!actor.command(Command::Complete { + attempt, + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply + })); + assert_eq!( + result.await.unwrap(), + Err(if transition == "complete" { + SessionError::HandlerTimeout + } else { + SessionError::Unavailable + }) + ); + assert_eq!(received.await.unwrap(), Err(SessionError::HandlerTimeout)); + assert!(matches!(actor.phase, Phase::Closed)); + } +} +#[tokio::test(start_paused = true)] +async fn test_should_preserve_on_time_completion_when_the_ticket_waiter_resumes_after_handler_expiry() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let ticket = handle.submit(call, deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(ticket.wait().await, Ok(RuntimeResult::Success(payload()))); + assert!(!handle.is_closed()); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_an_offered_call_consumed_after_its_outer_deadline_before_receipting_delivery() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + call.deadline = Instant::now() + Duration::from_secs(2); + let ticket = handle.submit(call, deadline()).await.unwrap(); + let mut next = Box::pin(handle.next(Instant::now() + Duration::from_secs(10))); + assert!(poll_once(&mut next).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(next.await, Err(SessionError::Deadline)); + assert_eq!(ticket.wait().await, Err(SessionError::Deadline)); + tokio::time::timeout(Duration::from_secs(1), handle.closed()) + .await + .unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reconcile_consumed_delivery_when_receipt_and_old_outer_timer_become_ready_between_actor_polls() + { + let (commands, inbox) = mpsc::channel(MAILBOX_CAPACITY); + let (stop, stopped) = watch::channel(false); + let (bootstrap, _) = watch::channel(false); + let mut actor = SessionActor::builder() + .inbox(inbox) + .stop(stopped) + .bootstrap(bootstrap) + .initialization_error(None) + .idle_lease(Duration::from_secs(60)) + .idle_deadline(Instant::now() + Duration::from_secs(60)) + .build(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let attempt = call.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + pending: Pending { + attempt, + deadline: call.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }, + invocation: call, + receipt: acknowledged, + }; + let mut running = Box::pin(actor.run()); + assert!(poll_once(&mut running).is_pending()); + receipt.send(Instant::now()).unwrap(); + tokio::time::advance(Duration::from_secs(6)).await; + assert!(poll_once(&mut running).is_ready()); + assert_eq!(received.await.unwrap(), Err(SessionError::HandlerTimeout)); + drop((commands, stop)); +} + +fn held_http_session() -> (SessionHandle, SessionActor) { + let (commands, inbox) = mpsc::channel(MAILBOX_CAPACITY); + let (stop, stopped) = watch::channel(false); + let (bootstrap, initialized) = watch::channel(false); + let actor = SessionActor::builder() + .inbox(inbox) + .stop(stopped) + .bootstrap(bootstrap) + .initialization_error(None) + .idle_lease(Duration::from_secs(60)) + .idle_deadline(Instant::now() + Duration::from_secs(60)) + .build(); + let handle = SessionHandle { + inbox: commands, + stop, + bootstrap: initialized, + }; + (handle, actor) +} + +async fn http_body_command( + handle: &SessionHandle, + initialization: bool, + deadline: Instant, + permit: OwnedSemaphorePermit, +) -> Result<(), SessionError> { + if initialization { + handle + .initialization_error_with_body_permit(payload(), deadline, Some(permit)) + .await + } else { + handle + .complete_with_body_permit( + AttemptId::default(), + None, + Outcome::Success, + payload(), + deadline, + Some(permit), + ) + .await + } +} + +#[rstest::rstest] +#[case::completion(false)] +#[case::initialization_error(true)] +#[tokio::test(start_paused = true)] +async fn test_should_charge_http_body_until_accepted_command_is_retired( + #[case] initialization: bool, + #[values(false, true)] abandon: bool, + #[values(false, true)] dispose_actor: bool, +) { + let (handle, mut actor) = held_http_session(); + let (notification, mut notified) = oneshot::channel(); + actor.initialization_error = Some(notification); + let budget = Arc::new(Semaphore::new(4)); + let cutoff = Instant::now() + Duration::from_secs(1); + let mut observers: Vec<_> = (0..4) + .map(|_| { + Box::pin(http_body_command( + &handle, + initialization, + cutoff, + budget.clone().try_acquire_owned().unwrap(), + )) + }) + .collect(); + for observer in &mut observers { + assert!(poll_once(observer).is_pending()); + } + assert_eq!(actor.inbox.len(), 4); + assert_eq!(budget.available_permits(), 0); + if abandon { + observers.clear(); + } + tokio::time::advance(Duration::from_secs(1)).await; + for observer in &mut observers { + assert_eq!( + poll_once(observer), + Poll::Ready(Err(SessionError::Deadline)) + ); + } + drop(observers); + assert_eq!(actor.inbox.len(), 4); + assert!(!*handle.stop.borrow()); + assert_eq!(budget.available_permits(), 0); + assert!(budget.clone().try_acquire_owned().is_err()); + if dispose_actor { + drop(actor); + } else { + let mut running = Box::pin(actor.run()); + assert!(poll_once(&mut running).is_pending()); + assert_eq!(handle.inbox.capacity(), MAILBOX_CAPACITY); + assert_eq!(notified.try_recv(), Err(TryRecvError::Empty)); + handle.close(); + assert!(poll_once(&mut running).is_ready()); + } + assert_eq!(budget.available_permits(), 4); + assert!(budget.try_acquire_owned().is_ok()); +} + +#[rstest::rstest] +#[case::expired(SessionError::Deadline)] +#[case::full(SessionError::Busy)] +#[case::closed(SessionError::Unavailable)] +#[tokio::test(start_paused = true)] +async fn test_should_charge_http_body_only_when_its_command_is_admitted( + #[case] rejection: SessionError, + #[values(false, true)] initialization: bool, +) { + let (handle, mut actor) = held_http_session(); + let mut cutoff = deadline(); + match rejection { + SessionError::Deadline => cutoff = Instant::now(), + SessionError::Busy => { + for _ in 0..MAILBOX_CAPACITY { + let (reply, _) = oneshot::channel(); + handle.send(Command::Next { reply }, cutoff).unwrap(); + } + } + SessionError::Unavailable => actor.inbox.close(), + _ => unreachable!("fixed test rejection cases"), + } + let budget = Arc::new(Semaphore::new(1)); + let permit = budget.clone().try_acquire_owned().unwrap(); + assert_eq!( + http_body_command(&handle, initialization, cutoff, permit).await, + Err(rejection) + ); + assert_eq!(budget.available_permits(), 1); + drop(actor); +} diff --git a/crates/rustack-lambda-runtime/src/transport.rs b/crates/rustack-lambda-runtime/src/transport.rs new file mode 100644 index 00000000..8cc7a354 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/transport.rs @@ -0,0 +1,285 @@ +//! Retained, bounded stream workers shared by both ends of relay control. + +use std::time::Duration; + +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite}, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use crate::{ + protocol::{Acknowledgement, Frame, Message, read_frame, write_frame}, + relay::RelayError, +}; + +const MAILBOX_CAPACITY: usize = 2; + +pub(crate) fn preserve_worker_failure( + primary: Result<(), RelayError>, + joined: Result<(), RelayError>, +) -> Result<(), RelayError> { + if matches!(&primary, Err(RelayError::Worker)) || matches!(&joined, Err(RelayError::Worker)) { + Err(RelayError::Worker) + } else { + primary.and(joined) + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum InputEnd { + Close, + CloseAcknowledgement, +} + +impl InputEnd { + fn matches(self, message: &Message) -> bool { + matches!( + (self, message), + (Self::Close, Message::Close) + | ( + Self::CloseAcknowledgement, + Message::Ack(Acknowledgement::Close) + ) + ) + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Settings { + pub(crate) silence: Duration, + pub(crate) io_budget: Duration, + pub(crate) input_end: InputEnd, +} + +#[derive(Debug)] +struct Outbound { + frame: Frame, + deadline: Instant, + written: oneshot::Sender, +} + +#[derive(Debug)] +enum WorkerExit { + InputClose, + OutputClosed, +} + +#[derive(Debug)] +struct OwnerHandle { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +#[derive(Debug)] +pub(crate) struct Transport { + input: mpsc::Receiver, + output: mpsc::Sender, + status: watch::Receiver>>, + owner: OwnerHandle, +} + +impl Transport { + #[cfg(test)] + pub(crate) fn abort_owner(&self) { + if let Some(task) = &self.owner.task { + task.abort(); + } + } + + pub(crate) fn start(reader: R, writer: W, settings: Settings) -> Result + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let runtime = Handle::try_current().map_err(|_| RelayError::Worker)?; + let (stop, stopped) = watch::channel(false); + let (input, frames) = mpsc::channel(MAILBOX_CAPACITY); + let (output, queued) = mpsc::channel(MAILBOX_CAPACITY); + let (status, observed) = watch::channel(None); + let mut workers = JoinSet::new(); + workers.spawn(read_input(reader, input, settings).with_current_subscriber()); + workers.spawn(write_output(writer, queued).with_current_subscriber()); + let task = runtime.spawn( + Owner { + workers, + stop: stopped, + status, + } + .run() + .with_current_subscriber(), + ); + Ok(Self { + input: frames, + output, + status: observed, + owner: OwnerHandle { + stop, + task: Some(task), + settlement: None, + }, + }) + } + + pub(crate) fn finished(&self) -> bool { + *self.owner.stop.borrow() + || self.status.borrow().is_some() + || self.status.has_changed().is_err() + } + + pub(crate) fn stop(&self) { + self.owner.stop.send_replace(true); + } + + pub(crate) async fn recv(&mut self) -> Result { + loop { + if let Some(result) = self.status.borrow().as_ref() { + return Err(result.clone().err().unwrap_or(RelayError::Eof)); + } + tokio::select! { + biased; + changed = self.status.changed() => { + changed.map_err(|_| RelayError::Worker)?; + } + frame = self.input.recv() => match frame { + Some(frame) => return Ok(frame), + // The reader drops its sender before the supervisor joins + // it. Await that settlement so EOF cannot mask its actual + // protocol/deadline failure. + None => { self.status.changed().await.map_err(|_| RelayError::Worker)?; } + }, + } + } + } + + pub(crate) async fn send(&self, frame: Frame, deadline: Instant) -> Result<(), RelayError> { + if self.finished() { + return Err(RelayError::Output); + } + let (written, receipt) = oneshot::channel(); + timeout_at( + deadline, + self.output.send(Outbound { + frame, + deadline, + written, + }), + ) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Output)?; + match timeout_at(deadline, receipt).await { + Ok(Ok(true)) if Instant::now() < deadline => Ok(()), + Ok(Ok(true)) => Err(RelayError::Deadline), + _ => Err(RelayError::Output), + } + } + + pub(crate) async fn shutdown(&mut self) -> Result<(), RelayError> { + self.stop(); + if let Some(result) = &self.owner.settlement { + return result.clone(); + } + let result = match self.owner.task.as_mut() { + Some(task) => task + .await + .map_err(|_| RelayError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.owner.task.take(); + self.owner.settlement = Some(result.clone()); + result + } +} + +impl Drop for Transport { + fn drop(&mut self) { + self.stop(); + } +} + +#[derive(Debug)] +struct Owner { + workers: JoinSet>, + stop: watch::Receiver, + status: watch::Sender>>, +} + +impl Owner { + async fn run(mut self) -> Result<(), RelayError> { + let mut outcome = loop { + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stop| *stop).await; } => break Ok(()), + worker = self.workers.join_next() => match worker { + Some(Ok(Ok(WorkerExit::InputClose))) => {}, + Some(Ok(Err(error))) => break Err(error), + _ => break Err(RelayError::Worker), + } + } + }; + self.workers.abort_all(); + while let Some(result) = self.workers.join_next().await { + let result = match result { + Ok(Ok(WorkerExit::InputClose)) => Ok(()), + Ok(Err(error)) => Err(error), + Err(error) if error.is_cancelled() => Ok(()), + Ok(Ok(WorkerExit::OutputClosed)) | Err(_) => Err(RelayError::Worker), + }; + outcome = preserve_worker_failure(outcome, result); + } + self.status.send_replace(Some(outcome.clone())); + outcome + } +} + +async fn read_input( + mut reader: R, + input: mpsc::Sender, + settings: Settings, +) -> Result { + loop { + let mut first = [0_u8]; + let count = timeout_at(after(settings.silence)?, reader.read(&mut first)) + .await + .map_err(|_| RelayError::Deadline)??; + if count == 0 { + return Err(RelayError::Eof); + } + let mut prefixed = first.as_slice().chain(&mut reader); + let frame = read_frame(&mut prefixed, after(settings.io_budget)?) + .await? + .ok_or(RelayError::Eof)?; + let close = settings.input_end.matches(frame.message()); + timeout_at(after(settings.io_budget)?, input.send(frame)) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Worker)?; + if close { + return Ok(WorkerExit::InputClose); + } + } +} + +async fn write_output( + mut writer: W, + mut output: mpsc::Receiver, +) -> Result { + while let Some(frame) = output.recv().await { + let result = write_frame(&mut writer, &frame.frame, frame.deadline).await; + let _ = frame.written.send(result.is_ok()); + result?; + } + Ok(WorkerExit::OutputClosed) +} + +pub(crate) fn after(duration: Duration) -> Result { + Instant::now() + .checked_add(duration) + .ok_or(RelayError::Configuration) +} diff --git a/crates/rustack-lambda-runtime/tests/docker.rs b/crates/rustack-lambda-runtime/tests/docker.rs new file mode 100644 index 00000000..1250b572 --- /dev/null +++ b/crates/rustack-lambda-runtime/tests/docker.rs @@ -0,0 +1,757 @@ +//! Explicit, local-only qualification of the Linux relay and an official RIC. +//! +//! The CLI is a test transport. Production attach, deployment and pool behavior +//! are deliberately tested by the Docker executor's separate acceptance suite. + +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{env, os::unix::fs::FileTypeExt, process::Stdio, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + metadata::{deadline_unix_ms, local_trace_id}, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + read_frame, write_frame, + }, + session::{Invocation, RuntimeResult}, +}; +use serde_json::{Value, json}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + process::{Child, ChildStdin, ChildStdout, Command}, + time::{Instant, sleep_until, timeout, timeout_at}, +}; + +const OPERATION: Duration = Duration::from_secs(45); +const MAX_OUTPUT: usize = 64 * 1024; + +#[derive(Debug)] +struct Docker { + host: String, + architecture: Architecture, + relay_image: String, + function_image: String, +} + +impl Docker { + async fn configured() -> Result { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET") + .context("explicit local Docker socket is required")?; + ensure!( + socket.starts_with('/') && socket.len() <= 4096, + "Docker socket must be a bounded absolute Unix path" + ); + ensure!( + tokio::fs::metadata(&socket).await?.file_type().is_socket(), + "configured Docker socket is unavailable" + ); + let architecture = env::var("RUSTACK_RUNTIME_ARCH").context("architecture is required")?; + let (expected, architecture) = match architecture.as_str() { + "arm64" => ("aarch64", Architecture::Arm64), + "amd64" => ("x86_64", Architecture::X86_64), + _ => bail!("architecture must be arm64 or amd64"), + }; + let relay_image = env::var("RUSTACK_RUNTIME_IMAGE").context("relay image is required")?; + ensure!( + !relay_image.is_empty() + && relay_image.len() <= 1024 + && relay_image.bytes().all(|byte| byte.is_ascii_graphic()), + "relay image reference must be bounded printable ASCII" + ); + let function_image = format!( + "rustack-p0-handler:{}", + if architecture == Architecture::Arm64 { + "arm64" + } else { + "amd64" + } + ); + let docker = Self { + host: format!("unix://{socket}"), + architecture, + relay_image, + function_image, + }; + let info = docker + .checked(&["info", "--format", "{{.OSType}} {{.Architecture}}"]) + .await?; + ensure!( + info.trim() == format!("linux {expected}"), + "Docker daemon platform does not match this qualification lane" + ); + for image in [&docker.relay_image, &docker.function_image] { + docker.checked(&["image", "inspect", image]).await?; + } + Ok(docker) + } + + fn command(&self) -> Command { + let mut command = Command::new("docker"); + command + .arg("--host") + .arg(&self.host) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + command + } + + async fn output(&self, arguments: &[&str]) -> Result<(bool, String)> { + self.output_until(arguments, Instant::now() + OPERATION) + .await + } + + async fn output_until(&self, arguments: &[&str], deadline: Instant) -> Result<(bool, String)> { + ensure!(Instant::now() < deadline, "Docker CLI deadline expired"); + let mut child = self.command().args(arguments).spawn()?; + let stdout = child.stdout.take().context("CLI stdout pipe")?; + let stderr = child.stderr.take().context("CLI stderr pipe")?; + let outcome = timeout_at(deadline, async { + let (status, stdout, stderr) = + tokio::join!(child.wait(), bounded(stdout), bounded(stderr)); + Ok::<_, anyhow::Error>((status?.success(), stdout?, stderr?)) + }) + .await; + if let Ok(result) = outcome { + let (success, stdout, _) = result?; + Ok((success, String::from_utf8(stdout)?)) + } else { + child.kill().await.context("kill expired Docker CLI")?; + child.wait().await.context("reap expired Docker CLI")?; + bail!("Docker CLI operation deadline exceeded") + } + } + + async fn checked(&self, arguments: &[&str]) -> Result { + self.checked_until(arguments, Instant::now() + OPERATION) + .await + } + + async fn checked_until(&self, arguments: &[&str], deadline: Instant) -> Result { + let (success, output) = self.output_until(arguments, deadline).await?; + ensure!(success, "Docker CLI operation failed"); + Ok(output) + } +} + +async fn bounded(reader: impl AsyncRead + Unpin) -> Result> { + let mut output = Vec::new(); + reader + .take(u64::try_from(MAX_OUTPUT)? + 1) + .read_to_end(&mut output) + .await?; + ensure!(output.len() <= MAX_OUTPUT, "Docker CLI output exceeded cap"); + Ok(output) +} + +#[derive(Debug)] +struct Names { + relay: String, + function: String, + label: String, +} + +#[derive(Debug)] +struct Control { + child: Child, + input: Option, + output: Option, +} + +#[derive(Debug)] +struct Environment { + docker: Docker, + identity: EnvironmentId, + names: Names, + control: Option, +} + +impl Environment { + async fn new() -> Result { + let identity = EnvironmentId::default(); + Ok(Self { + docker: Docker::configured().await?, + identity, + names: Names { + relay: format!("rustack-runtime-relay-{identity}"), + function: format!("rustack-runtime-function-{identity}"), + label: format!("rustack.runtime.test={identity}"), + }, + control: None, + }) + } + + fn spawn(&mut self, idle_seconds: &str) -> Result<()> { + let mut command = self.docker.command(); + command + .args([ + "run", + "--pull=never", + "--rm", + "--interactive", + "--name", + &self.names.relay, + "--label", + &self.names.label, + "--network", + "none", + "--user", + "1000:1000", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + "64m", + "--cpus", + "0.1", + "--pids-limit", + "64", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=16777216", + "--env", + &format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={}", self.identity), + "--env", + &format!("RUSTACK_LAMBDA_RELAY_IDLE_SECONDS={idle_seconds}"), + &self.docker.relay_image, + ]) + .stdin(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = command.spawn()?; + let input = child.stdin.take().context("relay input pipe")?; + let output = child.stdout.take().context("relay output pipe")?; + self.control = Some(Control { + child, + input: Some(input), + output: Some(output), + }); + Ok(()) + } + + async fn start(&mut self, idle_seconds: &str) -> Result<()> { + self.spawn(idle_seconds)?; + ensure!( + self.next().await? + == Message::Hello { + architecture: self.docker.architecture + }, + "relay Hello/platform mismatch" + ); + ensure!(self.next().await? == Message::Ready, "missing relay Ready"); + let config = self + .docker + .checked(&["inspect", "--format", "{{json .Config}}", &self.names.relay]) + .await?; + let config: Value = serde_json::from_str(&config)?; + ensure!( + config["StdinOnce"] == true && config["Tty"] == false, + "qualification requires non-TTY stdin with EOF delivery" + ); + Ok(()) + } + + async fn next(&mut self) -> Result { + let control = self.control.as_mut().context("relay has not started")?; + let frame = read_frame( + control + .output + .as_mut() + .context("relay output is owned by client")?, + Instant::now() + OPERATION, + ) + .await? + .context("relay closed control output")?; + ensure!(frame.environment() == self.identity, "wrong environment"); + Ok(frame.into_message()) + } + + async fn send(&mut self, message: Message) -> Result<()> { + let input = self + .control + .as_mut() + .and_then(|control| control.input.as_mut()) + .context("relay input is closed")?; + write_frame( + input, + &Frame::new(self.identity, message), + Instant::now() + OPERATION, + ) + .await?; + Ok(()) + } + + async fn function(&mut self) -> Result<()> { + self.create_function().await?; + ensure!( + self.next().await? == Message::BootstrapReady, + "missing first RIC poll" + ); + Ok(()) + } + + async fn create_function(&self) -> Result<()> { + self.docker + .checked(&[ + "create", + "--pull=never", + "--name", + &self.names.function, + "--label", + &self.names.label, + "--network", + &format!("container:{}", self.names.relay), + "--user", + "1000:1000", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + "192m", + "--cpus", + "1", + "--pids-limit", + "256", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=536870912", + "--env", + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001", + "--env", + "AWS_LAMBDA_FUNCTION_NAME=spike", + "--env", + "AWS_LAMBDA_FUNCTION_VERSION=$LATEST", + "--env", + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192", + "--env", + "AWS_REGION=us-east-1", + "--env", + "AWS_DEFAULT_REGION=us-east-1", + &self.docker.function_image, + ]) + .await?; + self.docker + .checked(&["start", &self.names.function]) + .await?; + Ok(()) + } + + async fn invoke(&mut self, event: Value) -> Result { + let attempt = AttemptId::default(); + self.send(Message::Invoke { + attempt, + deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(60))?, + handler_timeout_ms: None, + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:spike:prod".parse()?, + trace_id: local_trace_id()?, + payload: Payload::try_from(Bytes::from(serde_json::to_vec(&event)?))?, + }) + .await?; + Ok(attempt) + } + + async fn result(&mut self, attempt: AttemptId, expected: Outcome) -> Result { + match self.next().await? { + Message::InvocationResult { + attempt: actual, + outcome, + payload, + } if actual == attempt && outcome == expected => { + serde_json::from_slice(payload.as_ref()).context("RIC JSON result") + } + _ => bail!("unexpected relay result identity/outcome/variant"), + } + } + + async fn wait(&mut self, success: bool) -> Result<()> { + let control = self.control.as_mut().context("relay has not started")?; + let status = timeout(OPERATION, control.child.wait()) + .await + .context("relay failed to exit with input ownership retained")??; + ensure!(status.success() == success, "unexpected relay exit status"); + Ok(()) + } + + async fn close(&mut self) -> Result<()> { + self.send(Message::Close).await?; + ensure!( + self.next().await? == Message::Ack(Acknowledgement::Close), + "missing flushed Close acknowledgement" + ); + self.wait(true).await + } + + async fn remove_function(&self) -> Result<()> { + self.docker + .checked(&["rm", "--force", &self.names.function]) + .await?; + Ok(()) + } + + async fn wait_busy(&self, attempt: AttemptId) -> Result<()> { + let marker = format!("P0_BUSY_START={attempt}"); + let busy_deadline = Instant::now() + OPERATION; + loop { + // The helper owns the deadline so it cannot be cancelled before + // its CLI process has been killed and reaped. + let logs = self + .docker + .checked_until(&["logs", &self.names.function], busy_deadline) + .await?; + if logs.contains(&marker) { + return Ok(()); + } + ensure!( + Instant::now() < busy_deadline, + "handler never entered its busy section" + ); + sleep_until((Instant::now() + Duration::from_millis(50)).min(busy_deadline)).await; + } + } + + async fn cleanup(&mut self) -> Result<()> { + // Names are reserved before create/run, even if the response is lost. + let removal = self + .docker + .output(&["rm", "--force", &self.names.function, &self.names.relay]) + .await; + let reaped = if let Some(control) = &mut self.control { + control.input.take(); + let killed = control.child.start_kill().context("kill relay CLI"); + let waited = if let Ok(result) = timeout(OPERATION, control.child.wait()).await { + result.map(|_| ()).context("reap relay CLI") + } else { + // An expired waiter must not discard process ownership. + let result = control + .child + .wait() + .await + .context("retained relay CLI reap"); + result.and_then(|_| bail!("relay CLI cleanup deadline")) + }; + killed.and(waited) + } else { + Ok(()) + }; + let reconciled = async { + let remaining = self + .docker + .checked(&[ + "ps", + "--all", + "--quiet", + "--filter", + &format!("label={}", self.names.label), + ]) + .await?; + ensure!(remaining.trim().is_empty(), "owned containers remain"); + Ok(()) + } + .await; + removal.map(|_| ()).and(reaped).and(reconciled) + } + + async fn finish(&mut self, outcome: Result<()>) -> Result<()> { + let cleanup = self.cleanup().await; + match (outcome, cleanup) { + (Err(error), Err(cleanup)) => { + Err(error.context(format!("cleanup failed: {cleanup:#}"))) + } + (Err(error), Ok(())) => Err(error), + (Ok(()), cleanup) => cleanup, + } + } +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_exit_on_close_eof_truncation_and_io_timeout_with_owned_pipes() -> Result<()> { + for mode in ["close", "eof", "truncated", "partial", "idle"] { + let mut environment = Environment::new().await?; + let result = async { + // Allow time for inspect during setup; idle expiry must still exit + // while the parent retains the open, otherwise silent stdin pipe. + environment + .start(if mode == "idle" { "5" } else { "600" }) + .await?; + if mode == "close" { + environment.close().await?; + } else { + let control = environment.control.as_mut().context("relay control")?; + if mode == "truncated" || mode == "partial" { + control + .input + .as_mut() + .context("input")? + .write_all(&[0]) + .await?; + } + if mode == "eof" || mode == "truncated" { + control.input.take(); + } + environment.wait(false).await?; + } + Ok(()) + } + .await; + environment + .finish(result) + .await + .with_context(|| format!("pipe case {mode}"))?; + } + Ok(()) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_reconcile_failed_setup_and_reap_an_expired_cli_helper() -> Result<()> { + for mode in ["setup", "scope", "cli"] { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + match mode { + "setup" => { + environment.docker.function_image = + format!("rustack-runtime-missing:{}", environment.identity); + ensure!( + environment.function().await.is_err(), + "missing image setup unexpectedly succeeded" + ); + } + "scope" => { + let input = environment + .control + .as_mut() + .and_then(|control| control.input.as_mut()) + .context("relay input")?; + write_frame( + input, + &Frame::new(EnvironmentId::default(), Message::Close), + Instant::now() + OPERATION, + ) + .await?; + environment.wait(false).await?; + } + _ => { + // docker wait owns a different CLI child and cannot finish + // while the relay is running. Its own deadline must reap + // that child without cancelling or stopping the relay. + let expired = environment + .docker + .checked_until( + &["wait", &environment.names.relay], + Instant::now() + Duration::from_millis(200), + ) + .await; + ensure!(expired.is_err(), "CLI helper did not reach its deadline"); + environment.close().await?; + } + } + Ok(()) + } + .await; + environment + .finish(result) + .await + .with_context(|| format!("failure case {mode}"))?; + } + Ok(()) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_compute_reuse_and_report_official_ric_handler_errors() -> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + environment.function().await?; + for (value, computed, counter) in [(35, 42, 1), (0, 7, 2)] { + let attempt = environment.invoke(json!({ "value": value })).await?; + let result = environment.result(attempt, Outcome::Success).await?; + ensure!( + result["computed"] == computed && result["counter"] == counter, + "incorrect computation or warm counter" + ); + ensure!( + result["requestId"] == attempt.to_string(), + "incorrect RIC attempt metadata" + ); + ensure!( + result["uid"] == 1000 && result["rootReadOnly"] == true, + "function user/rootfs limits differ" + ); + ensure!( + result["memoryMax"] == "201326592" && result["pidsMax"] == "256", + "function cgroup limits differ" + ); + } + let attempt = environment.invoke(json!({ "fail": true })).await?; + let error = environment.result(attempt, Outcome::FunctionError).await?; + ensure!( + error["errorType"] == "ValueError", + "handler error was not preserved" + ); + environment.remove_function().await?; + environment.close().await + } + .await; + environment.finish(result).await +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_cancel_a_running_official_ric_and_remove_both_containers() -> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + environment.function().await?; + let attempt = environment.invoke(json!({ "sleep": true })).await?; + environment.wait_busy(attempt).await?; + for _ in 0..2 { + environment.send(Message::Cancel { attempt }).await?; + ensure!( + environment.next().await? == Message::Ack(Acknowledgement::Cancel(attempt)), + "missing matching cancel acknowledgement" + ); + } + let running = environment + .docker + .checked(&[ + "inspect", + "--format", + "{{.State.Running}}", + &environment.names.function, + ]) + .await?; + ensure!( + running.trim() == "true", + "Cancel Ack must not claim external process termination" + ); + environment.remove_function().await?; + environment.close().await + } + .await; + environment.finish(result).await +} + +fn json_invocation(event: &Value) -> Result { + Ok(Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(60), + "arn:aws:lambda:us-east-1:000000000000:function:spike:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_drive_real_ric_through_host_client_and_join_its_streams() -> Result<()> { + for cancel in [false, true] { + let mut environment = Environment::new().await?; + let mut owner = None; + let result = async { + environment.spawn("600")?; + let config = ClientConfig::new( + environment.identity, + environment.docker.architecture, + Duration::from_secs(5), + )?; + let control = environment.control.as_mut().context("relay control")?; + owner = Some(ControlClient::start( + config, + control.output.take().context("client reader")?, + control.input.take().context("client writer")?, + )?); + let client = owner.as_mut().context("client owner")?; + ensure!( + client.next(Instant::now() + OPERATION).await? == ClientEvent::RelayReady, + "client relay handshake failed" + ); + environment.create_function().await?; + ensure!( + client.next(Instant::now() + OPERATION).await? == ClientEvent::BootstrapReady, + "client bootstrap handshake failed" + ); + for (value, computed, counter) in [(35, 42, 1), (0, 7, 2)] { + let call = json_invocation(&json!({"value": value}))?; + let expected = call.parts().0; + client.invoke(call, Instant::now() + OPERATION).await?; + match client.next(Instant::now() + OPERATION).await? { + ClientEvent::InvocationResult { + attempt, + result: RuntimeResult::Success(payload), + } if attempt == expected => { + let result: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + result["computed"] == computed + && result["counter"] == counter + && result["requestId"] == expected.to_string(), + "client computation/warm metadata mismatch" + ); + } + _ => bail!("unexpected client computation event"), + } + ensure!( + client.is_idle(), + "client did not restore Idle after success" + ); + } + let event = if cancel { + json!({"sleep": true}) + } else { + json!({"fail": true}) + }; + let call = json_invocation(&event)?; + let expected = call.parts().0; + client.invoke(call, Instant::now() + OPERATION).await?; + if cancel { + environment.wait_busy(expected).await?; + client.cancel(expected, Instant::now() + OPERATION).await?; + ensure!( + client.next(Instant::now() + OPERATION).await? + == ClientEvent::Cancelled(expected), + "client cancellation acknowledgement mismatch" + ); + } else { + match client.next(Instant::now() + OPERATION).await? { + ClientEvent::InvocationResult { + attempt, + result: RuntimeResult::FunctionError(payload), + } if attempt == expected => { + let result: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + result["errorType"] == "ValueError", + "client function error mismatch" + ); + } + _ => bail!("unexpected client handler error event"), + } + } + ensure!(!client.is_idle(), "failed/cancelled client is reusable"); + environment.remove_function().await?; + client.shutdown(Instant::now() + OPERATION).await?; + environment.wait(true).await + } + .await; + // Always join streams, remove/reconcile containers and reap the CLI, + // including failures after streams have moved out of Control. + let client_cleanup = if let Some(client) = &mut owner { + client + .shutdown(Instant::now() + OPERATION) + .await + .map_err(anyhow::Error::from) + } else { + Ok(()) + }; + let cleanup = environment.cleanup().await; + result.and(client_cleanup).and(cleanup)?; + } + Ok(()) +} diff --git a/crates/rustack-s3-http/src/upload.rs b/crates/rustack-s3-http/src/upload.rs index 29bed563..72d46abd 100644 --- a/crates/rustack-s3-http/src/upload.rs +++ b/crates/rustack-s3-http/src/upload.rs @@ -583,7 +583,7 @@ mod tests { ] { let provider = StaticCredentialProvider::new(vec![("key".to_owned(), "secret".to_owned())]); - let key = derive_signing_key("secret", "20260101", "us-east-1", "s3"); + let key = derive_signing_key("secret", "20260101", "us-east-1", "s3").unwrap(); let canonical = rustack_auth::canonical::build_canonical_request( "PUT", "/bucket/key", @@ -599,7 +599,8 @@ mod tests { "20260101/us-east-1/s3/aws4_request", &rustack_auth::hash_payload(canonical.as_bytes()), ), - ); + ) + .unwrap(); let make_parts = || { let mut request = parts(marker); if marker.ends_with("-TRAILER") { @@ -633,6 +634,7 @@ mod tests { rustack_auth::hash_payload(data) ), ) + .unwrap() }; let first = sign(&seed, b"hello"); let terminal = sign(&first, b""); @@ -644,7 +646,8 @@ mod tests { aws4_request\n{terminal}\n{}", rustack_auth::hash_payload(b"x-amz-checksum-crc32:NhCmhg==\n") ), - ); + ) + .unwrap(); format!("x-amz-checksum-crc32:NhCmhg==\r\nx-amz-trailer-signature:{signature}\r\n") } else { String::new() diff --git a/crates/rustack-sqs-http/src/service.rs b/crates/rustack-sqs-http/src/service.rs index fc77b306..dccd0bd0 100644 --- a/crates/rustack-sqs-http/src/service.rs +++ b/crates/rustack-sqs-http/src/service.rs @@ -248,9 +248,10 @@ mod tests { &rustack_auth::hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key("secret", "20260101", "us-east-1", "sqs"), + &derive_signing_key("secret", "20260101", "us-east-1", "sqs").unwrap(), &text, - ); + ) + .unwrap(); request.headers_mut().insert( "authorization", format!( diff --git a/crates/rustack-stepfunctions-core/Cargo.toml b/crates/rustack-stepfunctions-core/Cargo.toml new file mode 100644 index 00000000..01d58fe2 --- /dev/null +++ b/crates/rustack-stepfunctions-core/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "rustack-stepfunctions-core" +description = "Compiled ASL semantics and supervised Step Functions execution for Rustack" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +rustack-core = { workspace = true } +rustack-stepfunctions-model = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tracing = { workspace = true } +bytes = { workspace = true } +chrono = { workspace = true } +getrandom = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +regex = { workspace = true } +sha2 = { workspace = true } +subtle = { workspace = true } +secrecy = { workspace = true } +hex = { workspace = true } +winnow = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +ciborium = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } +rstest = { workspace = true } +proptest = { workspace = true } diff --git a/crates/rustack-stepfunctions-core/src/callback_parameters.rs b/crates/rustack-stepfunctions-core/src/callback_parameters.rs new file mode 100644 index 00000000..2dd936cc --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/callback_parameters.rs @@ -0,0 +1,82 @@ +//! Distinct outbound/history projections for one callback parameter evaluation. + +use std::fmt; + +use crate::Payload; + +/// Bounded history-only Parameters; no unchecked public constructor or serializer. +#[derive(Clone)] +#[non_exhaustive] +pub struct CallbackHistory { + pub(crate) payload: Payload, + pub(crate) redacted: bool, +} +impl fmt::Debug for CallbackHistory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("CallbackHistory()") + } +} +impl CallbackHistory { + /// Read the prepared history JSON, with sensitive expression values null. + /// + /// ``` + /// # fn journal(h:&rustack_stepfunctions_core::CallbackHistory){let safe=h.payload();} + /// ``` + #[must_use] + pub fn payload(&self) -> &Payload { + &self.payload + } + /// Whether local callback provenance removed any sensitive values. + /// + /// `TaskScheduled` has no AWS truncated metadata field; do not invent one. + /// + /// ``` + /// # fn inspect(h:&rustack_stepfunctions_core::CallbackHistory){let redacted=h.redacted();} + /// ``` + #[must_use] + pub fn redacted(&self) -> bool { + self.redacted + } +} +/// Actual invocation Parameters and a separately typed safe history projection. +/// +/// The combined object deliberately cannot be serialized into history: +/// +/// ```compile_fail +/// # fn persist(value:rustack_stepfunctions_core::CallbackParameters) { +/// let unsafe_history = serde_json::to_string(&value); +/// # } +/// ``` +#[derive(Clone)] +#[non_exhaustive] +pub struct CallbackParameters { + pub(crate) outbound: Payload, + pub(crate) history: CallbackHistory, +} +impl fmt::Debug for CallbackParameters { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("CallbackParameters()") + } +} +impl CallbackParameters { + /// Read the actual invocation JSON, including explicitly selected credential. + /// + /// Do not pass this value to the callback history adapter. + /// + /// ``` + /// # fn invoke(p:&rustack_stepfunctions_core::CallbackParameters){let actual=p.outbound();} + /// ``` + #[must_use] + pub fn outbound(&self) -> &Payload { + &self.outbound + } + /// Read the distinct projection reserved for journal/persistence consumers. + /// + /// ``` + /// # fn journal(p:&rustack_stepfunctions_core::CallbackParameters){let safe=p.history();} + /// ``` + #[must_use] + pub fn history(&self) -> &CallbackHistory { + &self.history + } +} diff --git a/crates/rustack-stepfunctions-core/src/choice.rs b/crates/rustack-stepfunctions-core/src/choice.rs new file mode 100644 index 00000000..867a7ada --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice.rs @@ -0,0 +1,513 @@ +//! Compiled short-circuit Choice predicates with shared evaluation budgets. + +use std::{borrow::Cow, cmp::Ordering, fmt}; + +use chrono::{DateTime, Timelike}; +use serde_json::{Number, Value}; + +use crate::{ + AslError, EvaluationContext, Payload, ReferencePath, + payload::{parse_component, validate_component}, +}; + +mod glob; +use glob::Glob; + +/// Immutable compiled Choice predicate; rule transition targets are compiler-owned. +#[derive(Clone)] +#[non_exhaustive] +pub struct Predicate(Node); + +#[derive(Clone)] +enum Node { + And(Vec), + Or(Vec), + Not(Box), + Compare { + variable: ReferencePath, + comparison: Comparison, + }, +} + +#[derive(Clone)] +enum Comparison { + Test(Test, bool), + String(Order, Operand), + Numeric(Order, Operand), + Timestamp(Order, Operand), + Boolean(Operand), + Matches(Glob), +} + +#[derive(Clone)] +enum Operand { + Literal(T), + Path(ReferencePath), +} + +enum Resolved<'a, T> { + Literal(&'a T), + Value(&'a Value), +} + +enum Family { + String, + Numeric, + Timestamp, +} + +#[derive(Clone, Copy)] +enum Test { + Present, + Null, + String, + Numeric, + Boolean, + Timestamp, +} + +#[derive(Clone, Copy)] +enum Order { + Equal, + Less, + Greater, + LessEqual, + GreaterEqual, +} + +#[derive(Clone)] +struct Timestamp { + seconds: i64, + fraction: String, + source_bytes: usize, +} + +impl Timestamp { + fn compare(&self, other: &Self) -> Ordering { + self.seconds + .cmp(&other.seconds) + .then_with(|| self.fraction.cmp(&other.fraction)) + } +} + +impl Order { + fn accepts(self, ordering: Ordering) -> bool { + match self { + Self::Equal => ordering.is_eq(), + Self::Less => ordering.is_lt(), + Self::Greater => ordering.is_gt(), + Self::LessEqual => !ordering.is_gt(), + Self::GreaterEqual => !ordering.is_lt(), + } + } +} + +impl fmt::Debug for Predicate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Predicate()") + } +} + +impl Predicate { + /// Preflight source JSON before compiling a predicate without its Next field. + /// + /// # Errors + /// Rejects duplicate/unknown fields, invalid operators and structural limits. + /// + /// ``` + /// use rustack_stepfunctions_core::Predicate; + /// let predicate = Predicate::parse(br#"{"Variable":"$.n","NumericEquals":3}"#)?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn parse(input: &[u8]) -> Result { + let value = parse_component(input)?; + Self::compile(&value) + } + + /// Compile an existing predicate tree, borrowing before validation/cloning. + /// + /// # Errors + /// Rejects invalid predicates, more than1,024 nodes or32 predicate levels. + /// + /// ``` + /// use rustack_stepfunctions_core::Predicate; + /// let predicate = Predicate::compile(&serde_json::json!({"Variable":"$.x","IsPresent":true}))?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn compile(value: &Value) -> Result { + validate_component(value)?; + Self::node(value, 0, &mut 1024) + } + + /// Evaluate left-to-right against validated data and internal context. + /// + /// # Errors + /// Missing references (except `IsPresent`), unsupported numeric ranges or + /// shared work/string-byte exhaustion fail explicitly, without coercion. + /// + /// ``` + /// use rustack_stepfunctions_core::{Predicate, Payload, EvaluationContext}; + /// let data = Payload::parse(br#"{"n":3}"#)?; + /// let predicate = Predicate::compile(&serde_json::json!({"Variable":"$.n","NumericEquals":3}))?; + /// assert!(predicate.evaluate(&data,&EvaluationContext::from(&data))?); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn evaluate(&self, data: &Payload, context: &EvaluationContext) -> Result { + self.eval( + data, + context, + &mut Budget { + units: 10_000, + bytes: 1024 * 1024, + }, + ) + } + + pub(crate) fn node_count(&self) -> usize { + match &self.0 { + Node::And(children) | Node::Or(children) => children + .iter() + .fold(1_usize, |n, p| n.saturating_add(p.node_count())), + Node::Not(child) => 1_usize.saturating_add(child.node_count()), + Node::Compare { .. } => 1, + } + } + + fn node(value: &Value, depth: usize, remaining: &mut usize) -> Result { + if depth >= 32 { + return Err(AslError::InvalidPredicate); + } + *remaining = remaining.checked_sub(1).ok_or(AslError::InvalidPredicate)?; + let object = value.as_object().ok_or(AslError::InvalidPredicate)?; + for (key, is_and) in [("And", true), ("Or", false)] { + if let Some(children) = object.get(key) { + if object.len() != 1 { + return Err(AslError::InvalidPredicate); + } + let children = children + .as_array() + .filter(|v| !v.is_empty() && v.len() <= 128) + .ok_or(AslError::InvalidPredicate)?; + let compiled = children + .iter() + .map(|v| Self::node(v, depth.saturating_add(1), remaining)) + .collect::, _>>()?; + return Ok(Self(if is_and { + Node::And(compiled) + } else { + Node::Or(compiled) + })); + } + } + if let Some(child) = object.get("Not") { + if object.len() != 1 { + return Err(AslError::InvalidPredicate); + } + return Ok(Self(Node::Not(Box::new(Self::node( + child, + depth.saturating_add(1), + remaining, + )?)))); + } + if object.len() != 2 { + return Err(AslError::InvalidPredicate); + } + let variable = object + .get("Variable") + .and_then(Value::as_str) + .ok_or(AslError::InvalidPredicate)? + .parse()?; + let (operator, value) = object + .iter() + .find(|(k, _)| k.as_str() != "Variable") + .ok_or(AslError::InvalidPredicate)?; + Ok(Self(Node::Compare { + variable, + comparison: comparison(operator, value)?, + })) + } + + fn eval( + &self, + data: &Payload, + context: &EvaluationContext, + budget: &mut Budget, + ) -> Result { + budget.step()?; + match &self.0 { + Node::And(children) => { + for child in children { + if !child.eval(data, context, budget)? { + return Ok(false); + } + } + Ok(true) + } + Node::Or(children) => { + for child in children { + if child.eval(data, context, budget)? { + return Ok(true); + } + } + Ok(false) + } + Node::Not(child) => Ok(!child.eval(data, context, budget)?), + Node::Compare { + variable, + comparison, + } => { + let value = variable.lookup(data, context); + if let Comparison::Test(Test::Present, expected) = comparison { + return Ok(value.is_some() == *expected); + } + evaluate( + comparison, + value.ok_or(AslError::Missing)?, + data, + context, + budget, + ) + } + } + } +} + +fn comparison(operator: &str, value: &Value) -> Result { + let test = match operator { + "IsPresent" => Some(Test::Present), + "IsNull" => Some(Test::Null), + "IsString" => Some(Test::String), + "IsNumeric" => Some(Test::Numeric), + "IsBoolean" => Some(Test::Boolean), + "IsTimestamp" => Some(Test::Timestamp), + _ => None, + }; + if let Some(test) = test { + return Ok(Comparison::Test( + test, + value.as_bool().ok_or(AslError::InvalidPredicate)?, + )); + } + if operator == "StringMatches" { + return Ok(Comparison::Matches(Glob::compile( + value.as_str().ok_or(AslError::InvalidPredicate)?, + )?)); + } + let (name, path) = operator + .strip_suffix("Path") + .map_or((operator, false), |s| (s, true)); + if name == "BooleanEquals" { + return Ok(Comparison::Boolean(operand(value, path, Value::as_bool)?)); + } + let (family, suffix) = if let Some(s) = name.strip_prefix("String") { + (Family::String, s) + } else if let Some(s) = name.strip_prefix("Numeric") { + (Family::Numeric, s) + } else if let Some(s) = name.strip_prefix("Timestamp") { + (Family::Timestamp, s) + } else { + return Err(AslError::InvalidPredicate); + }; + let order = match suffix { + "Equals" => Order::Equal, + "LessThan" => Order::Less, + "GreaterThan" => Order::Greater, + "LessThanEquals" => Order::LessEqual, + "GreaterThanEquals" => Order::GreaterEqual, + _ => return Err(AslError::InvalidPredicate), + }; + match family { + Family::String => Ok(Comparison::String( + order, + operand(value, path, |v| v.as_str().map(str::to_owned))?, + )), + Family::Numeric => Ok(Comparison::Numeric( + order, + operand(value, path, |v| number(v).ok().flatten())?, + )), + Family::Timestamp => Ok(Comparison::Timestamp( + order, + operand(value, path, |v| v.as_str().and_then(timestamp))?, + )), + } +} + +fn operand( + value: &Value, + path: bool, + literal: impl FnOnce(&Value) -> Option, +) -> Result, AslError> { + if path { + Ok(Operand::Path( + value.as_str().ok_or(AslError::InvalidPredicate)?.parse()?, + )) + } else { + Ok(Operand::Literal( + literal(value).ok_or(AslError::InvalidPredicate)?, + )) + } +} + +fn resolve<'a, T>( + operand: &'a Operand, + data: &'a Payload, + context: &'a EvaluationContext, +) -> Result, AslError> { + match operand { + Operand::Literal(value) => Ok(Resolved::Literal(value)), + Operand::Path(path) => Ok(Resolved::Value( + path.lookup(data, context).ok_or(AslError::Missing)?, + )), + } +} + +pub(crate) fn number(value: &Value) -> Result, AslError> { + const LIMIT: f64 = 9_007_199_254_740_992.0; + if let Some(n) = value.as_u64() { + if n > 9_007_199_254_740_992 { + return Err(AslError::NumericRange); + } + } + if let Some(n) = value.as_i64() { + if !(-9_007_199_254_740_992..=9_007_199_254_740_992).contains(&n) { + return Err(AslError::NumericRange); + } + } + let result = value.as_f64(); + let floating = value.as_number().is_some_and(Number::is_f64); + if result.is_some_and(|n| !n.is_finite() || n.abs() > LIMIT || (floating && n.abs() == LIMIT)) { + return Err(AslError::NumericRange); + } + Ok(result) +} + +fn timestamp(value: &str) -> Option { + let date = DateTime::parse_from_rfc3339(value).ok()?; + if date.nanosecond() >= 1_000_000_000 { + None + } else { + let fraction = if let Some(dot) = value.find('.') { + let tail = value.get(dot.checked_add(1)?..)?; + let end = tail + .find(|ch: char| !ch.is_ascii_digit()) + .unwrap_or(tail.len()); + tail.get(..end)?.trim_end_matches('0') + } else { + "" + }; + Some(Timestamp { + seconds: date.timestamp(), + fraction: fraction.to_owned(), + source_bytes: value.len(), + }) + } +} + +fn evaluate( + comparison: &Comparison, + value: &Value, + data: &Payload, + context: &EvaluationContext, + budget: &mut Budget, +) -> Result { + match comparison { + Comparison::Test(test, expected) => { + let actual = match test { + Test::Present => true, + Test::Null => value.is_null(), + Test::String => value.is_string(), + Test::Numeric => value.is_number(), + Test::Boolean => value.is_boolean(), + Test::Timestamp => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text).is_some() + } else { + false + } + } + }; + Ok(actual == *expected) + } + Comparison::Boolean(operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(*value), + Resolved::Value(value) => value.as_bool(), + }; + Ok(value.as_bool().zip(expected).is_some_and(|(a, b)| a == b)) + } + Comparison::Numeric(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(*value), + Resolved::Value(value) => number(value)?, + }; + Ok(number(value)? + .zip(expected) + .and_then(|(a, b)| a.partial_cmp(&b)) + .is_some_and(|ordering| order.accepts(ordering))) + } + Comparison::String(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(value.as_str()), + Resolved::Value(value) => value.as_str(), + }; + if let Some((a, b)) = value.as_str().zip(expected) { + budget.text(a.len().saturating_add(b.len()))?; + Ok(order.accepts(a.cmp(b))) + } else { + Ok(false) + } + } + Comparison::Timestamp(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => { + budget.text(value.source_bytes)?; + Some(Cow::Borrowed(value)) + } + Resolved::Value(value) => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text).map(Cow::Owned) + } else { + None + } + } + }; + let actual = if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text) + } else { + None + }; + Ok(actual + .zip(expected) + .is_some_and(|(a, b)| order.accepts(a.compare(&b)))) + } + Comparison::Matches(glob) => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + glob.matches(text, budget) + } else { + Ok(false) + } + } + } +} + +struct Budget { + units: usize, + bytes: usize, +} +impl Budget { + fn step(&mut self) -> Result<(), AslError> { + self.units = self.units.checked_sub(1).ok_or(AslError::WorkLimit)?; + Ok(()) + } + fn text(&mut self, bytes: usize) -> Result<(), AslError> { + self.bytes = self.bytes.checked_sub(bytes).ok_or(AslError::WorkLimit)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-core/src/choice/glob.rs b/crates/rustack-stepfunctions-core/src/choice/glob.rs new file mode 100644 index 00000000..59711678 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice/glob.rs @@ -0,0 +1,85 @@ +//! Bounded Unicode glob, independent of user regular expressions. + +use super::Budget; +use crate::AslError; + +#[derive(Clone)] +pub(super) struct Glob(Vec); +#[derive(Clone, Copy)] +enum Token { + Literal(char), + Star, +} + +impl Glob { + pub(super) fn compile(pattern: &str) -> Result { + if pattern.len() > 4096 { + return Err(AslError::InvalidPredicate); + } + let mut tokens = Vec::new(); + let mut chars = pattern.chars(); + while let Some(ch) = chars.next() { + let token = match ch { + '*' => { + if matches!(tokens.last(), Some(Token::Star)) { + continue; + } + Token::Star + } + '\\' => match chars.next() { + Some(ch @ ('*' | '\\')) => Token::Literal(ch), + _ => return Err(AslError::InvalidPredicate), + }, + ch => Token::Literal(ch), + }; + if tokens.len() >= 1024 { + return Err(AslError::InvalidPredicate); + } + tokens.push(token); + } + Ok(Self(tokens)) + } + + pub(super) fn matches(&self, text: &str, budget: &mut Budget) -> Result { + let mut position = 0_usize; + let mut token = 0_usize; + let mut star = None; + while let Some(ch) = text.get(position..).and_then(|s| s.chars().next()) { + budget.step()?; + match self.0.get(token) { + Some(Token::Literal(expected)) if *expected == ch => { + position = position + .checked_add(ch.len_utf8()) + .ok_or(AslError::WorkLimit)?; + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + } + Some(Token::Star) => { + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + if token == self.0.len() { + return Ok(true); + } + star = Some((token, position)); + } + _ => { + let Some((after, backup)) = star else { + return Ok(false); + }; + let Some(next) = text.get(backup..).and_then(|s| s.chars().next()) else { + return Ok(false); + }; + let advanced = backup + .checked_add(next.len_utf8()) + .ok_or(AslError::WorkLimit)?; + position = advanced; + token = after; + star = Some((after, advanced)); + } + } + } + while matches!(self.0.get(token), Some(Token::Star)) { + budget.step()?; + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + } + Ok(token == self.0.len()) + } +} diff --git a/crates/rustack-stepfunctions-core/src/choice/tests.rs b/crates/rustack-stepfunctions-core/src/choice/tests.rs new file mode 100644 index 00000000..a8364c6f --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice/tests.rs @@ -0,0 +1,456 @@ +use std::error::Error; + +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::json::JsonError; +use serde_json::{Value, json}; + +use crate::{AslError, EvaluationContext, Payload, Predicate}; + +type TestResult = Result<(), Box>; + +fn evaluate(rule: &Value, data: &Value) -> Result { + let predicate = Predicate::compile(rule)?; + let data = Payload::from_value(data)?; + predicate.evaluate(&data, &EvaluationContext::from(&data)) +} + +#[test] +fn test_should_execute_all_independent_choice_catalog_examples() -> TestResult { + let catalog: Value = serde_json::from_str(include_str!( + "../../../../tests/workflows/semantic-cases.json" + ))?; + let mut count = 0; + for case in catalog["cases"].as_array().ok_or("missing cases")? { + if case["kind"] != "choice" { + continue; + } + assert_eq!( + evaluate(&case["rule"], &case["input"].clone())?, + case["expected"].as_bool().ok_or("missing expected")? + ); + count += 1; + } + assert_eq!(count, 4); + Ok(()) +} + +#[test] +fn test_should_cover_all_ordered_literal_and_path_operator_variants() -> TestResult { + let families = [ + ("String", vec![json!("a"), json!("b"), json!("c")]), + ("Numeric", vec![json!(-1), json!(0), json!(1)]), + ( + "Timestamp", + vec![ + json!("2020-01-01T00:00:00Z"), + json!("2020-01-02T00:00:00Z"), + json!("2020-01-03T00:00:00Z"), + ], + ), + ]; + let operators = [ + ("Equals", [false, true, false]), + ("LessThan", [true, false, false]), + ("GreaterThan", [false, false, true]), + ("LessThanEquals", [true, true, false]), + ("GreaterThanEquals", [false, true, true]), + ]; + for (family, values) in families { + let middle = values.get(1).ok_or("missing middle")?; + for (suffix, expected) in operators { + for path in [false, true] { + for (index, left) in values.iter().enumerate() { + let op = format!("{family}{suffix}{}", if path { "Path" } else { "" }); + let operand = if path { + json!("$.right") + } else { + middle.clone() + }; + let rule = json!({"Variable":"$.left",op:operand}); + assert_eq!( + evaluate(&rule, &json!({"left":left,"right":middle}))?, + *expected.get(index).ok_or("missing expected")?, + "{family} {suffix} {path} {index}" + ); + } + } + } + } + for path in [false, true] { + for value in [false, true] { + let op = if path { + "BooleanEqualsPath" + } else { + "BooleanEquals" + }; + let operand = if path { json!("$.right") } else { json!(true) }; + assert_eq!( + evaluate( + &json!({"Variable":"$.left",op:operand}), + &json!({"left":value,"right":true}) + )?, + value + ); + } + } + Ok(()) +} + +#[rstest] +#[case("IsPresent",json!(null),true)] +#[case("IsNull",json!(null),true)] +#[case("IsString",json!("7"),true)] +#[case("IsNumeric",json!(7),true)] +#[case("IsBoolean",json!(true),true)] +#[case("IsTimestamp",json!("2020-01-01T00:00:00Z"),true)] +#[case("IsNull",json!(0),false)] +#[case("IsString",json!(7),false)] +#[case("IsNumeric",json!("7"),false)] +#[case("IsBoolean",json!("true"),false)] +#[case("IsTimestamp",json!("invalid"),false)] +#[case("IsTimestamp",json!("2016-12-31T23:59:60Z"),false)] +fn test_should_apply_type_tests_without_coercion( + #[case] op: &str, + #[case] value: Value, + #[case] expected: bool, +) -> TestResult { + for requested in [true, false] { + assert_eq!( + evaluate(&json!({"Variable":"$.x",op:requested}), &json!({"x":value}))?, + expected == requested + ); + } + Ok(()) +} + +#[test] +fn test_should_short_circuit_in_definition_order_and_preserve_missing_errors() -> TestResult { + let present = json!({"Variable":"$.absent","IsPresent":true}); + let missing = json!({"Variable":"$.absent","IsString":true}); + assert!(!evaluate(&json!({"And":[present,missing]}), &json!({}))?); + assert!(evaluate( + &json!({"Or":[{"Not":present},missing]}), + &json!({}) + )?); + assert!(matches!( + evaluate(&json!({"And":[missing,present]}), &json!({})), + Err(AslError::Missing) + )); + for op in [ + "IsNull", + "IsString", + "IsNumeric", + "IsBoolean", + "IsTimestamp", + ] { + assert!(matches!( + evaluate(&json!({"Variable":"$.absent",op:false}), &json!({})), + Err(AslError::Missing) + )); + } + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","StringEqualsPath":"$.absent"}), + &json!({"x":1}) + ), + Err(AslError::Missing) + )); + Ok(()) +} + +#[test] +fn test_should_compare_instants_and_avoid_numeric_precision_aliasing() -> TestResult { + assert!(!evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.1234567891Z"}), + &json!({"x":"2020-01-01T00:00:00.1234567892Z"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.1234567891000Z"}), + &json!({"x":"2020-01-01T01:00:00.1234567891+01:00"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.123Z"}), + &json!({"x":"2020-01-01T01:00:00.123+01:00"}) + )?); + assert!(!evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00Z"}), + &json!({"x":"2020-01-01"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","NumericEquals":9_007_199_254_740_992_u64}), + &json!({"x":9_007_199_254_740_992_u64}) + )?); + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","NumericEquals":9_007_199_254_740_992_u64}), + &json!({"x":9_007_199_254_740_993_u64}) + ), + Err(AslError::NumericRange) + )); + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","NumericEqualsPath":"$.right"}), + &json!({"x":0,"right":-9_007_199_254_740_993_i64}) + ), + Err(AslError::NumericRange) + )); + assert!(evaluate( + &json!({"Variable":"$.x","IsNumeric":true}), + &json!({"x":u64::MAX}) + )?); + for (op, operand, left) in [ + ("StringEquals", json!("1"), json!(1)), + ("NumericEquals", json!(1), json!("1")), + ("BooleanEquals", json!(true), json!(1)), + ] { + assert!(!evaluate( + &json!({"Variable":"$.x",op:operand}), + &json!({"x":left}) + )?); + } + let predicate = + Predicate::compile(&json!({"Variable":"$$.State.Name","StringEquals":"current"}))?; + assert!(predicate.evaluate( + &Payload::parse(b"null")?, + &EvaluationContext::from_value(&json!({"State":{"Name":"current"}}))? + )?); + Ok(()) +} + +#[test] +fn test_should_reject_raw_float_boundary_lexemes_instead_of_aliasing_large_numbers() -> TestResult { + let integer = Predicate::parse(br#"{"Variable":"$.x","NumericEquals":9007199254740992}"#)?; + for text in [ + "9007199254740993.0", + "9007199254740993e0", + "9007199254740992.0", + "-9007199254740993.0", + "-9007199254740993e0", + ] { + let data = Payload::parse(format!("{{\"x\":{text}}}").as_bytes())?; + assert!( + matches!( + integer.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::NumericRange) + ), + "{text}" + ); + assert!( + matches!( + Predicate::parse( + format!("{{\"Variable\":\"$.x\",\"NumericEquals\":{text}}}").as_bytes() + ), + Err(AslError::InvalidPredicate) + ), + "{text}" + ); + } + let inside = Predicate::parse(br#"{"Variable":"$.x","NumericEquals":9007199254740991.0}"#)?; + let data = Payload::parse(br#"{"x":9007199254740991e0}"#)?; + assert!(inside.evaluate(&data, &EvaluationContext::from(&data))?); + Ok(()) +} + +#[test] +fn test_should_charge_precompiled_timestamp_literal_bytes_to_shared_budget() -> TestResult { + let actual = format!("2020-01-01T00:00:00.{}Z", "0".repeat(180_000)); + let expected = format!("2020-01-01T00:00:00.{}1Z", "0".repeat(180_000)); + let comparison = json!({"Variable":"$.x","TimestampLessThan":expected}); + let data = Payload::from_value(&json!({"x":actual}))?; + let two = Predicate::compile(&json!({"And":[comparison,comparison]}))?; + assert!(two.evaluate(&data, &EvaluationContext::from(&data))?); + let three = Predicate::compile(&json!({"And":[comparison,comparison,comparison]}))?; + assert!(matches!( + three.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::WorkLimit) + )); + let timestamp = format!("2020-01-01T00:00:00.{}1Z", "0".repeat(220_000)); + let comparison = json!({"Variable":"$.x","TimestampLessThan":timestamp}); + let rule = json!({"And":[comparison,comparison,comparison,comparison,comparison]}); + assert!(Predicate::compile(&rule).is_err()); + // A shared Path operand supplies the same long fraction repeatedly without + // exceeding the separately bounded source definition. + let comparison = json!({"Variable":"$.x","TimestampLessThanPath":"$.right"}); + let data = Payload::from_value(&json!({"x":"2020-01-01T00:00:00Z","right":timestamp}))?; + let rule = Predicate::compile( + &json!({"And":[comparison,comparison,comparison,comparison,comparison]}), + )?; + assert!(matches!( + rule.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +#[rstest] +#[case("", "", true)] +#[case("*", "", true)] +#[case("**", "中🙂", true)] +#[case("中*文", "中🙂文", true)] +#[case("a\\*b", "a*b", true)] +#[case("a\\\\b", "a\\b", true)] +#[case("a?b", "a?b", true)] +#[case("a?b", "axb", false)] +#[case("*aba*b", "ababab", true)] +#[case("a*b*c", "abcc", true)] +#[case("a*b*c", "abdcx", false)] +#[case("literal", "Literal", false)] +fn test_should_match_compiled_unicode_globs( + #[case] pattern: &str, + #[case] input: &str, + #[case] expected: bool, +) -> TestResult { + assert_eq!( + evaluate( + &json!({"Variable":"$.x","StringMatches":pattern}), + &json!({"x":input}) + )?, + expected + ); + Ok(()) +} + +#[rstest] +#[case(json!({}))] +#[case(json!([]))] +#[case(json!({"And":[]}))] +#[case(json!({"Or":[]}))] +#[case(json!({"And":[],"Not":{}}))] +#[case(json!({"Variable":"$.x","IsNull":"true"}))] +#[case(json!({"Variable":"$.x","NumericEquals":true}))] +#[case(json!({"Variable":"$.x","NumericEquals":u64::MAX}))] +#[case(json!({"Variable":"$.x","TimestampEquals":"invalid"}))] +#[case(json!({"Variable":"$.x","StringEquals":7}))] +#[case(json!({"Variable":"$.x","StringMatches":"\\q"}))] +#[case(json!({"Variable":"$.x","StringMatches":"\\"}))] +#[case(json!({"Variable":"$.x","IsNull":true,"IsString":false}))] +#[case(json!({"Variable":"$.x","Unknown":true}))] +#[case(json!({"Variable":"$.x","StringMatchesPath":"$.y"}))] +fn test_should_reject_ambiguous_invalid_or_unknown_predicates(#[case] value: Value) { + assert!(matches!( + Predicate::compile(&value), + Err(AslError::InvalidPredicate) + )); +} + +#[test] +fn test_should_bound_source_structure_compiled_nodes_depth_and_glob_tokens() -> TestResult { + assert!(matches!( + Predicate::parse(br#"{"Variable":"$.a","IsNull":true,"IsNull":false}"#), + Err(AslError::Json(JsonError::Duplicate)) + )); + assert!(Predicate::compile(&json!({"Variable":"$[*]","IsNull":true})).is_err()); + assert!(Predicate::compile(&json!({"Variable":"$.x","StringEqualsPath":"$[*]"})).is_err()); + let leaf = json!({"Variable":"$.x","IsPresent":true}); + let mut nested = leaf.clone(); + for _ in 0..31 { + nested = json!({"Not":nested}); + } + Predicate::compile(&nested)?; + assert!(matches!( + Predicate::compile(&json!({"Not":nested})), + Err(AslError::InvalidPredicate) + )); + Predicate::compile(&json!({"And":vec![leaf.clone();128]}))?; + assert!(matches!( + Predicate::compile(&json!({"And":vec![leaf.clone();129]})), + Err(AslError::InvalidPredicate) + )); + let group = json!({"And":vec![leaf;8]}); + assert!(matches!( + Predicate::compile(&json!({"And":vec![group;128]})), + Err(AslError::InvalidPredicate) + )); + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"🙂".repeat(1024)}))?; + assert!(matches!( + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"a".repeat(1025)})), + Err(AslError::InvalidPredicate) + )); + assert!(matches!( + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"🙂".repeat(1025)})), + Err(AslError::InvalidPredicate) + )); + Ok(()) +} + +#[test] +fn test_should_share_step_and_string_byte_limits_across_the_complete_predicate() -> TestResult { + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","StringMatches":"*b"}), + &json!({"x":"a".repeat(20_000)}) + ), + Err(AslError::WorkLimit) + )); + assert!(evaluate( + &json!({"Variable":"$.x","StringMatches":"a*"}), + &json!({"x":"a".repeat(20_000)}) + )?); + let comparison = json!({"Variable":"$.x","StringEqualsPath":"$.x"}); + let input = json!({"x":"a".repeat(260_000)}); + assert!(evaluate( + &json!({"And":[comparison,comparison]}), + &input.clone() + )?); + assert!(matches!( + evaluate(&json!({"And":[comparison,comparison,comparison]}), &input), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +#[test] +fn test_should_reject_borrowed_extreme_predicate_without_recursive_drop_and_redact() -> TestResult { + let mut tree = Value::Null; + for _ in 0..50_000 { + tree = Value::Array(vec![tree]); + } + let rejected = Predicate::compile(&tree); + let mut owned = vec![tree]; + while let Some(value) = owned.pop() { + if let Value::Array(mut children) = value { + owned.append(&mut children); + } + } + assert!(matches!(rejected, Err(AslError::Json(JsonError::Depth)))); + let predicate = Predicate::compile(&json!({"Variable":"$.secret","StringEquals":"private"}))?; + assert_eq!(format!("{predicate:?}"), "Predicate()"); + Ok(()) +} + +fn glob_oracle(pattern: &str, text: &str) -> bool { + let chars: Vec<_> = text.chars().collect(); + let mut row = vec![false; chars.len() + 1]; + row[0] = true; + for ch in pattern.chars() { + let mut next = vec![false; row.len()]; + if ch == '*' { + next[0] = row[0]; + } + for (i, input) in chars.iter().enumerate() { + next[i + 1] = if ch == '*' { + next[i] || row[i + 1] + } else { + row[i] && ch == *input + }; + } + row = next; + } + row.last().copied().unwrap_or(false) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + #[test] + fn test_should_match_an_independent_glob_dynamic_program(pattern in "[ab*]{0,12}", text in "[ab]{0,18}") { + let actual=evaluate(&json!({"Variable":"$.x","StringMatches":pattern}),&json!({"x":text})).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(actual,glob_oracle(&pattern,&text)); + } + + #[test] + fn test_should_preserve_exact_integer_order_inside_the_supported_range(a in -1_000_000_i64..1_000_000, b in -1_000_000_i64..1_000_000) { + let actual=evaluate(&json!({"Variable":"$.a","NumericLessThanPath":"$.b"}),&json!({"a":a,"b":b})).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(actual,a) -> fmt::Result { + f.write_str("Diagnostic()") + } +} +impl Diagnostic { + read!(Diagnostic,severity,DiagnosticSeverity,s=>s.severity,"Read severity."); + read!(Diagnostic,code,&'static str,s=>s.code,"Read a stable Rustack diagnostic code."); + read!(Diagnostic,message,&'static str,s=>s.message,"Read the static bounded message."); + read!(Diagnostic,location,&str,s=>s.location.as_str(),"Read the bounded RFC6901 pointer; empty means root."); +} + +#[derive(Error)] +enum Source { + #[error("invalid definition JSON")] + Json(#[from] JsonError), + #[error("invalid compiled component")] + Asl(#[from] AslError), +} +impl fmt::Debug for Source { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// Failed compilation with one diagnostic and an optional typed source. +#[derive(Error)] +#[error("ASL definition compilation failed")] +#[non_exhaustive] +pub struct CompileError { + diagnostic: Diagnostic, + #[source] + source: Option, +} +impl fmt::Debug for CompileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl CompileError { + read!(CompileError,diagnostic,&Diagnostic,s=>&s.diagnostic,"Read the diagnostic without formatting its source."); + pub(super) fn json(location: &Location, error: JsonError) -> Self { + let limited = matches!( + error, + JsonError::Bytes + | JsonError::Depth + | JsonError::Members + | JsonError::String + | JsonError::Key + ); + let mut failure = if limited { + location.error( + Code::LimitExceeded, + "definition limits: 1MiB, 64 JSON levels, 65536 members, 1MiB strings, 256KiB keys", + ) + } else { + location.error(Code::InvalidJson, "invalid or duplicate definition JSON") + }; + failure.source = Some(Source::Json(error)); + failure + } + pub(super) fn asl(location: &Location, error: AslError) -> Self { + let message = match &error { + AslError::InvalidTemplate => { + "invalid template shape/collision or limit (32 containers, 4096 compiled nodes)" + } + AslError::InvalidExpression => { + "invalid expression or limit (4096 bytes, 1024 nodes, 10 calls, 128 arguments)" + } + AslError::InvalidPredicate => { + "invalid Choice predicate or limit (32 levels, 1024 nodes, 128 children)" + } + AslError::InvalidPath | AslError::InvalidReference | AslError::InvalidResultTarget => { + "invalid path or limit (1024 bytes, 128 segments)" + } + AslError::InvalidWait => { + "invalid STANDARD Wait source (integer seconds at most 31536000)" + } + AslError::Json(_) => { + "invalid component JSON (1MiB, 64 levels, 4096 members, 256KiB strings/keys)" + } + AslError::DataLimit => "compiled component exceeds its encoded JSON budget", + _ => "invalid compiled state component", + }; + let mut failure = location.error(Code::InvalidComponent, message); + failure.source = Some(Source::Asl(error)); + failure + } +} + +pub(super) struct Location(String); +impl Location { + pub(super) fn root() -> Self { + Self(String::new()) + } + pub(super) fn field(&self, token: &str) -> Result { + let extra = token.bytes().filter(|b| matches!(b, b'~' | b'/')).count(); + let length = self + .0 + .len() + .saturating_add(1) + .saturating_add(token.len()) + .saturating_add(extra); + if length > 4096 { + return Err(self.error(Code::LimitExceeded, "diagnostic location limit exceeded")); + } + let mut value = String::with_capacity(length); + value.push_str(&self.0); + value.push('/'); + value.push_str(&token.replace('~', "~0").replace('/', "~1")); + Ok(Self(value)) + } + pub(super) fn warning(&self, code: Code, message: &'static str) -> Diagnostic { + self.diagnostic(DiagnosticSeverity::Warning, code, message) + } + fn diagnostic( + &self, + severity: DiagnosticSeverity, + code: Code, + message: &'static str, + ) -> Diagnostic { + Diagnostic { + severity, + code: code.as_str(), + message, + location: self.0.clone(), + } + } + pub(super) fn error(&self, code: Code, message: &'static str) -> CompileError { + CompileError { + diagnostic: self.diagnostic(DiagnosticSeverity::Error, code, message), + source: None, + } + } +} + +#[derive(Clone, Copy)] +pub(super) enum Code { + InvalidJson, + InvalidComponent, + InvalidSchema, + MissingField, + UnsupportedField, + LimitExceeded, + InvalidStateName, + MissingTransitionTarget, + ProfileUnavailable, + DependencyDisabled, + InvalidResource, + NoReachableTerminal, + UnreachableState, +} +impl Code { + fn as_str(self) -> &'static str { + match self { + Self::InvalidJson => "RUSTACK_INVALID_JSON", + Self::InvalidComponent => "RUSTACK_INVALID_COMPONENT", + Self::InvalidSchema => "RUSTACK_INVALID_SCHEMA", + Self::MissingField => "RUSTACK_MISSING_FIELD", + Self::UnsupportedField => "RUSTACK_UNSUPPORTED_FIELD", + Self::LimitExceeded => "RUSTACK_LIMIT_EXCEEDED", + Self::InvalidStateName => "RUSTACK_INVALID_STATE_NAME", + Self::MissingTransitionTarget => "RUSTACK_MISSING_TRANSITION_TARGET", + Self::ProfileUnavailable => "RUSTACK_PROFILE_UNAVAILABLE", + Self::DependencyDisabled => "RUSTACK_DEPENDENCY_DISABLED", + Self::InvalidResource => "RUSTACK_INVALID_RESOURCE", + Self::NoReachableTerminal => "RUSTACK_NO_REACHABLE_TERMINAL", + Self::UnreachableState => "RUSTACK_UNREACHABLE_STATE", + } + } +} diff --git a/crates/rustack-stepfunctions-core/src/compiler/ir.rs b/crates/rustack-stepfunctions-core/src/compiler/ir.rs new file mode 100644 index 00000000..99fead05 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/compiler/ir.rs @@ -0,0 +1,528 @@ +//! Immutable typed state storage and checked borrowed execution views. + +use std::{ + fmt, + num::{NonZeroU8, NonZeroU32}, + sync::Arc, +}; + +use bytes::Bytes; +use serde_json::Value; + +use super::{Catcher, Diagnostic, Integration, RecoveryPolicy, TenantScope}; +use crate::{ + AslError, DataFlow, Expression, Predicate, ReferencePath, ResultPath, Template, WaitSource, +}; + +/// State index constructible only by the compiler; interpreted in its machine. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub struct StateId(pub(super) usize); + +/// Pre-resolved sequential control transfer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Transition { + /// Continue within the compiled scope. + Next(StateId), + /// Complete this machine successfully. + End, +} + +/// Prevalidated per-attempt timeout source. +#[derive(Clone)] +#[non_exhaustive] +pub enum TaskTimeout { + /// Bounded only by the execution/operator deadline. + Omitted, + /// Positive static seconds. + Seconds(NonZeroU32), + /// Singular dynamic seconds resolved on entry. + Path(ReferencePath), +} +impl fmt::Debug for TaskTimeout { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("TaskTimeout()") + } +} + +/// Normalized Inline Map concurrency, independent of task/frame capacity. +#[derive(Clone)] +#[non_exhaustive] +pub enum MapConcurrency { + /// Positive static limit; omission and zero normalize to forty. + Limit(NonZeroU8), + /// Singular reference resolved from the effective Map input on entry. + Path(ReferencePath), +} +impl fmt::Debug for MapConcurrency { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("MapConcurrency()") + } +} + +/// Failure field view preserving omission, literal text or compiled expression. +#[non_exhaustive] +#[derive(Clone, Copy)] +pub enum FailureView<'a> { + /// No value was supplied. + Omitted, + /// Bounded literal field. + Literal(&'a str), + /// Must evaluate to a bounded string on entry. + Dynamic(&'a Expression), +} +impl fmt::Debug for FailureView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("FailureView()") + } +} + +/// Catch error-object merge behavior. +#[non_exhaustive] +pub enum CatchResult<'a> { + /// Replace the failed state's raw input with its error object. + Replace, + /// Preserve its original raw input. + Discard, + /// Merge into the original input through a validated data-root reference. + Path(&'a ResultPath), +} +impl fmt::Debug for CatchResult<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("CatchResult()") + } +} + +/// Borrowed validated state instruction; execution does not interpret ASL maps. +#[non_exhaustive] +pub enum StateView<'a> { + /// Pass with a possible static result and resolved continuation. + Pass { + /// Compiled data processing. + flow: &'a DataFlow, + /// Literal result, if supplied. + result: Option<&'a Value>, + /// Pre-resolved continuation. + next: Transition, + }, + /// Actual Lambda call plan; the executor owns invocation lifetime. + Task { + /// Compiled data processing. + flow: &'a DataFlow, + /// Validated integration. + resource: &'a Integration, + /// Per-attempt timeout source. + timeout: &'a TaskTimeout, + /// Ordered recovery rules. + recovery: &'a RecoveryPolicy, + /// Pre-resolved continuation. + next: Transition, + }, + /// Independent branch machines sharing this state's effective input. + Parallel { + /// Compiled input and result processing. + flow: &'a DataFlow, + /// Branches in definition/result order, each with a lexical state index. + branches: &'a [Arc], + /// Ordered enclosing-state recovery. + recovery: &'a RecoveryPolicy, + /// Continuation in the enclosing machine. + next: Transition, + }, + /// Normalized Inline Map instruction with one reusable processor artifact. + Map { + /// Input/output/result processing; excludes legacy selector Parameters. + flow: &'a DataFlow, + /// Array source, defaulting to the effective input root. + items: &'a ReferencePath, + /// Item input template evaluated with Map.Item context, when supplied. + selector: Option<&'a Template>, + /// Immutable processor with an independent lexical index. + processor: &'a Arc, + /// Normalized local concurrency plan. + concurrency: &'a MapConcurrency, + /// Ordered enclosing-state recovery. + recovery: &'a RecoveryPolicy, + /// Continuation in the enclosing machine. + next: Transition, + }, + /// Ordered conditional branches with optional fallback. + Choice { + /// Input/output processing only. + flow: &'a DataFlow, + /// Compiled predicates and targets. + branches: &'a [(Predicate, StateId)], + /// Optional default target. + default: Option, + }, + /// Wait plan; timers remain actor-owned. + Wait { + /// Input/output processing. + flow: &'a DataFlow, + /// Compiled wait source. + source: &'a WaitSource, + /// Pre-resolved continuation. + next: Transition, + }, + /// Successful terminal with input/output transforms. + Succeed { + /// Compiled input/output processing. + flow: &'a DataFlow, + }, + /// Failed terminal with independent bounded fields. + Fail { + /// Error field. + error: FailureView<'a>, + /// Cause field. + cause: FailureView<'a>, + }, +} +impl fmt::Debug for StateView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StateView()") + } +} + +pub(super) enum Failure { + Omitted, + Literal(String), + Dynamic(Expression), +} +impl Failure { + fn view(&self) -> FailureView<'_> { + match self { + Self::Omitted => FailureView::Omitted, + Self::Literal(v) => FailureView::Literal(v), + Self::Dynamic(v) => FailureView::Dynamic(v), + } + } + fn units(&self) -> usize { + match self { + Self::Dynamic(v) => v.node_count(), + Self::Omitted | Self::Literal(_) => 1, + } + } +} + +pub(super) enum Node { + Pass { + flow: DataFlow, + result: Option>, + next: Transition, + }, + Task { + flow: DataFlow, + resource: Integration, + timeout: TaskTimeout, + recovery: RecoveryPolicy, + next: Transition, + }, + Parallel { + flow: DataFlow, + branches: Vec>, + recovery: RecoveryPolicy, + next: Transition, + }, + Map { + flow: DataFlow, + items: ReferencePath, + selector: Option