From 955267f7fd9ecd79691ea3e8bbc7372ee32b59fe Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 15:46:35 -0700 Subject: [PATCH 001/106] phase 0: specify workflows M0-M4 and validate runtime contracts Record the English workflow specification set, pinned service model and semantic provenance. Retain reproducible Docker relay/RIC probes with real results, error, cancellation, EOF and reconciliation evidence on native arm64 and emulated amd64. Workspace build, tests, formatting, Clippy and rustdoc pass. Independent review findings W07-W09 are fixed. Existing advisory failures remain recorded as B01 for P1; native x86/Desktop/rootless qualification and all production runtime/engine phases remain required. --- Cargo.toml | 1 + Makefile | 36 + codegen/services/stepfunctions.toml | 60 + codegen/smithy-model/stepfunctions.json | 8218 +++++++++++++++++ .../stepfunctions.provenance.json | 10 + docs/index.md | 3 + docs/research/spike-lambda-container-relay.md | 41 + .../spike-stepfunctions-semantic-oracle.md | 36 + ...-stepfunctions-lambda-container-runtime.md | 64 + rust-toolchain.toml | 2 +- specs/index.md | 40 +- specs/rustack-workflows-00-prd.md | 92 + specs/rustack-workflows-10-api-design.md | 123 + ...tack-workflows-20-lambda-runtime-design.md | 131 + specs/rustack-workflows-30-asl-design.md | 134 + ...ustack-workflows-40-integrations-design.md | 134 + .../rustack-workflows-60-operations-design.md | 142 + .../rustack-workflows-70-verification-plan.md | 105 + specs/rustack-workflows-80-roadmap.md | 64 + specs/rustack-workflows-90-impl-plan.md | 145 + specs/rustack-workflows-93-review.md | 42 + specs/rustack-workflows-99-decisions.md | 107 + specs/ruststack-lambda-design.md | 2 + specs/ruststack-lambda-executor-design.md | 2 + tests/workflows/evidence/botocore-wire.json | 22 + tests/workflows/evidence/relay-amd64.json | 45 + tests/workflows/evidence/relay-arm64.json | 45 + tests/workflows/provenance.json | 17 + tests/workflows/requirements.txt | 2 + tests/workflows/semantic-cases.json | 659 ++ tests/workflows/test_botocore_wire.py | 65 + tools/workflows-spikes/.dockerignore | 2 + tools/workflows-spikes/Cargo.lock | 1076 +++ tools/workflows-spikes/Cargo.toml | 27 + tools/workflows-spikes/Handler.Dockerfile | 4 + tools/workflows-spikes/Relay.Dockerfile | 5 + tools/workflows-spikes/handler.py | 27 + tools/workflows-spikes/src/bin/driver.rs | 524 ++ tools/workflows-spikes/src/bin/relay.rs | 203 + tools/workflows-spikes/src/lib.rs | 277 + 40 files changed, 12732 insertions(+), 2 deletions(-) create mode 100644 codegen/services/stepfunctions.toml create mode 100644 codegen/smithy-model/stepfunctions.json create mode 100644 codegen/smithy-model/stepfunctions.provenance.json create mode 100644 docs/research/spike-lambda-container-relay.md create mode 100644 docs/research/spike-stepfunctions-semantic-oracle.md create mode 100644 docs/research/survey-stepfunctions-lambda-container-runtime.md create mode 100644 specs/rustack-workflows-00-prd.md create mode 100644 specs/rustack-workflows-10-api-design.md create mode 100644 specs/rustack-workflows-20-lambda-runtime-design.md create mode 100644 specs/rustack-workflows-30-asl-design.md create mode 100644 specs/rustack-workflows-40-integrations-design.md create mode 100644 specs/rustack-workflows-60-operations-design.md create mode 100644 specs/rustack-workflows-70-verification-plan.md create mode 100644 specs/rustack-workflows-80-roadmap.md create mode 100644 specs/rustack-workflows-90-impl-plan.md create mode 100644 specs/rustack-workflows-93-review.md create mode 100644 specs/rustack-workflows-99-decisions.md create mode 100644 tests/workflows/evidence/botocore-wire.json create mode 100644 tests/workflows/evidence/relay-amd64.json create mode 100644 tests/workflows/evidence/relay-arm64.json create mode 100644 tests/workflows/provenance.json create mode 100644 tests/workflows/requirements.txt create mode 100644 tests/workflows/semantic-cases.json create mode 100644 tests/workflows/test_botocore_wire.py create mode 100644 tools/workflows-spikes/.dockerignore create mode 100644 tools/workflows-spikes/Cargo.lock create mode 100644 tools/workflows-spikes/Cargo.toml create mode 100644 tools/workflows-spikes/Handler.Dockerfile create mode 100644 tools/workflows-spikes/Relay.Dockerfile create mode 100644 tools/workflows-spikes/handler.py create mode 100644 tools/workflows-spikes/src/bin/driver.rs create mode 100644 tools/workflows-spikes/src/bin/relay.rs create mode 100644 tools/workflows-spikes/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index f0c41f55..2ec944a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ authors = ["Tyr Chen "] repository = "https://github.com/tyrchen/rustack" [workspace.lints.rust] +rust_2024_compatibility = "warn" missing_docs = "warn" missing_debug_implementations = "warn" diff --git a/Makefile b/Makefile index 4dc8ed10..62309eea 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,42 @@ build: @cargo build +# P0 evidence tooling is isolated from the production workspace/runtime. +WORKFLOWS_SPIKE_DIR ?= /tmp/rustack-spikes +WORKFLOWS_DOCKER_SOCKET ?= /var/run/docker.sock +WORKFLOWS_ARCH ?= $(if $(filter arm64 aarch64,$(shell uname -m)),arm64,amd64) +WORKFLOWS_IMAGE_BUILDER ?= docker buildx + +workflows-model-check: + @python3 -c 'import hashlib,json,pathlib; p=pathlib.Path("codegen/smithy-model"); m=json.loads((p/"stepfunctions.provenance.json").read_text()); raw=(p/"stepfunctions.json").read_bytes(); assert len(raw)==m["bytes"] and hashlib.sha256(raw).hexdigest()==m["sha256"], "SFN source hash mismatch"; s=json.loads(raw)["shapes"][m["service"]]; assert s["version"]=="2016-11-23" and "aws.protocols#awsJson1_0" in s["traits"] and s["traits"]["aws.auth#sigv4"]["name"]=="states"; print("Pinned SFN model verified:",len(s["operations"]),"operations")' + +workflows-fixture-check: workflows-model-check + @python3 -c 'import json,pathlib; p=pathlib.Path("tests/workflows"); f=json.loads((p/"semantic-cases.json").read_text()); m=json.loads((p/"provenance.json").read_text()); source=json.loads(pathlib.Path("codegen/smithy-model/stepfunctions.provenance.json").read_text()); ids=[c["id"] for c in f["cases"]]; assert f["schemaVersion"]==1 and len(ids)==len(set(ids)) and all(c["source"] in f["sources"] for c in f["cases"]); assert m["awsModelCommit"]==source["commit"] and m["awsModelSha256"]==source["sha256"]; print("Fixture catalog structure/provenance verified:",len(ids),"cases; interpreter execution is not claimed")' + +workflows-wire-probe: + @PYTHONDONTWRITEBYTECODE=1 uv run --no-project --python 3.13 --with-requirements tests/workflows/requirements.txt python -m unittest discover -s tests/workflows -p 'test_botocore_wire.py' -v + +workflows-codegen-spike: workflows-model-check + @cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output "$(WORKFLOWS_SPIKE_DIR)/stepfunctions-generated/src" + +test-workflows-spikes: + @cargo test --manifest-path tools/workflows-spikes/Cargo.toml + @cargo clippy --manifest-path tools/workflows-spikes/Cargo.toml --all-targets -- -D warnings + +workflows-relay-spike-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build --manifest-path tools/workflows-spikes/Cargo.toml --bin relay --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(WORKFLOWS_SPIKE_DIR)/$(WORKFLOWS_ARCH)" + +workflows-relay-spike-images: workflows-relay-spike-build + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Relay.Dockerfile --tag "rustack-p0-relay:$(WORKFLOWS_ARCH)" "$(WORKFLOWS_SPIKE_DIR)/$(WORKFLOWS_ARCH)/relay" + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Handler.Dockerfile --tag "rustack-p0-handler:$(WORKFLOWS_ARCH)" tools/workflows-spikes + +workflows-relay-spike: workflows-relay-spike-images + @cargo run --manifest-path tools/workflows-spikes/Cargo.toml --bin driver -- "$(WORKFLOWS_DOCKER_SOCKET)" "$(WORKFLOWS_ARCH)" + +.PHONY: workflows-model-check workflows-fixture-check workflows-wire-probe workflows-codegen-spike \ + test-workflows-spikes workflows-relay-spike-build workflows-relay-spike-images workflows-relay-spike + check: @cargo check --all-targets --all-features diff --git a/codegen/services/stepfunctions.toml b/codegen/services/stepfunctions.toml new file mode 100644 index 00000000..0dc0cc33 --- /dev/null +++ b/codegen/services/stepfunctions.toml @@ -0,0 +1,60 @@ +[service] +name = "stepfunctions" +display_name = "Step Functions" +rust_prefix = "StepFunctions" +namespace = "com.amazonaws.sfn" +protocol = "awsJson1_0" + +[protocol] +serde_rename = "camelCase" +target_prefix = "AWSStepFunctions" +emit_serde_derives = true + +[operations] +all = [ + "CreateActivity", + "CreateStateMachine", + "CreateStateMachineAlias", + "DeleteActivity", + "DeleteStateMachine", + "DeleteStateMachineAlias", + "DeleteStateMachineVersion", + "DescribeActivity", + "DescribeExecution", + "DescribeMapRun", + "DescribeStateMachine", + "DescribeStateMachineAlias", + "DescribeStateMachineForExecution", + "GetActivityTask", + "GetExecutionHistory", + "ListActivities", + "ListExecutions", + "ListMapRuns", + "ListStateMachineAliases", + "ListStateMachines", + "ListStateMachineVersions", + "ListTagsForResource", + "PublishStateMachineVersion", + "RedriveExecution", + "SendTaskFailure", + "SendTaskHeartbeat", + "SendTaskSuccess", + "StartExecution", + "StartSyncExecution", + "StopExecution", + "TagResource", + "TestState", + "UntagResource", + "UpdateMapRun", + "UpdateStateMachine", + "UpdateStateMachineAlias", + "ValidateStateMachineDefinition", +] + +[errors.custom] +UnknownOperationException = { status = 400, message = "Unknown Step Functions operation" } +UnsupportedOperation = { status = 400, message = "Operation is outside the supported Step Functions profile" } + +[output] +file_layout = "flat" +always_serialize_arrays = true diff --git a/codegen/smithy-model/stepfunctions.json b/codegen/smithy-model/stepfunctions.json new file mode 100644 index 00000000..7fba49cf --- /dev/null +++ b/codegen/smithy-model/stepfunctions.json @@ -0,0 +1,8218 @@ +{ + "smithy": "2.0", + "metadata": { + "suppressions": [ + { + "id": "HttpMethodSemantics", + "namespace": "*" + }, + { + "id": "HttpResponseCodeSemantics", + "namespace": "*" + }, + { + "id": "PaginatedTrait", + "namespace": "*" + }, + { + "id": "HttpHeaderTrait", + "namespace": "*" + }, + { + "id": "HttpUriConflict", + "namespace": "*" + }, + { + "id": "Service", + "namespace": "*" + } + ] + }, + "shapes": { + "com.amazonaws.sfn#AWSStepFunctions": { + "type": "service", + "version": "2016-11-23", + "operations": [ + { + "target": "com.amazonaws.sfn#CreateActivity" + }, + { + "target": "com.amazonaws.sfn#CreateStateMachine" + }, + { + "target": "com.amazonaws.sfn#CreateStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DeleteActivity" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachine" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DeleteStateMachineVersion" + }, + { + "target": "com.amazonaws.sfn#DescribeActivity" + }, + { + "target": "com.amazonaws.sfn#DescribeExecution" + }, + { + "target": "com.amazonaws.sfn#DescribeMapRun" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachine" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecution" + }, + { + "target": "com.amazonaws.sfn#GetActivityTask" + }, + { + "target": "com.amazonaws.sfn#GetExecutionHistory" + }, + { + "target": "com.amazonaws.sfn#ListActivities" + }, + { + "target": "com.amazonaws.sfn#ListExecutions" + }, + { + "target": "com.amazonaws.sfn#ListMapRuns" + }, + { + "target": "com.amazonaws.sfn#ListStateMachineAliases" + }, + { + "target": "com.amazonaws.sfn#ListStateMachines" + }, + { + "target": "com.amazonaws.sfn#ListStateMachineVersions" + }, + { + "target": "com.amazonaws.sfn#ListTagsForResource" + }, + { + "target": "com.amazonaws.sfn#PublishStateMachineVersion" + }, + { + "target": "com.amazonaws.sfn#RedriveExecution" + }, + { + "target": "com.amazonaws.sfn#SendTaskFailure" + }, + { + "target": "com.amazonaws.sfn#SendTaskHeartbeat" + }, + { + "target": "com.amazonaws.sfn#SendTaskSuccess" + }, + { + "target": "com.amazonaws.sfn#StartExecution" + }, + { + "target": "com.amazonaws.sfn#StartSyncExecution" + }, + { + "target": "com.amazonaws.sfn#StopExecution" + }, + { + "target": "com.amazonaws.sfn#TagResource" + }, + { + "target": "com.amazonaws.sfn#TestState" + }, + { + "target": "com.amazonaws.sfn#UntagResource" + }, + { + "target": "com.amazonaws.sfn#UpdateMapRun" + }, + { + "target": "com.amazonaws.sfn#UpdateStateMachine" + }, + { + "target": "com.amazonaws.sfn#UpdateStateMachineAlias" + }, + { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinition" + } + ], + "traits": { + "aws.api#service": { + "sdkId": "SFN", + "arnNamespace": "states", + "cloudFormationName": "StepFunctions", + "cloudTrailEventSource": "states.amazonaws.com", + "docId": "states-2016-11-23", + "endpointPrefix": "states" + }, + "aws.auth#sigv4": { + "name": "states" + }, + "aws.protocols#awsJson1_0": {}, + "smithy.api#documentation": "Step Functions\n

With Step Functions, you can create workflows, also called state machines, to build distributed applications, automate processes, orchestrate microservices, and create data and machine learning pipelines.

\n

Through the Step Functions API, you can create, list, update, and delete state machines, activities, and other data types. You can start, stop, and redrive your state machines. Your activity workers can send task success, heartbeat, and failure responses.

\n

With API calls, you can also manage other aspects of your workflow, such as tags, versions, and aliases.

\n

For more information about developing solutions with Step Functions, see the \n Step Functions Developer Guide\n .

\n \n

If you use the Step Functions API actions using Amazon Web Services SDK integrations, make sure the API actions are in camel case and parameter names are in Pascal case. For example, you might use Step Functions API action startSyncExecution and specify its parameter as StateMachineArn.

\n ", + "smithy.api#title": "AWS Step Functions", + "smithy.api#xmlNamespace": { + "uri": "http://swf.amazonaws.com/doc/2015-07-20/" + }, + "smithy.rules#endpointBdd": { + "version": "1.1", + "parameters": { + "UseDualStack": { + "builtIn": "AWS::UseDualStack", + "required": true, + "default": false, + "documentation": "When true, use the dual-stack endpoint. If the configured endpoint does not support dual-stack, dispatching the request MAY return an error.", + "type": "boolean" + }, + "UseFIPS": { + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "When true, send this request to the FIPS-compliant regional endpoint. If the configured endpoint does not have a FIPS compliant endpoint, dispatching the request will return an error.", + "type": "boolean" + }, + "Endpoint": { + "builtIn": "SDK::Endpoint", + "required": false, + "documentation": "Override the endpoint used to send this request", + "type": "string" + }, + "Region": { + "builtIn": "AWS::Region", + "required": false, + "documentation": "The AWS region used to dispatch the request.", + "type": "string" + } + }, + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Endpoint" + } + ] + }, + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + }, + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + }, + { + "fn": "stringEquals", + "argv": [ + { + "ref": "Region" + }, + "us-gov-west-1" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + }, + true + ] + }, + { + "fn": "stringEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "name" + ] + }, + "aws-us-gov" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + }, + true + ] + } + ], + "results": [ + { + "conditions": [], + "error": "Invalid Configuration: FIPS and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": { + "ref": "Endpoint" + }, + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "FIPS and DualStack are enabled, but this partition does not support one or both", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "FIPS is enabled but this partition does not support FIPS", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "DualStack is enabled but this partition does not support DualStack", + "type": "error" + }, + { + "conditions": [], + "error": "Invalid Configuration: Missing Region", + "type": "error" + } + ], + "root": 2, + "nodeCount": 15, + "nodes": "/////wAAAAH/////AAAAAAAAAA4AAAADAAAAAQAAAAQF9eELAAAAAgAAAAUF9eELAAAAAwAAAAkAAAAGAAAABAAAAAcF9eEEAAAABwX14QUAAAAIAAAACAX14QkF9eEKAAAABAAAAAwAAAAKAAAABQX14QQAAAALAAAABgX14QcF9eEIAAAABgAAAA0F9eEGAAAACAX14QUF9eEGAAAAAwX14QEAAAAPAAAABAX14QIF9eED" + }, + "smithy.rules#endpointRuleSet": { + "version": "1.0", + "parameters": { + "UseDualStack": { + "builtIn": "AWS::UseDualStack", + "required": true, + "default": false, + "documentation": "When true, use the dual-stack endpoint. If the configured endpoint does not support dual-stack, dispatching the request MAY return an error.", + "type": "boolean" + }, + "UseFIPS": { + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "When true, send this request to the FIPS-compliant regional endpoint. If the configured endpoint does not have a FIPS compliant endpoint, dispatching the request will return an error.", + "type": "boolean" + }, + "Endpoint": { + "builtIn": "SDK::Endpoint", + "required": false, + "documentation": "Override the endpoint used to send this request", + "type": "string" + }, + "Region": { + "builtIn": "AWS::Region", + "required": false, + "documentation": "The AWS region used to dispatch the request.", + "type": "string" + } + }, + "rules": [ + { + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Endpoint" + } + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + } + ], + "error": "Invalid Configuration: FIPS and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported", + "type": "error" + }, + { + "conditions": [], + "endpoint": { + "url": { + "ref": "Endpoint" + }, + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + } + ], + "type": "tree" + }, + { + "conditions": [], + "rules": [ + { + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "stringEquals", + "argv": [ + { + "ref": "Region" + }, + "us-gov-west-1" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + false + ] + } + ], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [ + { + "fn": "stringEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "name" + ] + }, + "aws-us-gov" + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + false + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + } + ] + }, + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + } + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "FIPS and DualStack are enabled, but this partition does not support one or both", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + false + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsFIPS" + ] + }, + true + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states-fips.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "FIPS is enabled but this partition does not support FIPS", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + false + ] + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseDualStack" + }, + true + ] + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + true, + { + "fn": "getAttr", + "argv": [ + { + "ref": "PartitionResult" + }, + "supportsDualStack" + ] + } + ] + } + ], + "rules": [ + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dualStackDnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "DualStack is enabled but this partition does not support DualStack", + "type": "error" + } + ], + "type": "tree" + }, + { + "conditions": [], + "endpoint": { + "url": "https://states.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + } + ], + "type": "tree" + } + ], + "type": "tree" + }, + { + "conditions": [], + "error": "Invalid Configuration: Missing Region", + "type": "error" + } + ], + "type": "tree" + } + ] + }, + "smithy.rules#endpointTests": { + "testCases": [ + { + "documentation": "For custom endpoint with region not set and fips disabled", + "expect": { + "endpoint": { + "url": "https://example.com" + } + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": false + } + }, + { + "documentation": "For custom endpoint with fips enabled", + "expect": { + "error": "Invalid Configuration: FIPS and custom endpoint are not supported" + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": true + } + }, + { + "documentation": "For custom endpoint with fips disabled and dualstack enabled", + "expect": { + "error": "Invalid Configuration: Dualstack and custom endpoint are not supported" + }, + "params": { + "Endpoint": "https://example.com", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-gov-west-1.amazonaws.com" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-east-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-east-1.api.aws" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region us-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-east-1.amazonaws.com" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-east-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states.us-east-1.api.aws" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-east-1.amazonaws.com" + } + }, + "params": { + "Region": "us-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.cn-northwest-1.api.amazonwebservices.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.cn-northwest-1.amazonaws.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states.cn-northwest-1.api.amazonwebservices.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region cn-northwest-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.cn-northwest-1.amazonaws.com.cn" + } + }, + "params": { + "Region": "cn-northwest-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region eusc-de-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.eusc-de-east-1.amazonaws.eu" + } + }, + "params": { + "Region": "eusc-de-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region eusc-de-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.eusc-de-east-1.amazonaws.eu" + } + }, + "params": { + "Region": "eusc-de-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-iso-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-iso-east-1.c2s.ic.gov" + } + }, + "params": { + "Region": "us-iso-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-iso-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-iso-east-1.c2s.ic.gov" + } + }, + "params": { + "Region": "us-iso-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isob-east-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-isob-east-1.sc2s.sgov.gov" + } + }, + "params": { + "Region": "us-isob-east-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isob-east-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-isob-east-1.sc2s.sgov.gov" + } + }, + "params": { + "Region": "us-isob-east-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region eu-isoe-west-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.eu-isoe-west-1.cloud.adc-e.uk" + } + }, + "params": { + "Region": "eu-isoe-west-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region eu-isoe-west-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.eu-isoe-west-1.cloud.adc-e.uk" + } + }, + "params": { + "Region": "eu-isoe-west-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isof-south-1 with FIPS enabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-isof-south-1.csp.hci.ic.gov" + } + }, + "params": { + "Region": "us-isof-south-1", + "UseFIPS": true, + "UseDualStack": false + } + }, + { + "documentation": "For region us-isof-south-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-isof-south-1.csp.hci.ic.gov" + } + }, + "params": { + "Region": "us-isof-south-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS enabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-gov-west-1.api.aws" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": true, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS disabled and DualStack enabled", + "expect": { + "endpoint": { + "url": "https://states-fips.us-gov-west-1.api.aws" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": false, + "UseDualStack": true + } + }, + { + "documentation": "For region us-gov-west-1 with FIPS disabled and DualStack disabled", + "expect": { + "endpoint": { + "url": "https://states.us-gov-west-1.amazonaws.com" + } + }, + "params": { + "Region": "us-gov-west-1", + "UseFIPS": false, + "UseDualStack": false + } + }, + { + "documentation": "Missing region", + "expect": { + "error": "Invalid Configuration: Missing Region" + } + } + ], + "version": "1.0" + } + } + }, + "com.amazonaws.sfn#ActivityAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Activity already exists. EncryptionConfiguration may not be updated.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified activity does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity that failed during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of activities has been reached. Existing activities must be deleted\n before a new activity can be created.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ActivityList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ActivityListItem" + } + }, + "com.amazonaws.sfn#ActivityListItem": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the activity.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity.

" + } + }, + "com.amazonaws.sfn#ActivityScheduleFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity schedule failure that occurred during an\n execution.

" + } + }, + "com.amazonaws.sfn#ActivityScheduledEventDetails": { + "type": "structure", + "members": { + "resource": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the scheduled activity.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the activity task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the activity task.

" + } + }, + "heartbeatInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration between two heartbeats for the activity task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityStartedEventDetails": { + "type": "structure", + "members": { + "workerName": { + "target": "com.amazonaws.sfn#Identity", + "traits": { + "smithy.api#documentation": "

The name of the worker that the task is assigned to. These names are provided by the\n workers when calling GetActivityTask.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of an activity during an execution.

" + } + }, + "com.amazonaws.sfn#ActivitySucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the activity task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity that successfully terminated during an\n execution.

" + } + }, + "com.amazonaws.sfn#ActivityTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an activity timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#ActivityWorkerLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of workers concurrently polling for activity tasks has been\n reached.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#AliasDescription": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#Arn": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 256 + } + } + }, + "com.amazonaws.sfn#AssignedVariables": { + "type": "map", + "key": { + "target": "com.amazonaws.sfn#VariableName" + }, + "value": { + "target": "com.amazonaws.sfn#VariableValue" + } + }, + "com.amazonaws.sfn#AssignedVariablesDetails": { + "type": "structure", + "members": { + "truncated": { + "target": "com.amazonaws.sfn#truncated", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether assigned variables were truncated in the response. Always false\n for API calls. In CloudWatch logs, the value will be true if the data is truncated due to size limits.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about assigned variables in an execution history event.

" + } + }, + "com.amazonaws.sfn#BilledDuration": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#BilledMemoryUsed": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#BillingDetails": { + "type": "structure", + "members": { + "billedMemoryUsedInMB": { + "target": "com.amazonaws.sfn#BilledMemoryUsed", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Billed memory consumption of your workflow, in MB.

" + } + }, + "billedDurationInMilliseconds": { + "target": "com.amazonaws.sfn#BilledDuration", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Billed duration of your workflow, in milliseconds.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

An object that describes workflow billing details.

" + } + }, + "com.amazonaws.sfn#CharacterRestrictedName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + }, + "smithy.api#pattern": "^(?=.*[a-zA-Z_\\-\\.])[a-zA-Z0-9_\\-\\.]+$" + } + }, + "com.amazonaws.sfn#ClientToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 64 + }, + "smithy.api#pattern": "^[!-~]+$" + } + }, + "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails": { + "type": "structure", + "members": { + "included": { + "target": "com.amazonaws.sfn#includedDetails", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether input or output was included in the response. Always true\n for API calls.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about execution input or output.

" + } + }, + "com.amazonaws.sfn#CloudWatchLogsLogGroup": { + "type": "structure", + "members": { + "logGroupArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The ARN of the the CloudWatch log group to which you want your logs emitted to. The ARN\n must end with :*\n

" + } + } + }, + "traits": { + "smithy.api#documentation": "

" + } + }, + "com.amazonaws.sfn#ConflictException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Updating or deleting a resource can cause an inconsistent state. This error occurs when there're concurrent requests for DeleteStateMachineVersion, PublishStateMachineVersion, or UpdateStateMachine with the publish parameter set to true.

\n

HTTP Status Code: 409

", + "smithy.api#error": "client", + "smithy.api#httpError": 409 + } + }, + "com.amazonaws.sfn#ConnectorParameters": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 262144 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#CreateActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#ActivityLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + } + ], + "traits": { + "smithy.api#documentation": "

Creates an activity. An activity is a task that you write in any programming language and\n host on any machine that has access to Step Functions. Activities must poll Step Functions using the\n GetActivityTask API action and respond using SendTask* API\n actions. This function lets Step Functions know the existence of your activity and returns an\n identifier for use in a state machine and when polling from the activity.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n \n \n

\n CreateActivity is an idempotent API. Subsequent requests won\u2019t create a\n duplicate resource if it was already created. CreateActivity's idempotency\n check is based on the activity name. If a following request has different\n tags values, Step Functions will ignore these differences and treat it as an\n idempotent request of the previous. In this case, tags will not be updated,\n even if they are different.

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#CreateActivityInput": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity to create. This name must be unique for your Amazon Web Services account and region.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

The list of tags to add to a resource.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateActivityOutput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created activity.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#CreateStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidLoggingConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#InvalidTracingConfiguration" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates a state machine. A state machine consists of a collection of states that can do\n work (Task states), determine to which states to transition next\n (Choice states), stop an execution with an error (Fail states),\n and so on. State machines are specified using a JSON-based, structured language. For more\n information, see Amazon States\n Language in the Step Functions User Guide.

\n

If you set the publish parameter of this API action to true, it\n publishes version 1 as the first revision of the state machine.

\n

\n For additional control over security, you can encrypt your data using a customer-managed key for Step Functions state machines. You can configure a symmetric KMS key and data key reuse period when creating or updating a State Machine. The execution history and state machine definition will be encrypted with the key applied to the State Machine.\n

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n \n \n

\n CreateStateMachine is an idempotent API. Subsequent requests won\u2019t create a\n duplicate resource if it was already created. CreateStateMachine's idempotency\n check is based on the state machine name, definition,\n type, LoggingConfiguration,\n TracingConfiguration, and EncryptionConfiguration The check is also based on the publish and versionDescription parameters. If a following request has a different\n roleArn or tags, Step Functions will ignore these differences and treat\n it as an idempotent request of the previous. In this case, roleArn and\n tags will not be updated, even if they are different.

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#CreateStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#CreateStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates an alias for a state machine that points to one or two versions of the same state machine. You can set your application to call StartExecution with an alias and update the version the alias uses without changing the client's code.

\n

You can also map an alias to split StartExecution requests between two\n versions of a state machine. To do this, add a second RoutingConfig object in the\n routingConfiguration parameter. You must also specify the percentage of\n execution run requests each version should receive in both RoutingConfig objects.\n Step Functions randomly chooses which version runs a given execution based on the\n percentage you specify.

\n

To create an alias that points to a single version, specify a single\n RoutingConfig object with a weight set to 100.

\n

You can create up to 100 aliases for each state machine. You must delete unused aliases using the DeleteStateMachineAlias API action.

\n

\n CreateStateMachineAlias is an idempotent API. Step Functions bases the\n idempotency check on the stateMachineArn, description,\n name, and routingConfiguration parameters. Requests that contain\n the same values for these parameters return a successful idempotent response without creating\n a duplicate resource.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#CreateStateMachineAliasInput": { + "type": "structure", + "members": { + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description for the state machine alias.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#CharacterRestrictedName", + "traits": { + "smithy.api#documentation": "

The name of the state machine alias.

\n

To avoid conflict with version ARNs, don't use an integer in the name of the alias.

", + "smithy.api#required": {} + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of a state machine alias. The routing configuration shifts\n execution traffic between two state machine versions. routingConfiguration\n contains an array of RoutingConfig objects that specify up to two state machine\n versions. Step Functions then randomly choses which version to run an execution with based\n on the weight assigned to each RoutingConfig.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineAliasOutput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine alias.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineInput": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role to use for this state machine.

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

Determines whether a Standard or Express state machine is created. The default is\n STANDARD. You cannot update the type of a state machine once it\n has been created.

" + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration", + "traits": { + "smithy.api#documentation": "

Defines what execution history events are logged and where they are logged.

\n \n

By default, the level is set to OFF. For more information see\n Log\n Levels in the Step Functions User Guide.

\n " + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

Tags to be added when creating a state machine.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "publish": { + "target": "com.amazonaws.sfn#Publish", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Set to true to publish the first version of the state machine during creation. The default is false.

" + } + }, + "versionDescription": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

Sets description about the state machine version. You can only set the description if the publish parameter is set to true. Otherwise, if you set versionDescription, but publish to false, this API action throws ValidationException.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#CreateStateMachineOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

", + "smithy.api#required": {} + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the created state machine version. If you do not set the publish parameter to true, this field returns null value.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#Definition": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 1048576 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#DeleteActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes an activity.

" + } + }, + "com.amazonaws.sfn#DeleteActivityInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteActivityOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine. This is an asynchronous operation. It sets the state machine's\n status to DELETING and begins the deletion process. A state machine is deleted only when all its executions are completed. On the next state transition, the state machine's executions are terminated.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n \n

This API action also deletes all versions and aliases associated with a state machine.

\n \n

For EXPRESS state machines, the deletion happens eventually (usually in\n less than a minute). Running executions may emit logs after DeleteStateMachine\n API is called.

\n " + } + }, + "com.amazonaws.sfn#DeleteStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine alias.

\n

After you delete a state machine alias, you can't use it to start executions. When you\n delete a state machine alias, Step Functions doesn't delete the state machine versions\n that alias references.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DeleteStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineAliasOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersion": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DeleteStateMachineVersionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DeleteStateMachineVersionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Deletes a state machine version. After\n you delete a version, you can't call StartExecution using that version's ARN\n or use the version with a state machine alias.

\n \n

Deleting a state machine version won't terminate its in-progress executions.

\n \n \n

You can't delete a state machine version currently referenced by one or more aliases. Before you delete a version, you must either delete the aliases or update them to point to another state machine version.

\n \n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersionInput": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version to delete.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DeleteStateMachineVersionOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeActivity": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeActivityInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeActivityOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + } + ], + "traits": { + "smithy.api#documentation": "

Describes an activity.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n " + } + }, + "com.amazonaws.sfn#DescribeActivityInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to describe.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeActivityOutput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the activity.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the activity.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the activity is created.

", + "smithy.api#required": {} + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings for configured server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine execution, such as the state machine associated with the execution, the execution input and output, and relevant execution metadata. If you've redriven an execution, you can use this API action to return information about the redrives of that execution. In addition, you can use this API action to return the Map Run Amazon Resource Name (ARN) if the execution was dispatched by a Map Run.

\n

If you specify a version or alias ARN when you call the StartExecution\n API action, DescribeExecution returns that ARN.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n \n

Executions of an EXPRESS state machine aren't supported by DescribeExecution unless a Map Run dispatched them.

" + } + }, + "com.amazonaws.sfn#DescribeExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to describe.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call DescribeStateMachine API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the executed stated machine.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

" + } + }, + "status": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution ended, the date the execution stopped.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

\n \n

This field is set only if the execution succeeds. If the execution fails, this field is\n null.

\n " + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

The X-Ray trace header that was passed to the execution.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n " + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run, which dispatched this execution.

" + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error string if the state machine execution failed.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

The cause string if the state machine execution failed.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version associated with the execution. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

\n

If you start an execution from a StartExecution request without specifying a\n state machine version or alias ARN, Step Functions returns a null value.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias associated with the execution. The alias ARN is a combination of state machine ARN and the alias name separated by a colon (:). For example, stateMachineARN:PROD.

\n

If you start an execution from a StartExecution request with a\n state machine version ARN, this field will be null.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is only updated if you successfully redrive an execution.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven. If you have not yet redriven an execution, the redriveDate is null.

\n

The redriveDate is unavailable if you redrive a Map Run that starts child workflow executions of type EXPRESS.

" + } + }, + "redriveStatus": { + "target": "com.amazonaws.sfn#ExecutionRedriveStatus", + "traits": { + "smithy.api#documentation": "

Indicates whether or not an execution can be redriven at a given point in time.

\n " + } + }, + "redriveStatusReason": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

When redriveStatus is NOT_REDRIVABLE, redriveStatusReason specifies the reason why an execution cannot be redriven.

\n " + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeMapRun": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeMapRunInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeMapRunOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a Map Run's configuration, progress, and results. If you've redriven a Map Run, this API action also returns information about the redrives of that Map Run. For more information, see Examining Map Run in the Step Functions Developer Guide.

" + } + }, + "com.amazonaws.sfn#DescribeMapRunInput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeMapRunOutput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a Map Run.

", + "smithy.api#required": {} + } + }, + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution in which the Map Run was started.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#MapRunStatus", + "traits": { + "smithy.api#documentation": "

The current status of the Map Run.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date when the Map Run was started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date when the Map Run was stopped.

" + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#MaxConcurrency", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of child workflow executions configured to run in parallel for the Map Run at the same time.

", + "smithy.api#required": {} + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#ToleratedFailurePercentage", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum percentage of failed child workflow executions before the Map Run fails.

", + "smithy.api#required": {} + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#ToleratedFailureCount", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of failed child workflow executions before the Map Run fails.

", + "smithy.api#required": {} + } + }, + "itemCounts": { + "target": "com.amazonaws.sfn#MapRunItemCounts", + "traits": { + "smithy.api#documentation": "

A JSON object that contains information about the total number of items, and the item count for each processing status, such as pending and failed.

", + "smithy.api#required": {} + } + }, + "executionCounts": { + "target": "com.amazonaws.sfn#MapRunExecutionCounts", + "traits": { + "smithy.api#documentation": "

A JSON object that contains information about the total number of child workflow executions for the Map Run, and the count of child workflow executions for each status, such as failed and succeeded.

", + "smithy.api#required": {} + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven a Map Run. If you have not yet redriven a Map Run, the redriveCount is 0. This count is only updated if you successfully redrive a Map Run.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date a Map Run was last redriven. If you have not yet redriven a Map Run, the redriveDate is null.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine's definition, its IAM role Amazon Resource Name (ARN), and configuration.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n \n

This API action returns the details for a state machine version if the\n stateMachineArn you specify is a state machine version ARN.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n " + } + }, + "com.amazonaws.sfn#DescribeStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Returns details about a state machine alias.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#DescribeStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineAliasOutput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine alias.

" + } + }, + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description of the alias.

" + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of the alias.

" + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was created.

" + } + }, + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine alias was last updated.

\n

For a newly created state machine, this is the same as the creation date.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#DescribeStateMachineForExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Provides information about a state machine's definition, its execution role ARN, and\n configuration. If a Map Run dispatched the execution, this action returns the Map Run\n Amazon Resource Name (ARN) in the response. The state machine returned is the state machine associated with the\n Map Run.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n \n

This API action is not supported by EXPRESS state machines.

" + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution you want state machine information for.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineForExecutionOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine associated with the execution.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine associated with the execution.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role of the State Machine for the execution.

", + "smithy.api#required": {} + } + }, + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine associated with an execution was updated. For a newly\n created state machine, this is the creation date.

", + "smithy.api#required": {} + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration" + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run that started the child workflow execution. This field is returned only if the executionArn is a child workflow execution that was started by a Distributed Map state.

" + } + }, + "label": { + "target": "com.amazonaws.sfn#MapRunLabel", + "traits": { + "smithy.api#documentation": "

A user-defined or an auto-generated string that identifies a Map state. This field is returned only if the executionArn is a child workflow execution that was started by a Distributed Map state.

" + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the state machine. The first revision ID when you create the state machine is null.

\n

Use the state machine revisionId parameter to compare the revision of a state machine with the configuration of the state machine used for executions without performing a diff of the properties, such as definition and roleArn.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + }, + "variableReferences": { + "target": "com.amazonaws.sfn#VariableReferences", + "traits": { + "smithy.api#documentation": "

A map of state name to a list of variables referenced by that state. States that do not use variable references will not be shown in the response.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine for which you want the information.

\n

If you specify a state machine version ARN, this API returns details about that version. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

\n \n

\n When calling a labelled ARN for an encrypted state machine, the includedData = METADATA_ONLY parameter will not apply because Step Functions needs to decrypt the entire state machine definition to get the Distributed Map state\u2019s definition. In this case, the API caller needs to have kms:Decrypt permission.\n

\n " + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#DescribeStateMachineOutput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

\n

If you specified a state machine version ARN in your request, the API returns the version ARN. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#StateMachineStatus", + "traits": { + "smithy.api#documentation": "

The current status of the state machine.

" + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

\n

If called with includedData = METADATA_ONLY, the returned definition will be {}.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role used when creating this state machine. (The IAM role\n maintains security by granting Step Functions access to Amazon Web Services resources.)

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

The type of the state machine (STANDARD or\n EXPRESS).

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

\n

For a state machine version, creationDate is the date the version was created.

", + "smithy.api#required": {} + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration" + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "label": { + "target": "com.amazonaws.sfn#MapRunLabel", + "traits": { + "smithy.api#documentation": "

A user-defined or an auto-generated string that identifies a Map state. This parameter is present only if the stateMachineArn specified in input is a qualified state machine ARN.

" + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the state machine.

\n

Use the revisionId parameter to compare between versions of a state machine\n configuration used for executions without performing a diff of the properties, such as\n definition and roleArn.

" + } + }, + "description": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

The description of the state machine version.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + }, + "variableReferences": { + "target": "com.amazonaws.sfn#VariableReferences", + "traits": { + "smithy.api#documentation": "

A map of state name to a list of variables referenced by that state. States that do not use variable references will not be shown in the response.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#Enabled": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#EncryptionConfiguration": { + "type": "structure", + "members": { + "kmsKeyId": { + "target": "com.amazonaws.sfn#KmsKeyId", + "traits": { + "smithy.api#documentation": "

An alias, alias ARN, key ID, or key ARN of a symmetric encryption KMS key to encrypt data. To specify a KMS key in a different Amazon Web Services account, you must use the key ARN or alias ARN.

" + } + }, + "kmsDataKeyReusePeriodSeconds": { + "target": "com.amazonaws.sfn#KmsDataKeyReusePeriodSeconds", + "traits": { + "smithy.api#documentation": "

Maximum duration that Step Functions will reuse data keys. When the period expires, Step Functions will call GenerateDataKey. Only applies to customer managed keys.

" + } + }, + "type": { + "target": "com.amazonaws.sfn#EncryptionType", + "traits": { + "smithy.api#documentation": "

Encryption type

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

\n

\n For additional control over security, you can encrypt your data using a customer-managed key for Step Functions state machines and activities. You can configure a symmetric KMS key and data key reuse period when creating or updating a State Machine, and when creating an Activity. The execution history and state machine definition will be encrypted with the key applied to the State Machine. Activity inputs will be encrypted with the key applied to the Activity.\n

\n \n

Step Functions automatically enables encryption at rest using Amazon Web Services owned keys at no charge. However, KMS charges apply when using a customer managed key. For more information about pricing, see Key Management Service pricing.

\n \n

For more information on KMS, see What is Key Management Service?\n

" + } + }, + "com.amazonaws.sfn#EncryptionType": { + "type": "enum", + "members": { + "AWS_OWNED_KEY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "AWS_OWNED_KEY" + } + }, + "CUSTOMER_MANAGED_KMS_KEY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CUSTOMER_MANAGED_KMS_KEY" + } + } + } + }, + "com.amazonaws.sfn#ErrorMessage": { + "type": "string" + }, + "com.amazonaws.sfn#EvaluationFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + }, + "location": { + "target": "com.amazonaws.sfn#EvaluationFailureLocation", + "traits": { + "smithy.api#documentation": "

The location of the field in the state in which the evaluation error occurred.

" + } + }, + "state": { + "target": "com.amazonaws.sfn#StateName", + "traits": { + "smithy.api#documentation": "

The name of the state in which the evaluation error occurred.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an evaluation failure that occurred while processing a state, for example, when a JSONata expression throws an error. This event will only be present in state machines that have QueryLanguage set to JSONata, or individual states set to JSONata.

" + } + }, + "com.amazonaws.sfn#EvaluationFailureLocation": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#EventId": { + "type": "long", + "traits": { + "smithy.api#default": 0 + } + }, + "com.amazonaws.sfn#ExceptionHandlerIndex": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ExecutionAbortedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an abort of an execution.

" + } + }, + "com.amazonaws.sfn#ExecutionAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The execution has the same name as another execution (but a different\n input).

\n \n

Executions with the same name and input are considered\n idempotent.

\n ", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified execution does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an execution failure event.

" + } + }, + "com.amazonaws.sfn#ExecutionLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of running executions has been reached. Running executions must end or\n be stopped before a new execution can be started.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ExecutionListItem" + } + }, + "com.amazonaws.sfn#ExecutionListItem": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine that ran the execution.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution already ended, the date the execution stopped.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run. This field is returned only if mapRunArn was specified in the ListExecutions API action. If stateMachineArn was specified in ListExecutions, the mapRunArn isn't returned.

" + } + }, + "itemCount": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The total number of items processed in a child workflow execution. This field is returned only if mapRunArn was specified in the ListExecutions API action. If stateMachineArn was specified in ListExecutions, the itemCount field isn't returned.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine version associated with the execution.

\n

If the state machine execution was started with an unqualified ARN, it returns null.

\n

If the execution was started using a stateMachineAliasArn, both the stateMachineAliasArn and stateMachineVersionArn parameters contain the respective values.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias used to start an execution.

\n

If the state machine execution was started with an unqualified ARN or a version ARN, it returns null.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is only updated when you successfully redrive an execution.

" + } + }, + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an execution.

" + } + }, + "com.amazonaws.sfn#ExecutionNotRedrivable": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The execution Amazon Resource Name (ARN) that you specified for executionArn cannot be redriven.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#ExecutionRedriveFilter": { + "type": "enum", + "members": { + "REDRIVEN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVEN" + } + }, + "NOT_REDRIVEN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NOT_REDRIVEN" + } + } + } + }, + "com.amazonaws.sfn#ExecutionRedriveStatus": { + "type": "enum", + "members": { + "REDRIVABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVABLE" + } + }, + "NOT_REDRIVABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NOT_REDRIVABLE" + } + }, + "REDRIVABLE_BY_MAP_RUN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "REDRIVABLE_BY_MAP_RUN" + } + } + } + }, + "com.amazonaws.sfn#ExecutionRedrivenEventDetails": { + "type": "structure", + "members": { + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times you've redriven an execution. If you have not yet redriven an execution, the redriveCount is 0. This count is not updated for redrives that failed to start or are pending to be redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a redriven execution.

" + } + }, + "com.amazonaws.sfn#ExecutionStartedEventDetails": { + "type": "structure", + "members": { + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role used for executing Lambda tasks.

" + } + }, + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine alias used for starting the state machine execution.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine version used for starting the state machine execution.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of the execution.

" + } + }, + "com.amazonaws.sfn#ExecutionStatus": { + "type": "enum", + "members": { + "RUNNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RUNNING" + } + }, + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "TIMED_OUT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TIMED_OUT" + } + }, + "ABORTED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ABORTED" + } + }, + "PENDING_REDRIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_REDRIVE" + } + } + } + }, + "com.amazonaws.sfn#ExecutionSucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the successful termination of the execution.

" + } + }, + "com.amazonaws.sfn#ExecutionTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the execution timeout that occurred during the execution.

" + } + }, + "com.amazonaws.sfn#GetActivityTask": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#GetActivityTaskInput" + }, + "output": { + "target": "com.amazonaws.sfn#GetActivityTaskOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ActivityDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ActivityWorkerLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Used by workers to retrieve a task (with the specified activity ARN) which has been\n scheduled for execution by a running state machine. This initiates a long poll, where the\n service holds the HTTP connection open and responds as soon as a task becomes available (i.e.\n an execution of a task of this type is needed.) The maximum time the service holds on to the\n request before responding is 60 seconds. If no task is available within 60 seconds, the poll\n returns a taskToken with a null string.

\n \n

Workers should set their client side socket timeout to at least 65 seconds (5 seconds\n higher than the maximum time the service may hold the poll request).

\n

Polling with GetActivityTask can cause latency in some implementations. See\n Avoid\n Latency When Polling for Activity Tasks in the Step Functions Developer Guide.

\n " + } + }, + "com.amazonaws.sfn#GetActivityTaskInput": { + "type": "structure", + "members": { + "activityArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the activity to retrieve tasks from (assigned when you create the task\n using CreateActivity.)

", + "smithy.api#required": {} + } + }, + "workerName": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

You can provide an arbitrary name in order to identify the worker that the task is\n assigned to. This name is used when it is logged in the execution history.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#GetActivityTaskOutput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

A token that identifies the scheduled task. This token must be copied and included in\n subsequent calls to SendTaskHeartbeat, SendTaskSuccess or\n SendTaskFailure in order to report the progress or completion of the\n task.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveDataJobInput", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#GetExecutionHistory": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#GetExecutionHistoryInput" + }, + "output": { + "target": "com.amazonaws.sfn#GetExecutionHistoryOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + } + ], + "traits": { + "smithy.api#documentation": "

Returns the history of the specified execution as a list of events. By default, the\n results are returned in ascending order of the timeStamp of the events. Use the\n reverseOrder parameter to get the latest events first.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

This API action is not supported by EXPRESS state machines.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "events", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#GetExecutionHistoryInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution.

", + "smithy.api#required": {} + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "reverseOrder": { + "target": "com.amazonaws.sfn#ReverseOrder", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Lists events in descending order of their timeStamp.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "includeExecutionData": { + "target": "com.amazonaws.sfn#IncludeExecutionDataGetExecutionHistory", + "traits": { + "smithy.api#documentation": "

You can select whether execution data (input or output of a history event) is returned.\n The default is true.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#GetExecutionHistoryOutput": { + "type": "structure", + "members": { + "events": { + "target": "com.amazonaws.sfn#HistoryEventList", + "traits": { + "smithy.api#documentation": "

The list of events that occurred in the execution.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#HTTPBody": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPHeaders": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPMethod": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPProtocol": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPStatusCode": { + "type": "string" + }, + "com.amazonaws.sfn#HTTPStatusMessage": { + "type": "string" + }, + "com.amazonaws.sfn#HistoryEvent": { + "type": "structure", + "members": { + "timestamp": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the event occurred, expressed in seconds and fractional milliseconds since the Unix epoch, which is defined as January 1, 1970, at 00:00:00 Coordinated Universal Time (UTC).

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#HistoryEventType", + "traits": { + "smithy.api#documentation": "

The type of the event.

", + "smithy.api#required": {} + } + }, + "id": { + "target": "com.amazonaws.sfn#EventId", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The id of the event. Events are numbered sequentially, starting at one.

", + "smithy.api#required": {} + } + }, + "previousEventId": { + "target": "com.amazonaws.sfn#EventId", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The id of the previous event.

" + } + }, + "activityFailedEventDetails": { + "target": "com.amazonaws.sfn#ActivityFailedEventDetails" + }, + "activityScheduleFailedEventDetails": { + "target": "com.amazonaws.sfn#ActivityScheduleFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an activity schedule event that failed during an execution.

" + } + }, + "activityScheduledEventDetails": { + "target": "com.amazonaws.sfn#ActivityScheduledEventDetails" + }, + "activityStartedEventDetails": { + "target": "com.amazonaws.sfn#ActivityStartedEventDetails" + }, + "activitySucceededEventDetails": { + "target": "com.amazonaws.sfn#ActivitySucceededEventDetails" + }, + "activityTimedOutEventDetails": { + "target": "com.amazonaws.sfn#ActivityTimedOutEventDetails" + }, + "taskFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the failure of a task.

" + } + }, + "taskScheduledEventDetails": { + "target": "com.amazonaws.sfn#TaskScheduledEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that was scheduled.

" + } + }, + "taskStartFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskStartFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to start.

" + } + }, + "taskStartedEventDetails": { + "target": "com.amazonaws.sfn#TaskStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that was started.

" + } + }, + "taskSubmitFailedEventDetails": { + "target": "com.amazonaws.sfn#TaskSubmitFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that where the submit failed.

" + } + }, + "taskSubmittedEventDetails": { + "target": "com.amazonaws.sfn#TaskSubmittedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a submitted task.

" + } + }, + "taskSucceededEventDetails": { + "target": "com.amazonaws.sfn#TaskSucceededEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that succeeded.

" + } + }, + "taskTimedOutEventDetails": { + "target": "com.amazonaws.sfn#TaskTimedOutEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a task that timed out.

" + } + }, + "executionFailedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionFailedEventDetails" + }, + "executionStartedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionStartedEventDetails" + }, + "executionSucceededEventDetails": { + "target": "com.amazonaws.sfn#ExecutionSucceededEventDetails" + }, + "executionAbortedEventDetails": { + "target": "com.amazonaws.sfn#ExecutionAbortedEventDetails" + }, + "executionTimedOutEventDetails": { + "target": "com.amazonaws.sfn#ExecutionTimedOutEventDetails" + }, + "executionRedrivenEventDetails": { + "target": "com.amazonaws.sfn#ExecutionRedrivenEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the redrive attempt of an execution.

" + } + }, + "mapStateStartedEventDetails": { + "target": "com.amazonaws.sfn#MapStateStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about Map state that was started.

" + } + }, + "mapIterationStartedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that was started.

" + } + }, + "mapIterationSucceededEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that succeeded.

" + } + }, + "mapIterationFailedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that failed.

" + } + }, + "mapIterationAbortedEventDetails": { + "target": "com.amazonaws.sfn#MapIterationEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state that was aborted.

" + } + }, + "lambdaFunctionFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionFailedEventDetails" + }, + "lambdaFunctionScheduleFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionScheduleFailedEventDetails" + }, + "lambdaFunctionScheduledEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionScheduledEventDetails" + }, + "lambdaFunctionStartFailedEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionStartFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a lambda function that failed to start during an execution.

" + } + }, + "lambdaFunctionSucceededEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionSucceededEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that terminated successfully during an\n execution.

" + } + }, + "lambdaFunctionTimedOutEventDetails": { + "target": "com.amazonaws.sfn#LambdaFunctionTimedOutEventDetails" + }, + "stateEnteredEventDetails": { + "target": "com.amazonaws.sfn#StateEnteredEventDetails" + }, + "stateExitedEventDetails": { + "target": "com.amazonaws.sfn#StateExitedEventDetails" + }, + "mapRunStartedEventDetails": { + "target": "com.amazonaws.sfn#MapRunStartedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details, such as mapRunArn, and the start date and time of a Map Run. mapRunArn is the Amazon Resource Name (ARN) of the Map Run that was started.

" + } + }, + "mapRunFailedEventDetails": { + "target": "com.amazonaws.sfn#MapRunFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains error and cause details about a Map Run that failed.

" + } + }, + "mapRunRedrivenEventDetails": { + "target": "com.amazonaws.sfn#MapRunRedrivenEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the redrive attempt of a Map Run.

" + } + }, + "evaluationFailedEventDetails": { + "target": "com.amazonaws.sfn#EvaluationFailedEventDetails", + "traits": { + "smithy.api#documentation": "

Contains details about an evaluation failure that occurred while processing a state.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the events of an execution.

" + } + }, + "com.amazonaws.sfn#HistoryEventExecutionDataDetails": { + "type": "structure", + "members": { + "truncated": { + "target": "com.amazonaws.sfn#truncated", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Indicates whether input or output was truncated in the response. Always false\n for API calls. In CloudWatch logs, the value will be true if the data is truncated due to size limits.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Provides details about input or output in an execution history event.

" + } + }, + "com.amazonaws.sfn#HistoryEventList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#HistoryEvent" + }, + "traits": { + "smithy.api#documentation": "

Contains details about the events that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#HistoryEventType": { + "type": "enum", + "members": { + "ActivityFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityFailed" + } + }, + "ActivityScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityScheduled" + } + }, + "ActivityScheduleFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityScheduleFailed" + } + }, + "ActivityStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityStarted" + } + }, + "ActivitySucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivitySucceeded" + } + }, + "ActivityTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ActivityTimedOut" + } + }, + "ChoiceStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ChoiceStateEntered" + } + }, + "ChoiceStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ChoiceStateExited" + } + }, + "ExecutionAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionAborted" + } + }, + "ExecutionFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionFailed" + } + }, + "ExecutionStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionStarted" + } + }, + "ExecutionSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionSucceeded" + } + }, + "ExecutionTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionTimedOut" + } + }, + "FailStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FailStateEntered" + } + }, + "LambdaFunctionFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionFailed" + } + }, + "LambdaFunctionScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionScheduled" + } + }, + "LambdaFunctionScheduleFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionScheduleFailed" + } + }, + "LambdaFunctionStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionStarted" + } + }, + "LambdaFunctionStartFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionStartFailed" + } + }, + "LambdaFunctionSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionSucceeded" + } + }, + "LambdaFunctionTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "LambdaFunctionTimedOut" + } + }, + "MapIterationAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationAborted" + } + }, + "MapIterationFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationFailed" + } + }, + "MapIterationStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationStarted" + } + }, + "MapIterationSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapIterationSucceeded" + } + }, + "MapStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateAborted" + } + }, + "MapStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateEntered" + } + }, + "MapStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateExited" + } + }, + "MapStateFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateFailed" + } + }, + "MapStateStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateStarted" + } + }, + "MapStateSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapStateSucceeded" + } + }, + "ParallelStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateAborted" + } + }, + "ParallelStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateEntered" + } + }, + "ParallelStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateExited" + } + }, + "ParallelStateFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateFailed" + } + }, + "ParallelStateStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateStarted" + } + }, + "ParallelStateSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ParallelStateSucceeded" + } + }, + "PassStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PassStateEntered" + } + }, + "PassStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PassStateExited" + } + }, + "SucceedStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SucceedStateEntered" + } + }, + "SucceedStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SucceedStateExited" + } + }, + "TaskFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskFailed" + } + }, + "TaskScheduled": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskScheduled" + } + }, + "TaskStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStarted" + } + }, + "TaskStartFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStartFailed" + } + }, + "TaskStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateAborted" + } + }, + "TaskStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateEntered" + } + }, + "TaskStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskStateExited" + } + }, + "TaskSubmitFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSubmitFailed" + } + }, + "TaskSubmitted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSubmitted" + } + }, + "TaskSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskSucceeded" + } + }, + "TaskTimedOut": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TaskTimedOut" + } + }, + "WaitStateAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateAborted" + } + }, + "WaitStateEntered": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateEntered" + } + }, + "WaitStateExited": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WaitStateExited" + } + }, + "MapRunAborted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunAborted" + } + }, + "MapRunFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunFailed" + } + }, + "MapRunStarted": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunStarted" + } + }, + "MapRunSucceeded": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunSucceeded" + } + }, + "ExecutionRedriven": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ExecutionRedriven" + } + }, + "MapRunRedriven": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MapRunRedriven" + } + }, + "EvaluationFailed": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "EvaluationFailed" + } + } + } + }, + "com.amazonaws.sfn#Identity": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + } + } + }, + "com.amazonaws.sfn#IncludeExecutionData": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#IncludeExecutionDataGetExecutionHistory": { + "type": "boolean" + }, + "com.amazonaws.sfn#IncludedData": { + "type": "enum", + "members": { + "ALL_DATA": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ALL_DATA" + } + }, + "METADATA_ONLY": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "METADATA_ONLY" + } + } + } + }, + "com.amazonaws.sfn#InspectionData": { + "type": "structure", + "members": { + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The raw state input.

" + } + }, + "afterArguments": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The input after Step Functions applies an Arguments filter. This event will only be present when QueryLanguage for the state machine or individual states is set to JSONata. For more info, see Transforming data with Step Functions.

" + } + }, + "afterInputPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The input after Step Functions applies the InputPath filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "afterParameters": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after Step Functions applies the Parameters filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "result": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The state's raw result.

" + } + }, + "afterResultSelector": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective result after Step Functions applies the ResultSelector filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "afterResultPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective result combined with the raw state input after Step Functions applies the ResultPath filter. Not populated when QueryLanguage is JSONata.

" + } + }, + "request": { + "target": "com.amazonaws.sfn#InspectionDataRequest", + "traits": { + "smithy.api#documentation": "

The raw HTTP request that is sent when you test an HTTP Task.

" + } + }, + "response": { + "target": "com.amazonaws.sfn#InspectionDataResponse", + "traits": { + "smithy.api#documentation": "

The raw HTTP response that is returned when you test an HTTP Task.

" + } + }, + "variables": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

JSON string that contains the set of workflow variables after execution of the state. The set will include variables assigned in the state and variables set up as test state input.

" + } + }, + "errorDetails": { + "target": "com.amazonaws.sfn#InspectionErrorDetails", + "traits": { + "smithy.api#documentation": "

An object containing data about a handled exception in the tested state.

" + } + }, + "afterItemsPath": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemsPath filter is applied. Not populated when the QueryLanguage is JSONata.

" + } + }, + "afterItemSelector": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

An array containing the inputs for each Map iteration, transformed by the ItemSelector specified in a Map state.

" + } + }, + "afterItemBatcher": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemBatcher filter is applied in a Map state.

" + } + }, + "afterItemsPointer": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The effective input after the ItemsPointer filter is applied in a Map state.

" + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#InspectionToleratedFailureCount", + "traits": { + "smithy.api#documentation": "

The tolerated failure threshold for a Map state as defined in number of Map state iterations.

" + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#InspectionToleratedFailurePercentage", + "traits": { + "smithy.api#documentation": "

The tolerated failure threshold for a Map state as defined in percentage of Map state iterations.

" + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#InspectionMaxConcurrency", + "traits": { + "smithy.api#documentation": "

The max concurrency of the Map state.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP request and response information.

", + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionDataRequest": { + "type": "structure", + "members": { + "protocol": { + "target": "com.amazonaws.sfn#HTTPProtocol", + "traits": { + "smithy.api#documentation": "

The protocol used to make the HTTP request.

" + } + }, + "method": { + "target": "com.amazonaws.sfn#HTTPMethod", + "traits": { + "smithy.api#documentation": "

The HTTP method used for the HTTP request.

" + } + }, + "url": { + "target": "com.amazonaws.sfn#URL", + "traits": { + "smithy.api#documentation": "

The API endpoint used for the HTTP request.

" + } + }, + "headers": { + "target": "com.amazonaws.sfn#HTTPHeaders", + "traits": { + "smithy.api#documentation": "

The request headers associated with the HTTP request.

" + } + }, + "body": { + "target": "com.amazonaws.sfn#HTTPBody", + "traits": { + "smithy.api#documentation": "

The request body for the HTTP request.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP request information.

" + } + }, + "com.amazonaws.sfn#InspectionDataResponse": { + "type": "structure", + "members": { + "protocol": { + "target": "com.amazonaws.sfn#HTTPProtocol", + "traits": { + "smithy.api#documentation": "

The protocol used to return the HTTP response.

" + } + }, + "statusCode": { + "target": "com.amazonaws.sfn#HTTPStatusCode", + "traits": { + "smithy.api#documentation": "

The HTTP response status code for the HTTP response.

" + } + }, + "statusMessage": { + "target": "com.amazonaws.sfn#HTTPStatusMessage", + "traits": { + "smithy.api#documentation": "

The message associated with the HTTP status code.

" + } + }, + "headers": { + "target": "com.amazonaws.sfn#HTTPHeaders", + "traits": { + "smithy.api#documentation": "

The response headers associated with the HTTP response.

" + } + }, + "body": { + "target": "com.amazonaws.sfn#HTTPBody", + "traits": { + "smithy.api#documentation": "

The HTTP response returned.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains additional details about the state's execution, including its input and output data processing flow, and HTTP response information. The inspectionLevel request parameter specifies which details are returned.

" + } + }, + "com.amazonaws.sfn#InspectionErrorDetails": { + "type": "structure", + "members": { + "catchIndex": { + "target": "com.amazonaws.sfn#ExceptionHandlerIndex", + "traits": { + "smithy.api#documentation": "

The array index of the Catch which handled the exception.

" + } + }, + "retryIndex": { + "target": "com.amazonaws.sfn#ExceptionHandlerIndex", + "traits": { + "smithy.api#documentation": "

The array index of the Retry which handled the exception.

" + } + }, + "retryBackoffIntervalSeconds": { + "target": "com.amazonaws.sfn#RetryBackoffIntervalSeconds", + "traits": { + "smithy.api#documentation": "

The duration in seconds of the backoff for a retry on a failed state invocation.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

An object containing data about a handled exception in the tested state.

" + } + }, + "com.amazonaws.sfn#InspectionLevel": { + "type": "enum", + "members": { + "INFO": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "INFO" + } + }, + "DEBUG": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DEBUG" + } + }, + "TRACE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TRACE" + } + } + } + }, + "com.amazonaws.sfn#InspectionMaxConcurrency": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionToleratedFailureCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InspectionToleratedFailurePercentage": { + "type": "float", + "traits": { + "smithy.api#range": { + "min": 0, + "max": 100 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#InvalidArn": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided Amazon Resource Name (ARN) is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidDefinition": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided Amazon States Language definition is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidEncryptionConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Received when encryptionConfiguration is specified but various conditions exist which make the configuration invalid. For example, if type is set to CUSTOMER_MANAGED_KMS_KEY, but kmsKeyId is null, or kmsDataKeyReusePeriodSeconds is not between 60 and 900, or the KMS key is not symmetric or inactive.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidExecutionInput": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided JSON input data is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidLoggingConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Configuration is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidName": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided name is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidOutput": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided JSON output data is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidToken": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The provided token is not valid.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#InvalidTracingConfiguration": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Your tracingConfiguration key does not match, or enabled has not\n been set to true or false.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsAccessDeniedException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Either your KMS key policy or API caller does not have the required permissions.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsDataKeyReusePeriodSeconds": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 60, + "max": 900 + } + } + }, + "com.amazonaws.sfn#KmsInvalidStateException": { + "type": "structure", + "members": { + "kmsKeyState": { + "target": "com.amazonaws.sfn#KmsKeyState", + "traits": { + "smithy.api#documentation": "

Current status of the KMS; key. For example: DISABLED, PENDING_DELETION, PENDING_IMPORT, UNAVAILABLE, CREATING.

" + } + }, + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The KMS key is not in valid state, for example: Disabled or Deleted.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#KmsKeyId": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2048 + } + } + }, + "com.amazonaws.sfn#KmsKeyState": { + "type": "enum", + "members": { + "DISABLED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DISABLED" + } + }, + "PENDING_DELETION": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_DELETION" + } + }, + "PENDING_IMPORT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PENDING_IMPORT" + } + }, + "UNAVAILABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "UNAVAILABLE" + } + }, + "CREATING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CREATING" + } + } + } + }, + "com.amazonaws.sfn#KmsThrottlingException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Received when KMS returns ThrottlingException for a KMS call that Step Functions makes on behalf of the caller.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#LambdaFunctionFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that failed during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionScheduleFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a failed Lambda function schedule event that occurred during an\n execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionScheduledEventDetails": { + "type": "structure", + "members": { + "resource": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the scheduled Lambda function.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data input to the Lambda function. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about input for an execution history event.

" + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the Lambda function.

" + } + }, + "taskCredentials": { + "target": "com.amazonaws.sfn#TaskCredentials", + "traits": { + "smithy.api#documentation": "

The credentials that Step Functions uses for the task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionStartFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a lambda function that failed to start during an execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionSucceededEventDetails": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON data output by the Lambda function. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function that successfully terminated during an\n execution.

" + } + }, + "com.amazonaws.sfn#LambdaFunctionTimedOutEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the timeout.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Lambda function timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#ListActivities": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListActivitiesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListActivitiesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists the existing activities.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n ", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "activities", + "pageSize": "maxResults" + }, + "smithy.test#smokeTests": [ + { + "id": "ListActivitiesSuccess", + "params": {}, + "vendorParams": { + "region": "us-west-2" + }, + "vendorParamsShape": "aws.test#AwsVendorParams", + "expect": { + "success": {} + } + } + ] + } + }, + "com.amazonaws.sfn#ListActivitiesInput": { + "type": "structure", + "members": { + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListActivitiesOutput": { + "type": "structure", + "members": { + "activities": { + "target": "com.amazonaws.sfn#ActivityList", + "traits": { + "smithy.api#documentation": "

The list of activities.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListExecutions": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListExecutionsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListExecutionsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Lists all executions of a state machine or a Map Run. You can list all executions related to a state machine by specifying a state machine Amazon Resource Name (ARN), or those related to a Map Run by specifying a Map Run ARN. Using this API action, you can also list all redriven executions.

\n

You can also provide a state machine alias ARN or version ARN to list the executions associated with a specific alias or version.

\n

Results are sorted by time, with the most recent execution first. Running executions are sorted by their startDate or redriveDate, and other executions are sorted by their stopDate.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n \n

This API action is not supported by EXPRESS state machines. However, you may list EXPRESS children started by a map run using the mapRunArn parameter.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "executions", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListExecutionsInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine whose executions is listed.

\n

You can specify either a mapRunArn or a stateMachineArn, but not both.

\n

You can also return a list of executions associated with a specific alias or version, by specifying an alias ARN or a version ARN in the stateMachineArn parameter.

" + } + }, + "statusFilter": { + "target": "com.amazonaws.sfn#ExecutionStatus", + "traits": { + "smithy.api#documentation": "

If specified, only list the executions whose current execution status matches the given\n filter.

\n

If you provide a PENDING_REDRIVE statusFilter, you must specify mapRunArn. \n For more information, see Child workflow execution redrive behaviour\n in the Step Functions Developer Guide.\n

\n

If you provide a stateMachineArn and a PENDING_REDRIVE statusFilter, the API returns a validation exception.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#ListExecutionsPageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run that started the child workflow executions. If the mapRunArn field is specified, a list of all of the child workflow executions started by a Map Run is returned. For more information, see Examining Map Run in the Step Functions Developer Guide.

\n

You can specify either a mapRunArn or a stateMachineArn, but not both.

" + } + }, + "redriveFilter": { + "target": "com.amazonaws.sfn#ExecutionRedriveFilter", + "traits": { + "smithy.api#documentation": "

Sets a filter to list executions based on whether or not they have been redriven.

\n

For a Distributed Map, redriveFilter sets a filter to list child workflow executions based on whether or not they have been redriven.

\n

If you do not provide a redriveFilter, Step Functions returns a list of both redriven and non-redriven executions.

\n

If you provide a state machine ARN in redriveFilter, the API returns a validation exception.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListExecutionsOutput": { + "type": "structure", + "members": { + "executions": { + "target": "com.amazonaws.sfn#ExecutionList", + "traits": { + "smithy.api#documentation": "

The list of matching executions.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#ListExecutionsPageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListExecutionsPageToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 3096 + } + } + }, + "com.amazonaws.sfn#ListMapRuns": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListMapRunsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListMapRunsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists all Map Runs that were started by a given state machine execution. Use this API action to obtain Map Run ARNs, and then call DescribeMapRun to obtain more information, if needed.

", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "mapRuns", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListMapRunsInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution for which the Map Runs must be listed.

", + "smithy.api#required": {} + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListMapRunsOutput": { + "type": "structure", + "members": { + "mapRuns": { + "target": "com.amazonaws.sfn#MapRunList", + "traits": { + "smithy.api#documentation": "

An array that lists information related to a Map Run, such as the Amazon Resource Name (ARN) of the Map Run and the ARN of the state machine that started the Map Run.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachineAliases": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachineAliasesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachineAliasesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + } + ], + "traits": { + "smithy.api#documentation": "

Lists aliases for a specified state machine ARN. Results are sorted by time, with the most recently created aliases listed first.

\n

To list aliases that reference a state machine version, you can specify the version ARN in the stateMachineArn parameter.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#ListStateMachineAliasesInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine for which you want to list aliases.

\n

If you specify a state machine version ARN, this API returns a list of aliases for that version.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachineAliasesOutput": { + "type": "structure", + "members": { + "stateMachineAliases": { + "target": "com.amazonaws.sfn#StateMachineAliasList", + "traits": { + "smithy.api#documentation": "

Aliases for the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachineVersions": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachineVersionsInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachineVersionsOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Lists versions for the specified state machine Amazon Resource Name (ARN).

\n

The results are sorted in descending order of the version creation time.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#ListStateMachineVersionsInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachineVersionsOutput": { + "type": "structure", + "members": { + "stateMachineVersions": { + "target": "com.amazonaws.sfn#StateMachineVersionList", + "traits": { + "smithy.api#documentation": "

Versions for the state machine.

", + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListStateMachines": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListStateMachinesInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListStateMachinesOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + } + ], + "traits": { + "smithy.api#documentation": "

Lists the existing state machines.

\n

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

\n \n

This operation is eventually consistent. The results are best effort and may not reflect very recent updates and changes.

\n ", + "smithy.api#paginated": { + "inputToken": "nextToken", + "outputToken": "nextToken", + "items": "stateMachines", + "pageSize": "maxResults" + } + } + }, + "com.amazonaws.sfn#ListStateMachinesInput": { + "type": "structure", + "members": { + "maxResults": { + "target": "com.amazonaws.sfn#PageSize", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of results that are returned per call. You can use nextToken to obtain further pages of results.\n The default is 100 and the maximum allowed page size is 1000. A value of 0 uses the default.

\n

This is only an upper limit. The actual number of results returned per call might be fewer than the specified maximum.

" + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListStateMachinesOutput": { + "type": "structure", + "members": { + "stateMachines": { + "target": "com.amazonaws.sfn#StateMachineList", + "traits": { + "smithy.api#required": {} + } + }, + "nextToken": { + "target": "com.amazonaws.sfn#PageToken", + "traits": { + "smithy.api#documentation": "

If nextToken is returned, there are more results available. The value of nextToken is a unique pagination token for each page.\n Make the call again using the returned token to retrieve the next page. Keep all other arguments unchanged. Each pagination token expires after 24 hours. Using an expired pagination token will return an HTTP 400 InvalidToken error.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ListTagsForResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ListTagsForResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#ListTagsForResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

List tags for a given resource.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#ListTagsForResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ListTagsForResourceOutput": { + "type": "structure", + "members": { + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

An array of tags associated with the resource.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#LogDestination": { + "type": "structure", + "members": { + "cloudWatchLogsLogGroup": { + "target": "com.amazonaws.sfn#CloudWatchLogsLogGroup", + "traits": { + "smithy.api#documentation": "

An object describing a CloudWatch log group. For more information, see AWS::Logs::LogGroup in the CloudFormation User Guide.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

" + } + }, + "com.amazonaws.sfn#LogDestinationList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#LogDestination" + } + }, + "com.amazonaws.sfn#LogLevel": { + "type": "enum", + "members": { + "ALL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ALL" + } + }, + "ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ERROR" + } + }, + "FATAL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FATAL" + } + }, + "OFF": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "OFF" + } + } + } + }, + "com.amazonaws.sfn#LoggingConfiguration": { + "type": "structure", + "members": { + "level": { + "target": "com.amazonaws.sfn#LogLevel", + "traits": { + "smithy.api#documentation": "

Defines which category of execution history events are logged.

" + } + }, + "includeExecutionData": { + "target": "com.amazonaws.sfn#IncludeExecutionData", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Determines whether execution data is included in your log. When set to false,\n data is excluded.

" + } + }, + "destinations": { + "target": "com.amazonaws.sfn#LogDestinationList", + "traits": { + "smithy.api#documentation": "

An array of objects that describes where your execution history events will be logged.\n Limited to size 1. Required, if your log level is not set to OFF.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

The LoggingConfiguration data type is used to set CloudWatch Logs\n options.

" + } + }, + "com.amazonaws.sfn#LongArn": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2000 + } + } + }, + "com.amazonaws.sfn#LongObject": { + "type": "long" + }, + "com.amazonaws.sfn#MapIterationEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the iteration\u2019s parent Map state.

" + } + }, + "index": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The index of the array belonging to the Map state iteration.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an iteration of a Map state.

" + } + }, + "com.amazonaws.sfn#MapIterationFailureCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#MapRunExecutionCounts": { + "type": "structure", + "members": { + "pending": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run, but haven't started executing yet.

", + "smithy.api#required": {} + } + }, + "running": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and are currently in-progress.

", + "smithy.api#required": {} + } + }, + "succeeded": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and have completed successfully.

", + "smithy.api#required": {} + } + }, + "failed": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run, but have failed.

", + "smithy.api#required": {} + } + }, + "timedOut": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and have timed out.

", + "smithy.api#required": {} + } + }, + "aborted": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run and were running, but were either stopped by the user or by Step Functions because the Map Run failed.

", + "smithy.api#required": {} + } + }, + "total": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of child workflow executions that were started by a Map Run.

", + "smithy.api#required": {} + } + }, + "resultsWritten": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Returns the count of child workflow executions whose results were written by ResultWriter. For more information, see ResultWriter in the Step Functions Developer Guide.

", + "smithy.api#required": {} + } + }, + "failuresNotRedrivable": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of FAILED, ABORTED, or TIMED_OUT child workflow executions that cannot be redriven because their execution status is terminal. For example, child workflows with an execution status of FAILED, ABORTED, or TIMED_OUT and a redriveStatus of NOT_REDRIVABLE.

" + } + }, + "pendingRedrive": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of unsuccessful child workflow executions currently waiting to be redriven. The status of these child workflow executions could be FAILED, ABORTED, or TIMED_OUT in the original execution attempt or a previous redrive attempt.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about all of the child workflow executions started by a Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunFailedEventDetails": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the Map Run failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run failure event that occurred during a state machine execution.

" + } + }, + "com.amazonaws.sfn#MapRunItemCounts": { + "type": "structure", + "members": { + "pending": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items to process in child workflow executions that haven't started running yet.

", + "smithy.api#required": {} + } + }, + "running": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items being processed in child workflow executions that are currently in-progress.

", + "smithy.api#required": {} + } + }, + "succeeded": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have completed successfully.

", + "smithy.api#required": {} + } + }, + "failed": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have failed.

", + "smithy.api#required": {} + } + }, + "timedOut": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that have timed out.

", + "smithy.api#required": {} + } + }, + "aborted": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in child workflow executions that were either stopped by the user or by Step Functions, because the Map Run failed.

", + "smithy.api#required": {} + } + }, + "total": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The total number of items processed in all the child workflow executions started by a Map Run.

", + "smithy.api#required": {} + } + }, + "resultsWritten": { + "target": "com.amazonaws.sfn#UnsignedLong", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

Returns the count of items whose results were written by ResultWriter. For more information, see ResultWriter in the Step Functions Developer Guide.

", + "smithy.api#required": {} + } + }, + "failuresNotRedrivable": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of FAILED, ABORTED, or TIMED_OUT items in child workflow executions that cannot be redriven because the execution status of those child workflows is terminal. For example, child workflows with an execution status of FAILED, ABORTED, or TIMED_OUT and a redriveStatus of NOT_REDRIVABLE.

" + } + }, + "pendingRedrive": { + "target": "com.amazonaws.sfn#LongObject", + "traits": { + "smithy.api#documentation": "

The number of unsuccessful items in child workflow executions currently waiting to be redriven.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about items that were processed in all of the child workflow executions that were started by a Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunLabel": { + "type": "string" + }, + "com.amazonaws.sfn#MapRunList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#MapRunListItem" + } + }, + "com.amazonaws.sfn#MapRunListItem": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The executionArn of the execution from which the Map Run was started.

", + "smithy.api#required": {} + } + }, + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the Map Run.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the executed state machine.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date on which the Map Run started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date on which the Map Run stopped.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific Map Run.

" + } + }, + "com.amazonaws.sfn#MapRunRedrivenEventDetails": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run that was redriven.

" + } + }, + "redriveCount": { + "target": "com.amazonaws.sfn#RedriveCount", + "traits": { + "smithy.api#documentation": "

The number of times the Map Run has been redriven at this point in the execution's history including this event. The redrive count for a redriven Map Run is always greater than 0.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run that was redriven.

" + } + }, + "com.amazonaws.sfn#MapRunStartedEventDetails": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run that was started.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a Map Run that was started during a state machine execution.

" + } + }, + "com.amazonaws.sfn#MapRunStatus": { + "type": "enum", + "members": { + "RUNNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RUNNING" + } + }, + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "ABORTED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ABORTED" + } + } + } + }, + "com.amazonaws.sfn#MapStateStartedEventDetails": { + "type": "structure", + "members": { + "length": { + "target": "com.amazonaws.sfn#UnsignedInteger", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The size of the array for Map state iterations.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Details about a Map state that was started.

" + } + }, + "com.amazonaws.sfn#MaxConcurrency": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#MissingRequiredParameter": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

Request is missing a required parameter. This error occurs if both definition\n and roleArn are not specified.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#MockErrorOutput": { + "type": "structure", + "members": { + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

A string denoting the error code of the exception thrown when invoking the tested state. This field is required if mock.errorOutput is specified.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A string containing the cause of the exception thrown when executing the state's logic.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

A JSON object that contains a mocked error.

" + } + }, + "com.amazonaws.sfn#MockInput": { + "type": "structure", + "members": { + "result": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A JSON string containing the mocked result of the state invocation.

" + } + }, + "errorOutput": { + "target": "com.amazonaws.sfn#MockErrorOutput", + "traits": { + "smithy.api#documentation": "

The mocked error output when calling TestState. When specified, the mocked response is returned as a JSON object that contains an error and cause field.

" + } + }, + "fieldValidationMode": { + "target": "com.amazonaws.sfn#MockResponseValidationMode", + "traits": { + "smithy.api#documentation": "

Determines the level of strictness when validating mocked results against their respective API models. Values include:

\n \n

If no value is specified, the default value is STRICT.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

A JSON object that contains a mocked result or errorOutput.

" + } + }, + "com.amazonaws.sfn#MockResponseValidationMode": { + "type": "enum", + "members": { + "STRICT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "STRICT" + } + }, + "PRESENT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "PRESENT" + } + }, + "NONE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "NONE" + } + } + } + }, + "com.amazonaws.sfn#Name": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + } + } + }, + "com.amazonaws.sfn#PageSize": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 1000 + } + } + }, + "com.amazonaws.sfn#PageToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 1024 + } + } + }, + "com.amazonaws.sfn#Publish": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#PublishStateMachineVersion": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#PublishStateMachineVersionInput" + }, + "output": { + "target": "com.amazonaws.sfn#PublishStateMachineVersionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Creates a version from the\n current revision of a state machine. Use versions to create immutable snapshots of your state\n machine. You can start executions from versions either directly or with an alias. To create an\n alias, use CreateStateMachineAlias.

\n

You can publish up to 1000 versions for each state machine. You must manually delete unused versions using the DeleteStateMachineVersion API action.

\n

\n PublishStateMachineVersion is an idempotent API. It doesn't create a\n duplicate state machine version if it already exists for the current revision. Step Functions bases PublishStateMachineVersion's idempotency check on the\n stateMachineArn, name, and revisionId parameters.\n Requests with the same parameters return a successful idempotent response. If you don't\n specify a revisionId, Step Functions checks for a previously published\n version of the state machine's current revision.

\n

\n Related operations:\n

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#PublishStateMachineVersionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

Only publish the state machine version if the current state machine's revision ID matches the specified ID.

\n

Use this option to avoid publishing a version if the state machine changed since you last\n updated it. If the specified revision ID doesn't match the state machine's current revision\n ID, the API returns ConflictException.

\n \n

To specify an initial revision ID for a state machine with no revision ID assigned,\n specify the string INITIAL for the revisionId parameter. For\n example, you can specify a revisionID of INITIAL when you create a\n state machine using the CreateStateMachine API action.

\n " + } + }, + "description": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

An optional description of the state machine version.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#PublishStateMachineVersionOutput": { + "type": "structure", + "members": { + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the version was created.

", + "smithy.api#required": {} + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) (ARN) that identifies the state machine version.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#RedriveCount": { + "type": "integer" + }, + "com.amazonaws.sfn#RedriveExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#RedriveExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#RedriveExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ExecutionLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#ExecutionNotRedrivable" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Restarts unsuccessful executions of Standard workflows that didn't complete successfully in the last 14 days. These include failed, aborted, or timed out executions. When you redrive an execution, it continues the failed execution from the unsuccessful step and uses the same input. Step Functions preserves the results and execution history of the successful steps, and doesn't rerun these steps when you redrive an execution. Redriven executions use the same state machine definition and execution ARN as the original execution attempt.

\n

For workflows that include an Inline Map or Parallel state, RedriveExecution API action reschedules and redrives only the iterations and branches that failed or aborted.

\n

To redrive a workflow that includes a Distributed Map state whose Map Run failed, you must redrive the parent workflow. The parent workflow redrives all the unsuccessful states, including a failed Map Run. If a Map Run was not started in the original execution attempt, the redriven parent workflow starts the Map Run.

\n \n

This API action is not supported by EXPRESS state machines.

\n

However, you can restart the unsuccessful executions of Express child workflows in a Distributed Map by redriving its Map Run. When you redrive a Map Run, the Express child workflows are rerun using the StartExecution API action. For more information, see Redriving Map Runs.

\n \n

You can redrive executions if your original execution meets the following conditions:

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#RedriveExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to be redriven.

", + "smithy.api#required": {} + } + }, + "clientToken": { + "target": "com.amazonaws.sfn#ClientToken", + "traits": { + "smithy.api#documentation": "

A unique, case-sensitive identifier that you provide to ensure the idempotency of the request. If you don\u2019t specify a client token, the Amazon Web Services SDK automatically generates a client token and uses it for the request to ensure idempotency. The API will return idempotent responses for the last 10 client tokens used to successfully redrive the execution. These client tokens are valid for up to 15 minutes after they are first used.

", + "smithy.api#idempotencyToken": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#RedriveExecutionOutput": { + "type": "structure", + "members": { + "redriveDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution was last redriven.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ResourceNotFound": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "resourceName": { + "target": "com.amazonaws.sfn#Arn" + } + }, + "traits": { + "smithy.api#documentation": "

Could not find the referenced resource.

", + "smithy.api#error": "client", + "smithy.api#httpError": 404 + } + }, + "com.amazonaws.sfn#RetrierRetryCount": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#RetryBackoffIntervalSeconds": { + "type": "integer", + "traits": { + "smithy.api#range": { + "min": 0 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#RevealSecrets": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#ReverseOrder": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#RevisionId": { + "type": "string" + }, + "com.amazonaws.sfn#RoutingConfigurationList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#RoutingConfigurationListItem" + }, + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2 + } + } + }, + "com.amazonaws.sfn#RoutingConfigurationListItem": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies one or two state machine versions defined in the routing configuration.

\n

If you specify the ARN of a second version, it must belong to the same state machine as the first version.

", + "smithy.api#required": {} + } + }, + "weight": { + "target": "com.amazonaws.sfn#VersionWeight", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The percentage of traffic you want to route to a state machine version. The sum of the\n weights in the routing configuration must be equal to 100.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the routing configuration of a state machine alias. In a routing\n configuration, you define an array of objects that specify up to two state machine versions.\n You also specify the percentage of traffic to be routed to each version.

" + } + }, + "com.amazonaws.sfn#SendTaskFailure": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskFailureInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskFailureOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers, Task states using the callback\n pattern, and optionally Task states using the job run pattern to report that the task identified by the taskToken failed.

\n

For an execution with encryption enabled, Step Functions will encrypt the error and cause fields using the KMS key for the execution role.

\n

A caller can mark a task as fail without using any KMS permissions in the execution role if the caller provides a null value for both error and cause fields because no data needs to be encrypted.

" + } + }, + "com.amazonaws.sfn#SendTaskFailureInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskFailureOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SendTaskHeartbeat": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskHeartbeatInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskHeartbeatOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers and Task states using the callback\n pattern, and optionally Task states using the job run pattern to report to Step Functions that the task represented by the specified\n taskToken is still making progress. This action resets the\n Heartbeat clock. The Heartbeat threshold is specified in the state\n machine's Amazon States Language definition (HeartbeatSeconds). This action does not in itself\n create an event in the execution history. However, if the task times out, the execution\n history contains an ActivityTimedOut entry for activities, or a\n TaskTimedOut entry for tasks using the job run or\n callback\n pattern.

\n \n

The Timeout of a task, defined in the state machine's Amazon States Language definition, is\n its maximum allowed duration, regardless of the number of SendTaskHeartbeat requests received. Use HeartbeatSeconds to configure the timeout interval\n for heartbeats.

\n " + } + }, + "com.amazonaws.sfn#SendTaskHeartbeatInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskHeartbeatOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SendTaskSuccess": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#SendTaskSuccessInput" + }, + "output": { + "target": "com.amazonaws.sfn#SendTaskSuccessOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidOutput" + }, + { + "target": "com.amazonaws.sfn#InvalidToken" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#TaskDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#TaskTimedOut" + } + ], + "traits": { + "smithy.api#documentation": "

Used by activity workers, Task states using the callback\n pattern, and optionally Task states using the job run pattern to report that the task identified by the taskToken completed\n successfully.

" + } + }, + "com.amazonaws.sfn#SendTaskSuccessInput": { + "type": "structure", + "members": { + "taskToken": { + "target": "com.amazonaws.sfn#TaskToken", + "traits": { + "smithy.api#documentation": "

The token that represents this task. Task tokens are generated by Step Functions when\n tasks are assigned to a worker, or in the context object when a\n workflow enters a task state. See GetActivityTaskOutput$taskToken.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output of the task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#SendTaskSuccessOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SensitiveCause": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 32768 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveData": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 262144 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveDataJobInput": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 1048576 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#SensitiveError": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ServiceQuotaExceededException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The request would cause a service quota to be exceeded.

\n

HTTP Status Code: 402

", + "smithy.api#error": "client", + "smithy.api#httpError": 402 + } + }, + "com.amazonaws.sfn#StartExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StartExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StartExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionAlreadyExists" + }, + { + "target": "com.amazonaws.sfn#ExecutionLimitExceeded" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Starts a state machine execution.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n \n

If you start an execution with an unqualified state machine ARN, Step Functions uses the latest revision of the state machine for the execution.

\n

To start executions of a state machine version, call\n StartExecution and provide the version ARN or the ARN of an alias that points to the version.

\n \n

\n StartExecution is idempotent for STANDARD workflows. For a\n STANDARD workflow, if you call StartExecution with the same name\n and input as a running execution, the call succeeds and return the same response as the\n original request. If the execution is closed or if the input is different, it returns a\n 400 ExecutionAlreadyExists error. You can reuse the name 90 days after it closes.

\n

\n StartExecution isn't idempotent for EXPRESS workflows.

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#StartExecutionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to execute.

\n

The stateMachineArn parameter accepts one of the following inputs:

\n ", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

Optional name of the execution. For STANDARD workflows, this name must be unique for your Amazon Web Services account, region, and state machine.\n If a previous execution with the same name exists, you can reuse the name 90 days after it closes. For EXPRESS workflows, execution names\n can be reused immediately.\n For more information, see \n Limits Related to State Machine Executions in the Step Functions Developer Guide.

\n

If you don't provide a name for the execution, Step Functions automatically generates a universally unique identifier (UUID) as the execution name.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the execution, for example:

\n

\n \"{\\\"first_name\\\" : \\\"Alejandro\\\"}\"\n

\n \n

If you don't include any JSON input data, you still must include the two braces, for\n example: \"{}\"\n

\n \n

Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

Passes the X-Ray trace header. The trace header can also be passed in the request\n payload.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n " + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StartExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#StartSyncExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StartSyncExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StartSyncExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#InvalidName" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#StateMachineTypeNotSupported" + } + ], + "traits": { + "smithy.api#documentation": "

Starts a Synchronous Express state machine execution. StartSyncExecution \n\t\t\t is not available for STANDARD workflows.

\n \n

\n StartSyncExecution will return a 200 OK response, even if your\n execution fails, because the status code in the API response doesn't reflect function\n errors. Error codes are reserved for errors that prevent your execution from running, such\n as permissions errors, limit errors, or issues with your state machine code and\n configuration.

\n ", + "smithy.api#endpoint": { + "hostPrefix": "sync-" + } + } + }, + "com.amazonaws.sfn#StartSyncExecutionInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine to execute.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the execution, for example:

\n

\n \"{\\\"first_name\\\" : \\\"Alejandro\\\"}\"\n

\n \n

If you don't include any JSON input data, you still must include the two braces, for\n example: \"{}\"\n

\n \n

Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

Passes the X-Ray trace header. The trace header can also be passed in the request\n payload.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n " + } + }, + "includedData": { + "target": "com.amazonaws.sfn#IncludedData", + "traits": { + "smithy.api#documentation": "

If your state machine definition is encrypted with a KMS key, callers must have kms:Decrypt permission to decrypt the definition. Alternatively, you can call the API with includedData = METADATA_ONLY to get a successful response without the encrypted definition.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StartSyncExecutionOutput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the execution.

", + "smithy.api#required": {} + } + }, + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

" + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the execution.

" + } + }, + "startDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is started.

", + "smithy.api#required": {} + } + }, + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

If the execution has already ended, the date the execution stopped.

", + "smithy.api#required": {} + } + }, + "status": { + "target": "com.amazonaws.sfn#SyncExecutionStatus", + "traits": { + "smithy.api#documentation": "

The current status of the execution.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the execution. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

\n \n

This field is set only if the execution succeeds. If the execution fails, this field is\n null.

\n " + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#CloudWatchEventsExecutionDataDetails" + }, + "traceHeader": { + "target": "com.amazonaws.sfn#TraceHeader", + "traits": { + "smithy.api#documentation": "

The X-Ray trace header that was passed to the execution.

\n \n

\n For X-Ray traces, all Amazon Web Services services use the X-Amzn-Trace-Id header from the HTTP request. Using the header is the preferred mechanism to identify a trace. StartExecution and StartSyncExecution API operations can also use traceHeader from the body of the request payload. If both sources are provided, Step Functions will use the header value (preferred) over the value in the request body.\n

\n " + } + }, + "billingDetails": { + "target": "com.amazonaws.sfn#BillingDetails", + "traits": { + "smithy.api#documentation": "

An object that describes workflow billing details, including billed duration and memory\n use.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#StateEnteredEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state.

", + "smithy.api#required": {} + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The string that contains the JSON input data for the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "inputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the input for an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a state entered during an execution.

" + } + }, + "com.amazonaws.sfn#StateExitedEventDetails": { + "type": "structure", + "members": { + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + }, + "assignedVariables": { + "target": "com.amazonaws.sfn#AssignedVariables", + "traits": { + "smithy.api#documentation": "

Map of variable name and value as a serialized JSON representation.

" + } + }, + "assignedVariablesDetails": { + "target": "com.amazonaws.sfn#AssignedVariablesDetails", + "traits": { + "smithy.api#documentation": "

Provides details about input or output in an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about an exit from a state during an execution.

" + } + }, + "com.amazonaws.sfn#StateMachineAliasList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineAliasListItem" + } + }, + "com.amazonaws.sfn#StateMachineAliasListItem": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine alias. The alias ARN is a combination of state machine ARN and the alias name separated by a colon (:). For example, stateMachineARN:PROD.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The creation date of a state machine alias.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific state machine alias.

" + } + }, + "com.amazonaws.sfn#StateMachineAlreadyExists": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

A state machine with the same name but a different definition or role ARN already\n exists.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineDeleting": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified state machine is being deleted.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The specified state machine does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineLimitExceeded": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The maximum number of state machines has been reached. Existing state machines must be\n deleted before a new state machine can be created.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineListItem" + } + }, + "com.amazonaws.sfn#StateMachineListItem": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies the state machine.

", + "smithy.api#required": {} + } + }, + "name": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The name of the state machine.

\n

A name must not contain:

\n \n

To enable logging with CloudWatch Logs, the name should only contain 0-9, A-Z, a-z, - and _.

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the state machine is created.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the state machine.

" + } + }, + "com.amazonaws.sfn#StateMachineStatus": { + "type": "enum", + "members": { + "ACTIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ACTIVE" + } + }, + "DELETING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DELETING" + } + } + } + }, + "com.amazonaws.sfn#StateMachineType": { + "type": "enum", + "members": { + "STANDARD": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "STANDARD" + } + }, + "EXPRESS": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "EXPRESS" + } + } + } + }, + "com.amazonaws.sfn#StateMachineTypeNotSupported": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

State machine type is not supported.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#StateMachineVersionList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#StateMachineVersionListItem" + } + }, + "com.amazonaws.sfn#StateMachineVersionListItem": { + "type": "structure", + "members": { + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) that identifies a state machine version. The version ARN is a combination of state machine ARN and the version number separated by a colon (:). For example, stateMachineARN:1.

", + "smithy.api#required": {} + } + }, + "creationDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The creation date of a state machine version.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a specific state machine version.

" + } + }, + "com.amazonaws.sfn#StateName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + } + } + }, + "com.amazonaws.sfn#StopExecution": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#StopExecutionInput" + }, + "output": { + "target": "com.amazonaws.sfn#StopExecutionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ExecutionDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsInvalidStateException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Stops an execution.

\n

This API action is not supported by EXPRESS state machines.

\n

For an execution with encryption enabled, Step Functions will encrypt the error and cause fields using the KMS key for the execution role.

\n

A caller can stop an execution without using any KMS permissions in the execution role if the caller provides a null value for both error and cause fields because no data needs to be encrypted.

" + } + }, + "com.amazonaws.sfn#StopExecutionInput": { + "type": "structure", + "members": { + "executionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution to stop.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#StopExecutionOutput": { + "type": "structure", + "members": { + "stopDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date the execution is stopped.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#SyncExecutionStatus": { + "type": "enum", + "members": { + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "TIMED_OUT": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "TIMED_OUT" + } + } + } + }, + "com.amazonaws.sfn#Tag": { + "type": "structure", + "members": { + "key": { + "target": "com.amazonaws.sfn#TagKey", + "traits": { + "smithy.api#documentation": "

The key of a tag.

" + } + }, + "value": { + "target": "com.amazonaws.sfn#TagValue", + "traits": { + "smithy.api#documentation": "

The value of a tag.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Tags are key-value pairs that can be associated with Step Functions state machines and\n activities.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#TagKey": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 128 + } + } + }, + "com.amazonaws.sfn#TagKeyList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#TagKey" + } + }, + "com.amazonaws.sfn#TagList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#Tag" + } + }, + "com.amazonaws.sfn#TagResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#TagResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#TagResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#TooManyTags" + } + ], + "traits": { + "smithy.api#documentation": "

Add a tag to a Step Functions resource.

\n

An array of key-value pairs. For more information, see Using\n Cost Allocation Tags in the Amazon Web Services Billing and Cost Management User\n Guide, and Controlling Access Using IAM\n Tags.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

" + } + }, + "com.amazonaws.sfn#TagResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + }, + "tags": { + "target": "com.amazonaws.sfn#TagList", + "traits": { + "smithy.api#documentation": "

The list of tags to add to a resource.

\n

Tags may only contain Unicode letters, digits, white space, or these symbols: _ . : / = + - @.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#TagResourceOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#TagValue": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + } + } + }, + "com.amazonaws.sfn#TaskCredentials": { + "type": "structure", + "members": { + "roleArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The ARN of an IAM role that Step Functions assumes for the task. The role can allow cross-account access to resources.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the credentials that Step Functions uses for a task.

" + } + }, + "com.amazonaws.sfn#TaskDoesNotExist": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The activity does not exist.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#TaskFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task failure event.

" + } + }, + "com.amazonaws.sfn#TaskScheduledEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "region": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The region of the scheduled task

", + "smithy.api#required": {} + } + }, + "parameters": { + "target": "com.amazonaws.sfn#ConnectorParameters", + "traits": { + "smithy.api#documentation": "

The JSON data passed to the resource referenced in a task state.\n Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

", + "smithy.api#required": {} + } + }, + "timeoutInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration of the task.

" + } + }, + "heartbeatInSeconds": { + "target": "com.amazonaws.sfn#TimeoutInSeconds", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum allowed duration between two heartbeats for the task.

" + } + }, + "taskCredentials": { + "target": "com.amazonaws.sfn#TaskCredentials", + "traits": { + "smithy.api#documentation": "

The credentials that Step Functions uses for the task.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task scheduled during an execution.

" + } + }, + "com.amazonaws.sfn#TaskStartFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to start during an execution.

" + } + }, + "com.amazonaws.sfn#TaskStartedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the start of a task during an execution.

" + } + }, + "com.amazonaws.sfn#TaskSubmitFailedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task that failed to submit during an execution.

" + } + }, + "com.amazonaws.sfn#TaskSubmittedEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The response from a resource when a task has started. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a task submitted to a resource .

" + } + }, + "com.amazonaws.sfn#TaskSucceededEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The full JSON response from a resource when a task has succeeded. This response becomes\n the output of the related task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "outputDetails": { + "target": "com.amazonaws.sfn#HistoryEventExecutionDataDetails", + "traits": { + "smithy.api#documentation": "

Contains details about the output of an execution history event.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about the successful completion of a task state.

" + } + }, + "com.amazonaws.sfn#TaskTimedOut": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + } + }, + "traits": { + "smithy.api#documentation": "

The task token has either expired or the task associated with the token has already been closed.

", + "smithy.api#error": "client" + } + }, + "com.amazonaws.sfn#TaskTimedOutEventDetails": { + "type": "structure", + "members": { + "resourceType": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The service name of the resource in a task state.

", + "smithy.api#required": {} + } + }, + "resource": { + "target": "com.amazonaws.sfn#Name", + "traits": { + "smithy.api#documentation": "

The action of the resource called by a task state.

", + "smithy.api#required": {} + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error code of the failure.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A more detailed explanation of the cause of the failure.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains details about a resource timeout that occurred during an execution.

" + } + }, + "com.amazonaws.sfn#TaskToken": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 2048 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#TestExecutionStatus": { + "type": "enum", + "members": { + "SUCCEEDED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SUCCEEDED" + } + }, + "FAILED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAILED" + } + }, + "RETRIABLE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "RETRIABLE" + } + }, + "CAUGHT_ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CAUGHT_ERROR" + } + } + } + }, + "com.amazonaws.sfn#TestState": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#TestStateInput" + }, + "output": { + "target": "com.amazonaws.sfn#TestStateOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidExecutionInput" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Accepts the definition of a single state and executes it. You can test a state without creating a state machine or updating an existing state machine. Using this API, you can test the following:

\n \n

You can call this API on only one state at a time. The states that you can test include the following:

\n \n

The TestState API assumes an IAM role which must contain the required IAM permissions for the resources your state is accessing. For information about the permissions a state might need, see IAM permissions to test a state.

\n

The TestState API can run for up to five minutes. If the execution of a state exceeds this duration, it fails with the States.Timeout error.

\n

\n TestState only supports the following when a mock is specified: Activity tasks, .sync or .waitForTaskToken\n service integration patterns, Parallel, or Map states.

", + "smithy.api#endpoint": { + "hostPrefix": "sync-" + } + } + }, + "com.amazonaws.sfn#TestStateConfiguration": { + "type": "structure", + "members": { + "retrierRetryCount": { + "target": "com.amazonaws.sfn#RetrierRetryCount", + "traits": { + "smithy.api#documentation": "

The number of retry attempts that have occurred for the state's Retry that applies to the mocked error.

" + } + }, + "errorCausedByState": { + "target": "com.amazonaws.sfn#TestStateStateName", + "traits": { + "smithy.api#documentation": "

The name of the state from which an error originates when an error is mocked for a Map or Parallel state.

" + } + }, + "mapIterationFailureCount": { + "target": "com.amazonaws.sfn#MapIterationFailureCount", + "traits": { + "smithy.api#documentation": "

The number of Map state iterations that failed during the Map state invocation.

" + } + }, + "mapItemReaderData": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The data read by ItemReader in Distributed Map states as found in its original source.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Contains configurations for the tested state.

" + } + }, + "com.amazonaws.sfn#TestStateInput": { + "type": "structure", + "members": { + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language (ASL) definition of the state or state machine.

", + "smithy.api#required": {} + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the execution role with the required IAM permissions for the state.

" + } + }, + "input": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A string that contains the JSON input data for the state.

" + } + }, + "inspectionLevel": { + "target": "com.amazonaws.sfn#InspectionLevel", + "traits": { + "smithy.api#documentation": "

Determines the values to return when a state is tested. You can specify one of the following types:

\n \n

Each of these levels also provide information about the status of the state execution and the next state to transition to.

" + } + }, + "revealSecrets": { + "target": "com.amazonaws.sfn#RevealSecrets", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Specifies whether or not to include secret information in the test result. For HTTP Tasks, a secret includes the data that an EventBridge connection adds to modify the HTTP request headers, query parameters, and body. Step Functions doesn't omit any information included in the state definition or the HTTP response.

\n

If you set revealSecrets to true, you must make sure that the IAM user that calls the TestState API has permission for the states:RevealSecrets action. For an example of IAM policy that sets the states:RevealSecrets permission, see IAM permissions to test a state. Without this permission, Step Functions throws an access denied error.

\n

By default, revealSecrets is set to false.

" + } + }, + "variables": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

JSON object literal that sets variables used in the state under test. Object keys are the variable names and values are the variable values.

" + } + }, + "stateName": { + "target": "com.amazonaws.sfn#TestStateStateName", + "traits": { + "smithy.api#documentation": "

Denotes the particular state within a state machine definition to be tested. If this field is specified, the definition must contain a fully-formed state machine definition.

" + } + }, + "mock": { + "target": "com.amazonaws.sfn#MockInput", + "traits": { + "smithy.api#documentation": "

Defines a mocked result or error for the state under test.

\n

A mock can only be specified for Task, Map, or Parallel states. If it is specified for another state type, an exception will be thrown.

" + } + }, + "context": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

A JSON string representing a valid Context object for the state under test. This field may only be specified if a mock is specified in the same request.

" + } + }, + "stateConfiguration": { + "target": "com.amazonaws.sfn#TestStateConfiguration", + "traits": { + "smithy.api#documentation": "

Contains configurations for the state under test.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#TestStateOutput": { + "type": "structure", + "members": { + "output": { + "target": "com.amazonaws.sfn#SensitiveData", + "traits": { + "smithy.api#documentation": "

The JSON output data of the state. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding.

" + } + }, + "error": { + "target": "com.amazonaws.sfn#SensitiveError", + "traits": { + "smithy.api#documentation": "

The error returned when the execution of a state fails.

" + } + }, + "cause": { + "target": "com.amazonaws.sfn#SensitiveCause", + "traits": { + "smithy.api#documentation": "

A detailed explanation of the cause for the error when the execution of a state fails.

" + } + }, + "inspectionData": { + "target": "com.amazonaws.sfn#InspectionData", + "traits": { + "smithy.api#documentation": "

Returns additional details about the state's execution, including its input and output data processing flow, and HTTP request and response information. The inspectionLevel request parameter specifies which details are returned.

" + } + }, + "nextState": { + "target": "com.amazonaws.sfn#StateName", + "traits": { + "smithy.api#documentation": "

The name of the next state to transition to. If you haven't defined a next state in your definition or if the execution of the state fails, this field doesn't contain a value.

" + } + }, + "status": { + "target": "com.amazonaws.sfn#TestExecutionStatus", + "traits": { + "smithy.api#documentation": "

The execution status of the state.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#TestStateStateName": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 1, + "max": 80 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#TimeoutInSeconds": { + "type": "long", + "traits": { + "smithy.api#default": 0 + } + }, + "com.amazonaws.sfn#Timestamp": { + "type": "timestamp" + }, + "com.amazonaws.sfn#ToleratedFailureCount": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#ToleratedFailurePercentage": { + "type": "float", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#TooManyTags": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "resourceName": { + "target": "com.amazonaws.sfn#Arn" + } + }, + "traits": { + "smithy.api#documentation": "

You've exceeded the number of tags allowed for a resource. See the Limits Topic in the\n Step Functions Developer Guide.

", + "smithy.api#error": "client", + "smithy.api#httpError": 400 + } + }, + "com.amazonaws.sfn#TraceHeader": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#pattern": "^\\p{ASCII}*$" + } + }, + "com.amazonaws.sfn#TracingConfiguration": { + "type": "structure", + "members": { + "enabled": { + "target": "com.amazonaws.sfn#Enabled", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

When set to true, X-Ray tracing is enabled.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Selects whether or not the state machine's X-Ray tracing is enabled. Default is\n false\n

" + } + }, + "com.amazonaws.sfn#URL": { + "type": "string" + }, + "com.amazonaws.sfn#UnsignedInteger": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#UnsignedLong": { + "type": "long", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0 + } + } + }, + "com.amazonaws.sfn#UntagResource": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UntagResourceInput" + }, + "output": { + "target": "com.amazonaws.sfn#UntagResourceOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + } + ], + "traits": { + "smithy.api#documentation": "

Remove a tag from a Step Functions resource

" + } + }, + "com.amazonaws.sfn#UntagResourceInput": { + "type": "structure", + "members": { + "resourceArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) for the Step Functions state machine or activity.

", + "smithy.api#required": {} + } + }, + "tagKeys": { + "target": "com.amazonaws.sfn#TagKeyList", + "traits": { + "smithy.api#documentation": "

The list of tags to remove from the resource.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UntagResourceOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateMapRun": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateMapRunInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateMapRunOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates an in-progress Map Run's configuration to include changes to the settings that control maximum concurrency and Map Run failure.

" + } + }, + "com.amazonaws.sfn#UpdateMapRunInput": { + "type": "structure", + "members": { + "mapRunArn": { + "target": "com.amazonaws.sfn#LongArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of a Map Run.

", + "smithy.api#required": {} + } + }, + "maxConcurrency": { + "target": "com.amazonaws.sfn#MaxConcurrency", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum number of child workflow executions that can be specified to run in parallel for the Map Run at the same time.

" + } + }, + "toleratedFailurePercentage": { + "target": "com.amazonaws.sfn#ToleratedFailurePercentage", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum percentage of failed items before the Map Run fails.

" + } + }, + "toleratedFailureCount": { + "target": "com.amazonaws.sfn#ToleratedFailureCount", + "traits": { + "smithy.api#default": null, + "smithy.api#documentation": "

The maximum number of failed items before the Map Run fails.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateMapRunOutput": { + "type": "structure", + "members": {}, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachine": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateStateMachineInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateStateMachineOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#InvalidDefinition" + }, + { + "target": "com.amazonaws.sfn#InvalidEncryptionConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidLoggingConfiguration" + }, + { + "target": "com.amazonaws.sfn#InvalidTracingConfiguration" + }, + { + "target": "com.amazonaws.sfn#KmsAccessDeniedException" + }, + { + "target": "com.amazonaws.sfn#KmsThrottlingException" + }, + { + "target": "com.amazonaws.sfn#MissingRequiredParameter" + }, + { + "target": "com.amazonaws.sfn#ServiceQuotaExceededException" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#StateMachineDoesNotExist" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates an existing state machine by modifying its definition,\n roleArn, loggingConfiguration, or EncryptionConfiguration. Running executions will continue\n to use the previous definition and roleArn. You must include at\n least one of definition or roleArn or you will receive a\n MissingRequiredParameter error.

\n

A qualified state machine ARN refers to a Distributed Map state defined within a state machine. For example, the qualified state machine ARN arn:partition:states:region:account-id:stateMachine:stateMachineName/mapStateLabel refers to a Distributed Map state with a label mapStateLabel in the state machine named stateMachineName.

\n

A qualified state machine ARN can either refer to a Distributed Map state defined within a state machine, a version ARN, or an alias ARN.

\n

The following are some examples of qualified and unqualified state machine ARNs:

\n \n

After you update your state machine, you can set the publish parameter to\n true in the same action to publish a new version. This\n way, you can opt-in to strict versioning of your state machine.

\n \n

Step Functions assigns monotonically increasing integers for state machine versions, starting at version number 1.

\n \n \n

All StartExecution calls within a few seconds use the updated\n definition and roleArn. Executions started immediately after you\n call UpdateStateMachine may use the previous state machine\n definition and roleArn.

\n ", + "smithy.api#idempotent": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineAlias": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#UpdateStateMachineAliasInput" + }, + "output": { + "target": "com.amazonaws.sfn#UpdateStateMachineAliasOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ConflictException" + }, + { + "target": "com.amazonaws.sfn#InvalidArn" + }, + { + "target": "com.amazonaws.sfn#ResourceNotFound" + }, + { + "target": "com.amazonaws.sfn#StateMachineDeleting" + }, + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Updates the configuration of an existing state machine alias by modifying its description or routingConfiguration.

\n

You must specify at least one of the description or routingConfiguration parameters to update a state machine alias.

\n \n

\n UpdateStateMachineAlias is an idempotent API. Step Functions bases the\n idempotency check on the stateMachineAliasArn, description, and\n routingConfiguration parameters. Requests with the same parameters return an\n idempotent response.

\n \n \n

This operation is eventually consistent. All StartExecution requests\n made within a few seconds use the latest alias configuration. Executions started immediately\n after calling UpdateStateMachineAlias may use the previous routing\n configuration.

\n \n

\n Related operations:\n

\n " + } + }, + "com.amazonaws.sfn#UpdateStateMachineAliasInput": { + "type": "structure", + "members": { + "stateMachineAliasArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine alias.

", + "smithy.api#required": {} + } + }, + "description": { + "target": "com.amazonaws.sfn#AliasDescription", + "traits": { + "smithy.api#documentation": "

A description of the state machine alias.

" + } + }, + "routingConfiguration": { + "target": "com.amazonaws.sfn#RoutingConfigurationList", + "traits": { + "smithy.api#documentation": "

The routing configuration of the state machine alias.

\n

An array of RoutingConfig objects that specifies up to two state machine versions that the alias starts executions for.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineAliasOutput": { + "type": "structure", + "members": { + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine alias was updated.

", + "smithy.api#required": {} + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineInput": { + "type": "structure", + "members": { + "stateMachineArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the state machine.

", + "smithy.api#required": {} + } + }, + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. See Amazon States Language.

" + } + }, + "roleArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the IAM role of the state machine.

" + } + }, + "loggingConfiguration": { + "target": "com.amazonaws.sfn#LoggingConfiguration", + "traits": { + "smithy.api#documentation": "

Use the LoggingConfiguration data type to set CloudWatch Logs\n options.

" + } + }, + "tracingConfiguration": { + "target": "com.amazonaws.sfn#TracingConfiguration", + "traits": { + "smithy.api#documentation": "

Selects whether X-Ray tracing is enabled.

" + } + }, + "publish": { + "target": "com.amazonaws.sfn#Publish", + "traits": { + "smithy.api#default": false, + "smithy.api#documentation": "

Specifies whether the state machine version is published. The default is\n false. To publish a version after updating the state machine, set\n publish to true.

" + } + }, + "versionDescription": { + "target": "com.amazonaws.sfn#VersionDescription", + "traits": { + "smithy.api#documentation": "

An optional description of the state machine version to publish.

\n

You can only specify the versionDescription parameter if you've set publish to true.

" + } + }, + "encryptionConfiguration": { + "target": "com.amazonaws.sfn#EncryptionConfiguration", + "traits": { + "smithy.api#documentation": "

Settings to configure server-side encryption.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#UpdateStateMachineOutput": { + "type": "structure", + "members": { + "updateDate": { + "target": "com.amazonaws.sfn#Timestamp", + "traits": { + "smithy.api#documentation": "

The date and time the state machine was updated.

", + "smithy.api#required": {} + } + }, + "revisionId": { + "target": "com.amazonaws.sfn#RevisionId", + "traits": { + "smithy.api#documentation": "

The revision identifier for the updated state machine.

" + } + }, + "stateMachineVersionArn": { + "target": "com.amazonaws.sfn#Arn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the published state machine version.

\n

If the publish parameter isn't set to true, this field returns null.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinition": { + "type": "operation", + "input": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionInput" + }, + "output": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionOutput" + }, + "errors": [ + { + "target": "com.amazonaws.sfn#ValidationException" + } + ], + "traits": { + "smithy.api#documentation": "

Validates the syntax of a state machine definition specified in Amazon States Language (ASL), a\n JSON-based, structured language.

\n

You can validate that a state machine definition is correct without creating a state\n machine resource.

\n

Suggested uses for ValidateStateMachineDefinition:

\n \n

Validation will look for problems in your state machine definition and return a\n result and a list of diagnostic\n elements.

\n

The result value will be OK when your\n workflow definition can be successfully created or updated. Note the result can be\n OK even when diagnostic warnings are present in the response. The\n result value will be FAIL when the\n workflow definition contains errors that would prevent you from creating or updating\n your state machine.

\n

The list of ValidateStateMachineDefinitionDiagnostic data elements can contain zero or more WARNING and/or ERROR elements.

\n \n

The ValidateStateMachineDefinition API might add\n new diagnostics in the future, adjust diagnostic codes, or change the message\n wording. Your automated processes should only rely on the value of the result field value (OK, FAIL). Do not rely on the exact order, count, or\n wording of diagnostic messages.

\n " + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionCode": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnostic": { + "type": "structure", + "members": { + "severity": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity", + "traits": { + "smithy.api#documentation": "

A value of ERROR means that you cannot create or update a state machine with this definition.

\n

\n WARNING level diagnostics alert you to potential issues, but they will not prevent you from creating or updating your state machine.

", + "smithy.api#required": {} + } + }, + "code": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionCode", + "traits": { + "smithy.api#documentation": "

Identifying code for the diagnostic.

", + "smithy.api#required": {} + } + }, + "message": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionMessage", + "traits": { + "smithy.api#documentation": "

Message describing the diagnostic condition.

", + "smithy.api#required": {} + } + }, + "location": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionLocation", + "traits": { + "smithy.api#documentation": "

Location of the issue in the state machine, if available.

\n

For errors specific to a field, the location could be in the format: /States//, for example: /States/FailState/ErrorPath.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Describes potential issues found during state machine validation. Rather than raise an\n exception, validation will return a list of diagnostic\n elements containing diagnostic information.

\n \n

The ValidateStateMachineDefinitionlAPI might add\n new diagnostics in the future, adjust diagnostic codes, or change the message\n wording. Your automated processes should only rely on the value of the result field value (OK, FAIL). Do not rely on the exact order, count, or\n wording of diagnostic messages.

\n \n

\n List of warning codes\n

\n
\n
NO_DOLLAR
\n
\n

No .$ on a field that appears to be a JSONPath or Intrinsic Function.

\n
\n
NO_PATH
\n
\n

Field value looks like a path, but field name does not end with 'Path'.

\n
\n
PASS_RESULT_IS_STATIC
\n
\n

Attempt to use a path in the result of a pass state.

\n
\n
\n

\n List of error codes\n

\n
\n
INVALID_JSON_DESCRIPTION
\n
\n

JSON syntax problem found.

\n
\n
MISSING_DESCRIPTION
\n
\n

Received a null or empty workflow input.

\n
\n
SCHEMA_VALIDATION_FAILED
\n
\n

Schema validation reported errors.

\n
\n
INVALID_RESOURCE
\n
\n

The value of a Task-state resource field is invalid.

\n
\n
MISSING_END_STATE
\n
\n

The workflow does not have a terminal state.

\n
\n
DUPLICATE_STATE_NAME
\n
\n

The same state name appears more than once.

\n
\n
INVALID_STATE_NAME
\n
\n

The state name does not follow the naming convention.

\n
\n
STATE_MACHINE_NAME_EMPTY
\n
\n

The state machine name has not been specified.

\n
\n
STATE_MACHINE_NAME_INVALID
\n
\n

The state machine name does not follow the naming convention.

\n
\n
STATE_MACHINE_NAME_TOO_LONG
\n
\n

The state name exceeds the allowed length.

\n
\n
STATE_MACHINE_NAME_ALREADY_EXISTS
\n
\n

The state name already exists.

\n
\n
DUPLICATE_LABEL_NAME
\n
\n

A label name appears more than once.

\n
\n
INVALID_LABEL_NAME
\n
\n

You have provided an invalid label name.

\n
\n
MISSING_TRANSITION_TARGET
\n
\n

The value of \"Next\" field doesn't match a known state name.

\n
\n
TOO_DEEPLY_NESTED
\n
\n

The states are too deeply nested.

\n
\n
" + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnosticList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnostic" + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionInput": { + "type": "structure", + "members": { + "definition": { + "target": "com.amazonaws.sfn#Definition", + "traits": { + "smithy.api#documentation": "

The Amazon States Language definition of the state machine. For more information, see\n Amazon States Language (ASL).

", + "smithy.api#required": {} + } + }, + "type": { + "target": "com.amazonaws.sfn#StateMachineType", + "traits": { + "smithy.api#documentation": "

The target type of state machine for this definition. The default is STANDARD.

" + } + }, + "severity": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity", + "traits": { + "smithy.api#documentation": "

Minimum level of diagnostics to return. ERROR returns only ERROR diagnostics, whereas WARNING returns both WARNING and ERROR diagnostics. The default is ERROR.

" + } + }, + "maxResults": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionMaxResult", + "traits": { + "smithy.api#default": 0, + "smithy.api#documentation": "

The maximum number of diagnostics that are returned per call. The default and maximum value is 100. Setting the value to 0 will also use the default of 100.

\n

If the number of diagnostics returned in the response exceeds maxResults, the value of the truncated field in the response will be set to true.

" + } + } + }, + "traits": { + "smithy.api#input": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionLocation": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionMaxResult": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionMessage": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionOutput": { + "type": "structure", + "members": { + "result": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionResultCode", + "traits": { + "smithy.api#documentation": "

The result value will be OK when no syntax errors are found, or\n FAIL if the workflow definition does not pass verification.

", + "smithy.api#required": {} + } + }, + "diagnostics": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionDiagnosticList", + "traits": { + "smithy.api#documentation": "

An array of diagnostic errors and warnings found during validation of the state machine definition. Since warnings do not prevent deploying your workflow definition, the result value could be OK even when warning diagnostics are present in the response.

", + "smithy.api#required": {} + } + }, + "truncated": { + "target": "com.amazonaws.sfn#ValidateStateMachineDefinitionTruncated", + "traits": { + "smithy.api#documentation": "

The result value will be true if the number of diagnostics found in the workflow definition exceeds maxResults. When all diagnostics results are returned, the value will be false.

" + } + } + }, + "traits": { + "smithy.api#output": {} + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionResultCode": { + "type": "enum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "OK" + } + }, + "FAIL": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "FAIL" + } + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionSeverity": { + "type": "enum", + "members": { + "ERROR": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "ERROR" + } + }, + "WARNING": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "WARNING" + } + } + } + }, + "com.amazonaws.sfn#ValidateStateMachineDefinitionTruncated": { + "type": "boolean" + }, + "com.amazonaws.sfn#ValidationException": { + "type": "structure", + "members": { + "message": { + "target": "com.amazonaws.sfn#ErrorMessage" + }, + "reason": { + "target": "com.amazonaws.sfn#ValidationExceptionReason", + "traits": { + "smithy.api#documentation": "

The input does not satisfy the constraints specified by an Amazon Web Services service.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

The input does not satisfy the constraints specified by an Amazon Web Services service.

", + "smithy.api#error": "client", + "smithy.api#httpError": 400 + } + }, + "com.amazonaws.sfn#ValidationExceptionReason": { + "type": "enum", + "members": { + "API_DOES_NOT_SUPPORT_LABELED_ARNS": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "API_DOES_NOT_SUPPORT_LABELED_ARNS" + } + }, + "MISSING_REQUIRED_PARAMETER": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "MISSING_REQUIRED_PARAMETER" + } + }, + "CANNOT_UPDATE_COMPLETED_MAP_RUN": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "CANNOT_UPDATE_COMPLETED_MAP_RUN" + } + }, + "INVALID_ROUTING_CONFIGURATION": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "INVALID_ROUTING_CONFIGURATION" + } + } + } + }, + "com.amazonaws.sfn#VariableName": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VariableNameList": { + "type": "list", + "member": { + "target": "com.amazonaws.sfn#VariableName" + } + }, + "com.amazonaws.sfn#VariableReferences": { + "type": "map", + "key": { + "target": "com.amazonaws.sfn#StateName" + }, + "value": { + "target": "com.amazonaws.sfn#VariableNameList" + }, + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VariableValue": { + "type": "string", + "traits": { + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VersionDescription": { + "type": "string", + "traits": { + "smithy.api#length": { + "min": 0, + "max": 256 + }, + "smithy.api#sensitive": {} + } + }, + "com.amazonaws.sfn#VersionWeight": { + "type": "integer", + "traits": { + "smithy.api#default": 0, + "smithy.api#range": { + "min": 0, + "max": 100 + } + } + }, + "com.amazonaws.sfn#includedDetails": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + }, + "com.amazonaws.sfn#truncated": { + "type": "boolean", + "traits": { + "smithy.api#default": false + } + } + } +} \ No newline at end of file diff --git a/codegen/smithy-model/stepfunctions.provenance.json b/codegen/smithy-model/stepfunctions.provenance.json new file mode 100644 index 00000000..e677fca5 --- /dev/null +++ b/codegen/smithy-model/stepfunctions.provenance.json @@ -0,0 +1,10 @@ +{ + "repository": "https://github.com/aws/api-models-aws", + "commit": "18e485ce1c8dbf29d47e5ce00d9b8513da95912c", + "path": "models/sfn/service/2016-11-23/sfn-2016-11-23.json", + "sha256": "38dd64c074f5c05320978970dafa6e0f199ab2e33b6065232002e8a603c5f6ed", + "service": "com.amazonaws.sfn#AWSStepFunctions", + "serviceVersion": "2016-11-23", + "retrievedAt": "2026-10-25", + "bytes": 370185 +} diff --git a/docs/index.md b/docs/index.md index 98752c12..72402b8f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,9 @@ ## Research +- [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. +- [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. +- [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. - [LocalStack S3 Implementation Research](./research/localstack-s3-research.md) - Comprehensive analysis of what a Rust-based LocalStack S3 service needs to support for feature parity. - [s3s Crate Ecosystem Research](./research/s3s-crate-research.md) - Comprehensive analysis of the s3s Rust crate for building S3-compatible services. - [LocalStack Container & CI Research](./research/localstack-container-ci-research.md) - How LocalStack's Docker container works, gateway routing through port 4566, health endpoints, and the official GitHub Action for CI. diff --git a/docs/research/spike-lambda-container-relay.md b/docs/research/spike-lambda-container-relay.md new file mode 100644 index 00000000..4715cd8f --- /dev/null +++ b/docs/research/spike-lambda-container-relay.md @@ -0,0 +1,41 @@ +# Spike: Can Docker Attach Drive an Unmodified Lambda RIC Image? + +Status: PASS with platform qualifications · Date: 2026-10-25 · Consumer: [runtime design](../../specs/rustack-workflows-20-lambda-runtime-design.md) + +## Question + +Can a relay own an environment-local loopback Runtime API and exchange binary control frames through Docker attach while an ordinary Lambda image runs in the same network namespace, without published Runtime API ports or host networking? + +## Method + +The retained standalone crate is [tools/workflows-spikes](../../tools/workflows-spikes/Cargo.toml), with [relay](../../tools/workflows-spikes/src/bin/relay.rs), [driver](../../tools/workflows-spikes/src/bin/driver.rs), pinned [relay image](../../tools/workflows-spikes/Relay.Dockerfile) and [official-RIC fixture image](../../tools/workflows-spikes/Handler.Dockerfile). It is an executable feasibility experiment, not the production executor or hardened external CBOR parser. + +Host: macOS 27 arm64, Apple M5, 64 GiB RAM. Docker CLI 29.9.2, Engine 29.5.2, Colima 0.10.3, Lima 2.2.1, QEMU 11.1.2 and Cargo Lambda 1.9.2. A dedicated arm64 VZ VM uses 4 CPUs/6 GiB; a separate x86_64 QEMU VM uses 2 CPUs/4 GiB. Neither mounts the host home or forwards its SSH agent. The x86_64 VM is **emulated hardware**, not native x86 performance evidence. Docker Desktop, rootless Docker and a native x86 runner remain required M4 lanes. + +Reproduction requires Docker, Cargo Lambda/Zig, and buildx. On Homebrew, set the builder executable explicitly if Docker plugin discovery is unconfigured: + +```sh +make workflows-relay-spike \ + WORKFLOWS_DOCKER_SOCKET=/path/to/docker.sock \ + WORKFLOWS_ARCH=arm64 \ + WORKFLOWS_IMAGE_BUILDER=/opt/homebrew/bin/docker-buildx +``` + +Repeat with amd64 and its own socket. Make builds the Linux relay, builds both digest-pinned fixture images, runs actual invocations and removes all created containers. It never starts another AWS emulator or contacts an AWS service API. Registry image provisioning still requires network access on a cold cache. + +## Findings + +1. Binary, non-TTY, bidirectional Docker attach works with the selected Bollard API. Attach is established before relay start; Hello and Ready are received without relying on replayed logs. The RIC's first GET /next produces a distinct BootstrapReady signal. +2. The official Python 3.13 Lambda image enters its normal RIC path when AWS_LAMBDA_RUNTIME_API points to 127.0.0.1:9001 in the shared netns. Two calls compute 42 and 7 from different input values; the handler's counter advances 1→2. These are real handler results, not a transport echo. +3. Handler-side evidence confirms UID 1000, memory.max=201326592, pids.max=256 and read-only rootfs. The driver independently checks that the relay publishes no ports. This proves these controls in the measured VMs; it does not establish hostile multi-tenant isolation. +4. A raised ValueError travels through `/error`. The cancellation probe waits for the exact invocation's busy-start log marker before removing the sleeping function. Close/Ack succeeds, followed by removal of both owned containers. A single successful response is therefore not the entire evidence. +5. Seven framing tests cover fragmented binary data, oversized prefixes, trailing CBOR data, clean/truncated EOF and redacted Debug; five doc examples compile/pass. The standalone crate passes strict pedantic/boundary Clippy and rustdoc. Its own dependency audit reports zero vulnerabilities. +6. Initial single-run cold observations were 122 ms on arm64 and 2,526 ms on emulated amd64. These are feasibility observations under development load, **not p95 performance qualification**. Retained normalized run evidence is [arm64](../../tests/workflows/evidence/relay-arm64.json) and [emulated amd64](../../tests/workflows/evidence/relay-amd64.json), including EOF and label-reconciliation observations. + +## Decision + +**GO with qualifications:** retain the relay architecture and pay its two-container overhead in the environment budget. Configure DNS/extra-host entries on the relay/netns owner. Keep control transport separate from function SDK gateway reachability. The experiment is not permission to copy its reduced protocol validation or failure handling into production unchanged. + +## Risks and implementation requirements + +Production P1 still needs strict bounded CBOR preflight, typed variants, complete Runtime API identity/error behavior, cancellation supervision, quarantine/reconciliation, image revision lifecycle and dependency-log filtering. The real attach EOF comparison found StdinOnce=false leaves the relay running after input shutdown; true delivers EOF and ends it before cleanup. The production contract now requires true. Cleanup also reconciles a run-unique label, including a deliberately unrecorded create result, instead of relying only on received IDs. The experiment caught a Close/read-worker race: a reader must not terminate the environment before the owner consumes queued Close. Retain single-owner settlement in production. SIGKILL cleanup remains a startup reconciliation responsibility; there is no crash-recovery claim. diff --git a/docs/research/spike-stepfunctions-semantic-oracle.md b/docs/research/spike-stepfunctions-semantic-oracle.md new file mode 100644 index 00000000..0d377c64 --- /dev/null +++ b/docs/research/spike-stepfunctions-semantic-oracle.md @@ -0,0 +1,36 @@ +# Spike: Independent Step Functions Wire and Semantic Evidence + +Status: PASS with a confirmed generator remediation · Date: 2026-10-25 · Consumer: [API](../../specs/rustack-workflows-10-api-design.md), [ASL](../../specs/rustack-workflows-30-asl-design.md), [integrations](../../specs/rustack-workflows-40-integrations-design.md) + +## Question and method + +Can the selected STANDARD/JSONPath contracts be pinned to independent sources and ordinary SDK behavior before writing the engine? + +The [fixture catalog](../../tests/workflows/semantic-cases.json) contains 48 document-derived expectations, not interpreter test results. [Provenance](../../tests/workflows/provenance.json) pins SDK/dependency versions. Four [offline botocore tests](../../tests/workflows/test_botocore_wire.py) exercise its serializer/parser without creating a client, resolving credentials or contacting AWS. Run `make workflows-fixture-check workflows-wire-probe`. + +The [SFN Smithy model](../../codegen/smithy-model/stepfunctions.json) is pinned to aws/api-models-aws commit `18e485ce1c8dbf29d47e5ce00d9b8513da95912c`, SHA-256 `38dd64c074f5c05320978970dafa6e0f199ab2e33b6065232002e8a603c5f6ed`, 370,185 bytes. `make workflows-model-check` verifies its hash, protocol, signing name and 37-operation service. `make workflows-codegen-spike` independently generates six Rust files into `/tmp/rustack-spikes/stepfunctions-generated/src` from 312 shapes. + +The existing LocalStack submodule was initialized at its already-pinned commit `45dfee8ce27df00fcac3f91b8071c8156eb5c5c7`; its gitlink was not changed. Only relevant test/snapshot evidence was inspected, not the full interpreter architecture. No implementation code was copied. + +## Findings + +| Evidence | Result and binding consequence | +|---|---| +| botocore 1.43.118 / boto3 1.43.118 | StartExecution emits AWSStepFunctions.StartExecution and preserves the raw input JSON string | +| SDK numeric timestamp parsing |0.125→1970-01-01T00:00:00.125Z;−0.125→1969-12-31T23:59:59.875Z. HTTP timestamps are numeric fractional epoch seconds | +| Current generated CreateStateMachineOutput | A standalone Rust test confirms numeric creationDate fails ordinary chrono deserialization. Generation succeeds, but its current wire codec is insufficient | +| Callback error parser and current API docs | Known closed/expired tasks use TaskTimedOut; unknown/malformed tokens use InvalidToken. TaskDoesNotExist describes a nonexistent activity | +| Current AWS error guide | States.ALL excludes Runtime/DataLimitExceeded; explicit DataLimitExceeded matching is supported. Older upstream expectations cannot override this current documented contract | +| AWS-validated nested snapshots | `.sync` has string Input/Output; `.sync:2` has JSON Input/Output. Both include a TaskSubmitted event between TaskStarted and terminal completion; optimized null Input normalizes to an empty object | + +Nested evidence is directly inspectable in the [validated tests](../../vendors/localstack/tests/aws/services/stepfunctions/v2/callback/test_callback.py) at lines 285/333; paired output shapes are in [snapshots](../../vendors/localstack/tests/aws/services/stepfunctions/v2/callback/test_callback.snapshot.json) at lines 713/1251, with validation dates 2023-06-30 and2023-08-12 in the [validation record](../../vendors/localstack/tests/aws/services/stepfunctions/v2/callback/test_callback.validation.json). Snapshot timestamps are normalized and do not establish a date unit. A [historic direct AWS execution report](https://github.com/aws/aws-cdk/issues/5474) records optimized nested dates as epoch milliseconds; this is corroborating historical evidence, not a current cloud run. The local contract explicitly selects milliseconds for optimized envelopes while HTTP remains epoch seconds. + +Primary references: [Smithy JSON timestamp serialization](https://smithy.io/2.0/aws/protocols/aws-json-1_0-protocol.html#shape-serialization), [AWS callback errors](https://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskSuccess.html), [error handling](https://docs.aws.amazon.com/step-functions/latest/dg/concepts-error-handling.html), [nested integrations](https://docs.aws.amazon.com/step-functions/latest/dg/connect-stepfunctions.html). Individual cases identify their sources. + +## Decision + +**GO with concrete remediation:** keep the API/profile and correct the reviewed contracts before consumers. P2 task2.1 must add protocol-aware numeric timestamp generation and required/optional/fractional/negative round-trip tests before the model becomes a production crate. Do not hand-edit generated output or copy the existing KMS timestamp adapter's unchecked negative-fraction conversion. The transformation and intrinsic expectations remain independent fixtures for the future interpreter; passing catalog/SDK checks does not mark A04–A11 complete. + +## Evidence limits + +No live AWS differential suite was run, and no complete emulator was used as a semantic oracle. Historical AWS-validated snapshots supplement current official documentation only where they do not conflict. Current guide-compatible States.Runtime behavior is selected for unevaluable paths; internal Path/Intrinsic/ResultPath diagnostics remain separate. Native x86 hardware, Docker Desktop/rootless qualification, full execution-history parity and the final performance gates remain unverified. diff --git a/docs/research/survey-stepfunctions-lambda-container-runtime.md b/docs/research/survey-stepfunctions-lambda-container-runtime.md new file mode 100644 index 00000000..9f039a5a --- /dev/null +++ b/docs/research/survey-stepfunctions-lambda-container-runtime.md @@ -0,0 +1,64 @@ +# Survey: Step Functions and Container Lambda Compatibility + +Status: complete documentation survey; no Docker runtime experiment performed · Date: 2026-10-25 · Consumer: [Workflows PRD](../../specs/rustack-workflows-00-prd.md) + +## 1. Question and outcome + +Can Rustack interpret ASL itself and run the Lambda images deployed by other projects without depending on another AWS emulator? Yes, provided it implements three distinct contracts: workflow semantics, Lambda execution environments, and the invocation bridge. RIE is not a workflow engine; a successfully started container is not a successful invocation. + +The existing [Cargo Lambda spike](./spike-cargo-lambda-runtime-execution.md) validates ZIP/bootstrap/Runtime API behavior. This survey adds official protocol and Docker API documentation. No new upstream implementation source was inspected, so no vendor submodule was added. The current `vendors/localstack` directory is uninitialized and is not evidence of an implementation review. + +## 2. Inspected repository facts + +| Fact | Evidence | Consequence | +|---|---|---| +| Executor already supports Native/Auto/Squib; Docker is unavailable | [executor/mod.rs](../../crates/rustack-lambda-core/src/executor/mod.rs), [config.rs](../../crates/rustack-lambda-core/src/config.rs) | Add a backend incrementally; retain disabled default | +| Runtime API binds host 127.0.0.1:0 per instance | [runtime_api.rs](../../crates/rustack-lambda-core/src/executor/runtime_api.rs) | Changing the container hostname cannot make that host loopback socket reachable | +| Provider separately rejects Docker | [provider.rs](../../crates/rustack-lambda-core/src/provider.rs:927) | Adding an executor file alone cannot enable invocation | +| Invocation request omits ImageConfig/ephemeral storage, although versions store them | [types.rs](../../crates/rustack-lambda-core/src/executor/types.rs), [storage.rs](../../crates/rustack-lambda-core/src/storage.rs:115) | Capture a complete versioned execution revision | +| Cross-service adapters live in the binary | [lambda_s3_bridge.rs](../../apps/rustack/src/lambda_s3_bridge.rs) | Workflow core should not depend directly on Lambda core | +| Worker lifecycle is independent of snapshot participation | [runtime.rs](../../apps/rustack/src/runtime.rs), [snapshot.rs](../../apps/rustack/src/snapshot.rs) | Register workflow workers, not just an HTTP router | + +## 3. Protocol findings + +ASL now includes JSONPath, JSONata and variable syntax. The initial release therefore needs an explicit STANDARD + JSONPath profile and precise unsupported-feature diagnostics, rather than an unrestricted ASL claim. [ASL specification](https://states-language.net/spec.html), [workflow types](https://docs.aws.amazon.com/step-functions/latest/dg/choosing-workflow-type.html). + +AWS limits definitions to 1 MiB, state payloads to 256 KiB and Standard history to 25,000 events. Inline Map allows up to 40 concurrent iterations. Local memory/concurrency budgets must be separately documented. [Quotas](https://docs.aws.amazon.com/step-functions/latest/dg/service-quotas.html), [Inline Map](https://docs.aws.amazon.com/step-functions/latest/dg/state-map-inline.html). + +Ordinary optimized Lambda invocation waits for its synchronous result. Lambda does not support `.sync`, but supports callbacks. Nested Step Functions supports startExecution.sync and .sync:2, with string versus JSON Output respectively. [Integration matrix](https://docs.aws.amazon.com/step-functions/latest/dg/integrate-optimized.html), [Lambda integration](https://docs.aws.amazon.com/step-functions/latest/dg/connect-lambda.html), [nested workflows](https://docs.aws.amazon.com/step-functions/latest/dg/connect-stepfunctions.html). + +HTTP success does not establish Task success: Lambda can report FunctionError with HTTP200. Task deadlines differ from Lambda execution deadlines. Current AWS documentation allows explicitly matching States.DataLimitExceeded while excluding it from States.ALL; older statements that it is always uncatchable are insufficient. States.Runtime remains non-retriable. [Error handling](https://docs.aws.amazon.com/step-functions/latest/dg/concepts-error-handling.html). + +The Smithy service is com.amazonaws.sfn#AWSStepFunctions, version 2016-11-23, awsJson1_0, SigV4 states. Standard StartExecution is idempotent for a still-running execution with the same name/input; closed or differently supplied input returns ExecutionAlreadyExists. Names remain reserved 90 days after closure. Input is a JSON-bearing string on the wire. [Official model](https://raw.githubusercontent.com/aws/api-models-aws/main/models/sfn/service/2016-11-23/sfn-2016-11-23.json), [StartExecution](https://docs.aws.amazon.com/step-functions/latest/apireference/API_StartExecution.html). + +## 4. Image and Runtime API findings + +Images must target Linux, implement the Lambda Runtime API and tolerate a read-only root filesystem. ImageConfig overrides ENTRYPOINT/CMD/WORKDIR at the argv level. Resolve image tags to digests on deployment; moving a tag afterward does not update the function. [Image requirements](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html), [ImageConfig](https://docs.aws.amazon.com/lambda/latest/api/API_ImageConfig.html), [UpdateFunctionCode](https://docs.aws.amazon.com/lambda/latest/api/API_UpdateFunctionCode.html). + +Runtime API2018-06-01 defines next/response/error/init-error. Current documentation adds an optional Lambda-Runtime-Invocation-Id: supplied echoes must match, while older RICs can omit it. The proposed release covers single-concurrency Event Functions, not Managed Instances. [Runtime API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html). + +## 5. Docker design evidence and remaining experiment + +Docker documents `network=container:` namespace sharing. A small relay can own an environment-local loopback Runtime API and exchange tasks over Docker attach, avoiding host Runtime API publishing. **This full composition is a design inference, not a completed runtime experiment.** P0 must test attach framing/EOF, official RIC behavior, rootless/Desktop limits and cleanup. [Networking](https://docs.docker.com/engine/network/), [rootless](https://docs.docker.com/engine/security/rootless/). + +Control transport and handler SDK access are different paths. Docker attach can eliminate host reachability requirements for control traffic; handlers still need an operator-configured reachable gateway endpoint. No published Runtime API port does not mean arbitrary localhost SDK endpoints work inside a container. + +## 6. Dependency check + +Official documentation returned these versions on 2026-10-25; implementation must recheck them. + +| Dependency | Observed version/API | Recommendation | +|---|---|---| +| [bollard](https://docs.rs/bollard/latest/bollard/) | 0.21.1; generated query parameters, API negotiation, standalone pipe feature | Replace the old ~0.18 guidance; initial transport uses pipe only, no remote Docker | +| [winnow](https://docs.rs/winnow/latest/winnow/) | 1.0.4 parser combinators | Parse bounded Path/intrinsic grammar rather than embed a general expression engine | +| Tokio/bytes/ciborium/thiserror/serde/config/uuid | Already present in workspace | Reuse existing dependencies before adding a database or another async runtime | + +This survey is not a completed vulnerability audit, MSRV check or release build. New dependencies still require AGENTS.md audit/deny and feature review. + +## 7. Risk closure artifacts + +P0 must produce `spike-lambda-container-relay.md` for netns/attach/official RIC/native architectures, and `spike-stepfunctions-semantic-oracle.md` for source hashes, SDK versions and semantic fixtures. These are planned artifacts, not existing passing evidence. The implementation plan assigns their exit gates. + +## P0 follow-up + +The implementation run initialized the existing LocalStack gitlink without moving it, executed the retained relay experiments, and corrected the earlier protocol assumptions. See the [actual relay spike](./spike-lambda-container-relay.md) and [semantic oracle](./spike-stepfunctions-semantic-oracle.md). The original survey's lack-of-runtime-evidence statement is historical; these later memos carry the measured evidence and its limits. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9c2ee376..35440c48 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.99.0" +channel = "1.99.1" components = ["clippy", "rustfmt"] profile = "minimal" diff --git a/specs/index.md b/specs/index.md index 99434ca9..64b63d82 100644 --- a/specs/index.md +++ b/specs/index.md @@ -2,6 +2,43 @@ 本文件是 specs 的导航入口；文件中的 Draft / Implemented 状态以各原文为准，不代表经过本次运行验收。 +## Step Functions and Lambda Runtime — new specification set + +Status: implementation contract dated 2026-10-25; no new runtime implementation is claimed. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. + +| Order | Specification | Type / purpose | +|---|---|---| +| 00 | [Workflows PRD](./rustack-workflows-00-prd.md) | Product goals, compatibility profile and complete M0–M4 scope | +| 10 | [API and data model](./rustack-workflows-10-api-design.md) | Smithy protocol, resource/revision identity, idempotency and execution history | +| 20 | [Lambda runtime](./rustack-workflows-20-lambda-runtime-design.md) | Image resolution, Docker relay/runtime, immutable revisions, pooling and cleanup | +| 30 | [ASL engine](./rustack-workflows-30-asl-design.md) | Compiler, JSONPath data flow, eight state types, retries and deadlines | +| 40 | [Task integrations](./rustack-workflows-40-integrations-design.md) | Real Lambda invocation, nested `.sync`/`.sync:2`, callbacks and heartbeats | +| 60 | [Operations](./rustack-workflows-60-operations-design.md) | YAML configuration, budgets, topology, shutdown and snapshots | +| 70 | [Verification plan](./rustack-workflows-70-verification-plan.md) | Acceptance matrix, security boundaries, SDK/platform coverage and performance budgets | +| 80 | [Roadmap](./rustack-workflows-80-roadmap.md) | User-visible M0–M4 outcomes and calendar assumptions | +| 90 | [Implementation plan](./rustack-workflows-90-impl-plan.md) | P0 evidence gate and dependency-ordered P1–P5 tasks/exit criteria | +| 93 | [Implementation review](./rustack-workflows-93-review.md) | Independent findings, corrections and required phase/security follow-ups | +| 99 | [Key decisions and glossary](./rustack-workflows-99-decisions.md) | Alternatives, rationale, ownership and overloaded terms | + +Read 00 → 10 → 20 → 30 → 40; apply 60/70 throughout. Read 80 for delivery planning and 90 for the next engineering step. The [supporting survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) records source verification and remaining runtime experiments. + +```text +┌─ P0: runtime experiment + semantic fixtures + pinned sources ───────┐ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M0 / P1: image Lambda ──────┐ ┌─ M1 / P2: API + ASL spine ─────┐ +│ actual code + owned cleanup │──▶│ real Lambda tasks + history │ +└─────────────────────────────┘ └──────────────┬───────────────────┘ + ▼ +┌─ M2 / P3: Parallel/Map + nested .sync ──────────────────────────────┐ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ M3 / P4: callbacks + heartbeat +┌─ M4 / P5: ZIP + snapshots + consuming-project CI qualification ─────┐ +└───────────────────────────────────────────────────────────────────┘ +``` + +The new set supersedes conflicting historical Docker plans while preserving the hardening contracts. Ordinary Lambda invocation already waits for its result; `.sync` is implemented for nested Step Functions, not an invalid `lambda:invoke.sync` resource. + ## 推荐阅读顺序 1. [系统审查与改进 spec](./rustack-system-review.md)：安全、正确性、架构及开发者体验；按 R01–R18 定位问题，按 M0–M4 安排修复。 @@ -42,7 +79,8 @@ | SQS | [设计](./ruststack-sqs-design.md)、[Long-poll / DashMap safety fix](./fixes/sqs-longpoll-dashmap-safety.md) | | SSM | [Parameter Store](./ruststack-ssm-design.md) | | SNS | [设计](./ruststack-sns-design.md) | -| Lambda | [服务设计](./ruststack-lambda-design.md)、[Executor](./ruststack-lambda-executor-design.md)、[Squib runtime](./ruststack-lambda-squib-runtime-design.md)、[S3 code packages](./ruststack-lambda-s3-code-design.md) | +| Lambda | [Current runtime target](./rustack-workflows-20-lambda-runtime-design.md)、[Hardening](./rustack-lambda-hardening-design.md)、[服务设计](./ruststack-lambda-design.md)、[Executor](./ruststack-lambda-executor-design.md)、[Squib runtime](./ruststack-lambda-squib-runtime-design.md)、[S3 code packages](./ruststack-lambda-s3-code-design.md) | +| Step Functions | [PRD](./rustack-workflows-00-prd.md), [ASL engine](./rustack-workflows-30-asl-design.md), [integrations](./rustack-workflows-40-integrations-design.md), [implementation plan](./rustack-workflows-90-impl-plan.md) | | EventBridge | [设计](./ruststack-events-design.md) | | CloudWatch | [Metrics](./ruststack-cloudwatch-design.md)、[Logs](./ruststack-logs-design.md) | | KMS | [设计](./ruststack-kms-design.md) | diff --git a/specs/rustack-workflows-00-prd.md b/specs/rustack-workflows-00-prd.md new file mode 100644 index 00000000..57eea870 --- /dev/null +++ b/specs/rustack-workflows-00-prd.md @@ -0,0 +1,92 @@ +# Rustack Workflows: Step Functions and Real Lambda Execution + +Status: implementation contract; not implemented · Date: 2026-10-25 · Delivery: M0–M4 + +## 1. Problem and vision + +Other projects need to test real serverless workflows locally and in CI: deploy the same Lambda container images used on AWS, create state machines through ordinary AWS SDKs, exercise branching, retries, timeouts and child workflows, and inspect actual outputs and execution histories. Rustack has a Lambda control plane and Runtime API, but explicitly rejects Docker execution and has no Step Functions engine. Returning plausible API responses or echoing inputs cannot validate function code or workflow behavior. + +Rustack will own an ASL interpreter and Docker Lambda execution environments, reusing its gateway, Lambda provider, immutable revisions and snapshots. Clients change their AWS endpoint rather than rewrite production workflows. The complete release profile is **STANDARD workflows, an explicit JSONPath subset, all eight state types, real image Lambdas, Retry/Catch/timeouts, Parallel/Inline Map, nested `.sync`, and Lambda callbacks**. No Step Functions Local, LocalStack, Temporal or other replacement workflow engine is required. + +## 2. Existing implementation and specification authority + +The [research survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) records inspected code and official sources. These documents describe future work; historical Implemented labels are not new verification evidence. + +This specification set: + +- Preserves [Lambda hardening](./rustack-lambda-hardening-design.md): execution defaults to disabled, artifacts are immutable, and admission and cleanup remain bounded. +- Supersedes the RIE-forwarding architecture in the [original Lambda design](./ruststack-lambda-design.md), and the old Docker networking, dependency versions and broad cleanup rules in the [executor design](./ruststack-lambda-executor-design.md). +- Preserves Native and Squib as independent backends. Image packages require Docker; they never silently fall back to Native or Squib. +- Extends [runtime hardening](./rustack-runtime-hardening-design.md) worker ordering and [snapshot coverage](./ruststack-snapshot-binary-archive-design.md). It does not add a crash-consistent execution log. + +## 3. Users and measurable goals + +Primary users maintain Rust, Python or Node Lambda projects with CDK/Terraform/Pulumi-generated ASL. CI maintainers need repeatable Linux execution without cloud credentials. Apple Silicon developers need to execute Linux arm64 images without rebuilding handlers for macOS. + +| ID | Goal | Acceptance measure | +|---|---|---| +| G1 | Execute real images | Results originate inside the deployed handler; broken images, missing RICs and initialization failures fail explicitly, without echo fallback | +| G2 | Work with ordinary SDKs | Rust SDK and boto3 create/start/describe/history/stop scenarios pass without Rustack-only required wire fields | +| G3 | Make core ASL trustworthy | Every required case in the [verification matrix](./rustack-workflows-70-verification-plan.md) passes | +| G4 | Implement actual `.sync` | Parents await child termination; `.sync` and `.sync:2` preserve different Output types; invalid `lambda:invoke.sync` is rejected during creation | +| G5 | Reuse deployment artifacts | Native-architecture Linux x86_64, Linux arm64 and macOS arm64 + Docker Desktop lanes pass; the prepared example completes within five minutes excluding builds and pulls | +| G6 | Bound and clean up work | Overload is explicit; Stop, timeout and shutdown leave no unowned tasks or containers; cleanup failures remain tracked and diagnosable | +| G7 | Expose compatibility honestly | Unsupported syntax and integrations fail validation at a precise location; capabilities distinguish compiled, enabled, probed and limited behavior | + +Performance numbers are acceptance budgets, not existing measurements. + +## 4. Release scope + +| Capability | Milestone | Authoritative design | +|---|---|---| +| Image Lambda deploy/update/get, digest pinning, ImageConfig, versions, actual Invoke | M0 | [Lambda runtime](./rustack-workflows-20-lambda-runtime-design.md) | +| STANDARD APIs, Pass/Task/Choice/Wait/Succeed/Fail, data transformations, Retry/Catch | M1 | [API](./rustack-workflows-10-api-design.md), [ASL](./rustack-workflows-30-asl-design.md) | +| Parallel, Inline Map, nested startExecution/.sync/.sync:2 | M2 | [ASL](./rustack-workflows-30-asl-design.md), [integrations](./rustack-workflows-40-integrations-design.md) | +| Lambda `.waitForTaskToken`, SendTask* and heartbeats | M3 | [integrations](./rustack-workflows-40-integrations-design.md) | +| Container execution of provided.al2023 ZIPs, snapshots, platform and fault validation | M4 | [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md) | + +M0 is the smallest useful end-to-end slice: SDK → Lambda API → Docker handler → SDK result. M1 first closes SDK → ASL → Docker Lambda → history. **The complete requested outcome is M4**, not an M0/M1 scaffold. + +Excluded from this release: EXPRESS/StartSyncExecution, JSONata/Assign/variables, Distributed Map, Activities, Redrive/TestState, Step Functions versions/aliases, cross-account AssumeRole, arbitrary AWS SDK integrations, ECS/Batch/Glue `.sync`, an ECR emulator, user-image builds, VPC emulation, extensions, SnapStart, streaming responses, Managed Instances, an AWS IAM policy evaluator, and cloud-grade durable workflow recovery. + +Each exclusion requires explicit rejection or an advertised limitation, rather than storing ignored fields and reporting success. An eventual ECS `.sync` adapter requires a real ECS execution service; starting an arbitrary Docker container does not establish ECS compatibility. + +## 5. User experience + +Use the existing configuration entry point: `RUSTACK_CONFIG=./rustack.yaml rustack`. Enable `SERVICES=lambda,stepfunctions` and explicitly choose `LAMBDA_EXECUTOR=docker`. The consuming project builds its images beforehand. Local mode accepts an image such as `rustack-echo:integration`; an unchanged ECR reference must already exist in the daemon or match an operator-enabled registry pull policy. + +```json +{ + "StartAt": "Call", + "States": { + "Call": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "arn:aws:lambda:us-east-1:000000000000:function:echo", + "Payload.$": "$" + }, + "TimeoutSeconds": 15, + "Retry": [{"ErrorEquals": ["Lambda.TooManyRequestsException"], "MaxAttempts": 2}], + "OutputPath": "$.Payload", + "End": true + } + } +} +``` + +The SDK sequence is CreateFunction(PackageType=Image, Code.ImageUri=...) → wait for Active → CreateStateMachine(type=STANDARD) → StartExecution(input is a JSON string) → poll DescribeExecution → GetExecutionHistory. Production ASL needs no special testing fields. Ordinary `lambda:invoke` already waits for function completion; adding `.sync` is invalid. + +## 6. Compatibility declaration + +Unmodified production ASL works within the documented profile. Modern CDK output using JSONata or other integrations receives an explicit rejection. Resource names initially accept ASCII `[A-Za-z0-9_-]`, an advertised local restriction. Authentication follows Rustack's gateway trust modes; execution role ARNs are validated metadata, not evidence of AWS IAM authorization parity. + +Standard execution guarantees apply within a live Rustack runtime. Snapshots do not recover requests made after the last completed save or ensure exactly-once external side effects. + +## 7. Reading order and cross-references + +Read 00 → 10 → 20 → 30 → 40 for contracts. Read 60/70 alongside them. Document 80 describes user-visible delivery, 90 specifies engineering dependencies, and 99 records decisions. + +- → [API and data model](./rustack-workflows-10-api-design.md) +- → [roadmap](./rustack-workflows-80-roadmap.md), [implementation plan](./rustack-workflows-90-impl-plan.md) +- ↔ [key decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-10-api-design.md b/specs/rustack-workflows-10-api-design.md new file mode 100644 index 00000000..524878df --- /dev/null +++ b/specs/rustack-workflows-10-api-design.md @@ -0,0 +1,123 @@ +# Workflows 10: Step Functions API, Data Model and History + +Status: implementation contract · Depends on: [PRD](./rustack-workflows-00-prd.md) + +## 1. Protocol and crate boundaries + +Add `rustack-stepfunctions-model`, `rustack-stepfunctions-http` and `rustack-stepfunctions-core`. Keep the ASL compiler/interpreter in core's `asl/` and `engine/` modules rather than publishing another crate prematurely. HTTP owns codecs and protocol errors, not execution semantics. Generate model types with the existing Smithy generator. + +| Item | Contract | +|---|---| +| Source | [Official SFN Smithy](https://raw.githubusercontent.com/aws/api-models-aws/main/models/sfn/service/2016-11-23/sfn-2016-11-23.json); record source commit and SHA-256 when imported | +| Generator configuration | `codegen/services/stepfunctions.toml`; namespace `com.amazonaws.sfn`, service ID `AWSStepFunctions` | +| Protocol | `awsJson1_0`; POST `/`; Content-Type `application/x-amz-json-1.0`; target `AWSStepFunctions.` | +| JSON names | API camelCase; preserve ASL PascalCase using explicit serde renames, never global case conversion | +| Signing | SigV4 service `states`; reuse gateway identity, scope checks and strict/skip modes | +| Local service name | Feature, SERVICES and capabilities use `stepfunctions`; AWS signing and ARN service remain `states` | +| Routing | Register before the S3 catch-all; unknown AWSStepFunctions operations must not fall through to S3 | + +Future automation uses Makefile targets. The current generator needs protocol-aware numeric epoch-second timestamp serde for awsJson1_0; ordinary chrono serde produces strings. Before service generation, prove required/optional timestamps, fractional seconds and negative epochs in generated round trips. Keep protocol-default behavior scoped correctly so restJson/query timestamp bindings do not change accidentally. Resolve any union support required by the selected operations in the generator; do not hand-edit generated source. + +```text +┌─ AWS SDK clients ──────────────────────────────────────────────────┐ +│ POST / + AWSStepFunctions.* JSON strings(input/definition) │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ identity / body / deadline boundary +┌─ stepfunctions-http ───────────────────────────────────────────────┐ +│ Smithy codec → wire validation → typed commands → protocol errors │ +└─────────────────────────────┬──────────────────────────────────────┘ + bounded mpsc +┌─ core / RegistryActor ──────▼───────────────────────────────────────┐ +│ immutable machine revisions │ execution-name tombstones │ tokens │ +│ serialized Create/Update/Start/Stop; owned replies for read views │ +└──────────────┬───────────────────────────────┬──────────────────────┘ + ▼ ▼ +┌─ ExecutionActor ──────────────┐ ┌─ bounded history/read views ────┐ +│ revision, frames, generations │──▶│ event sequence, terminal data │ +│ task handles and timers │ │ snapshot DTOs, no live handles │ +└───────────────────────────────┘ └────────────────────────────────┘ +``` + +## 2. API coverage + +| Milestone | Operations | Required behavior | +|---|---|---| +| M1 | CreateStateMachine, UpdateStateMachine, DeleteStateMachine, DescribeStateMachine, ListStateMachines | Immutable revisions, STANDARD only, deletion lifecycle | +| M1 | ValidateStateMachineDefinition | Same compiler as create/update; OK/FAIL, diagnostics, severity, maxResults, truncated | +| M1 | StartExecution, DescribeExecution, ListExecutions, StopExecution | JSON strings, idempotency, asynchronous execution, single terminal settlement | +| M1 | GetExecutionHistory, DescribeStateMachineForExecution | Read the revision captured by this execution, not latest | +| M1 | TagResource, UntagResource, ListTagsForResource | Atomic validation; basic IaC create/read/update/delete coverage | +| M3 | SendTaskSuccess, SendTaskFailure, SendTaskHeartbeat | Tokens bind to execution, scope and attempt generation | + +Other known actions return HTTP400 local `UnsupportedOperation`, documented in capabilities; unknown actions return protocol UnknownOperationException. EXPRESS, qualified state-machine ARNs, publish/version/alias requests are explicitly rejected. StartSyncExecution is not a shortcut for awaiting STANDARD workflows. + +Create/Update accept absent loggingConfiguration or level OFF. Enabled tracing, nondefault encryption and publish=true return ValidationException. roleArn is validated metadata, not IAM policy evaluation. Delete marks the machine DELETING, rejects new Start/Update, and lets accepted executions finish. When the last execution closes, remove mutable definition and tags; execution/history/name records retain their independent TTLs. Repeated deletion during DELETING is idempotent; after removal return StateMachineDoesNotExist. + +CreateStateMachine is idempotent for the same name, raw definition and supported effective type/logging/tracing/encryption/publication settings. A repeated request with different roleArn/tags returns the existing resource without modifying those fields; a conflicting definition returns StateMachineAlreadyExists. Validate unsupported fields before considering a request idempotent. This follows [AWS CreateStateMachine](https://docs.aws.amazon.com/step-functions/latest/apireference/API_CreateStateMachine.html). Explicit Update changes the latest revision and affects only subsequently started executions. + +StopExecution of RUNNING work requests cancellation and returns the committed stopDate after terminal settlement, bounded by the control deadline. A repeated Stop on an already terminal record returns its existing stopDate without rewriting outcome/history; an absent execution returns ExecutionDoesNotExist. Concurrent success/Stop settles once. If resource cleanup remains uncertain, retain quarantine ownership and expose failure diagnostics rather than rewriting a completed execution to RUNNING. Read/List operations validate every supported filter; version/alias/MapRun/redrive filters are rejected until implemented. + +## 3. Domain types and ownership + +Follow AGENTS.md § Type Design & API: private validated fields, TryFrom/FromStr, non-exhaustive public structs and redacted Debug. Multi-field construction uses typed-builder. + +| Type | Fields and invariants | Owner | +|---|---|---| +| TenantScope | Validated partition, region, account | Gateway creates; core revalidates | +| StateMachineName / ExecutionName | 1–80 ASCII bytes, `[A-Za-z0-9_-]` | RegistryActor | +| StateName | 1–128 UTF-8 bytes; no controls; exact matching without normalization | Compiler | +| StateMachineArn / ExecutionArn | Structured `states` ARN, matching scope, ≤256 bytes | Boundary newtypes | +| DefinitionRevision | Revision ID, original definition, hash, Arc, role/config | Immutable Arc | +| ExecutionRecord | ARN, captured revision, raw and parsed input, start/stop times, outcome | ExecutionActor | +| TaskAttemptId | Execution UUID, frame, entry generation, retry ordinal, task UUID | Engine | +| HistoryEvent | AWS event type/details, id, previousEventId, timestamp | Execution history owner | +| ResourceHandle | Lambda invocation, owned child or callback handle | Engine; must be settled | + +ARN shapes are `arn::states:::stateMachine:` and `...:execution::`. No version/alias suffixes or MapRun labels initially. RegistryActor linearizes revision updates against execution starts. + +Validate execution-role metadata as an IAM role ARN in the same partition/account, with empty region, a bounded role path/name and≤256 total bytes. Do not parse it with the state-machine ARN grammar or accept arbitrary service ARNs. This is structural validation, not an AssumeRole operation or permission evaluation. + +Use `Running` versus `Terminal(Succeeded|Failed|TimedOut|Aborted)` domain states. Terminal variants carry either output or error/cause, preventing combinations such as RUNNING with stopDate. Map to AWS RUNNING/SUCCEEDED/FAILED/TIMED_OUT/ABORTED; do not expose unsupported PENDING_REDRIVE. + +## 4. Input validation and limits + +| Boundary | Validation | +|---|---| +| Control request body | Local 2 MiB limit, independently enforced before deserialization; JSON escaping counts toward this envelope limit | +| Decoded definition | 1,048,576 UTF-8 bytes; duplicate keys rejected; JSON depth ≤64 | +| Execution input/output and each transformation | 262,144 UTF-8 bytes; omitted input becomes `{}`, explicit empty string is invalid, JSON null is valid | +| Input equality | Preserve decoded raw UTF-8 string; byte equality, without whitespace normalization | +| ARN / trace / pagination token | 256 / 256 ASCII / 2,048 bytes | +| Tags | ≤50; key 1–128 bytes, value 0–256 bytes; AWS tag character allowlist, reserved aws: prefix rejected | +| Error / cause | 256 / 32,768 bytes; reject oversized client input; bounded internal causes carry an explicit truncation marker | +| Compiled structure | ≤1,000 states total, machine nesting ≤16, ≤128 branches per Parallel, path ≤1,024 bytes, intrinsic expression ≤4,096 bytes | + +The 2 MiB envelope is a local transport policy, not exact AWS request-size parity; it can reject heavily escaped or metadata-heavy definitions despite their decoded size fitting. AWS's independent definition/payload quotas still apply. Payload strings allow legal JSON text; identifier ASCII restrictions must not corrupt business data. Every collection has count and aggregate-byte limits. Unknown API/config/ASL fields are rejected, not collected without bounds. Smaller local structural limits must appear in validation diagnostics. + +## 5. StartExecution idempotency and revision isolation + +Atomically reserve `(scope, machine identity, execution name)` and capture a revision. Validate input first, then look up the name. A RUNNING execution with identical raw input returns its original ARN/startDate, even after a definition update. Different input or any closed execution returns ExecutionAlreadyExists. Name tombstones last at least 90 days after closure, independently of shorter history retention. If tombstone capacity is exhausted, reject new starts rather than reuse names early. See [AWS StartExecution](https://docs.aws.amazon.com/step-functions/latest/apireference/API_StartExecution.html). + +Generate a secure random UUID when name is absent. Delete/recreate must not allow collision with an existing execution ARN; old tombstones continue blocking the corresponding ARN until expiry. + +Ordering: validate/compile → reserve admission → atomically claim name and create RUNNING record plus ExecutionStarted → register execution actor → return200. Registration failure rolls back the reservation and permits. A lost HTTP response does not undo a started execution. Disconnecting after Start never cancels it. Each execution uses its captured machine revision; Lambda alias resolution happens per Task attempt, as specified in the integration design. + +## 6. History and pagination + +History IDs increase from 1. previousEventId records the causal predecessor, not merely the most recently appended event across concurrent branches. Required events include ExecutionStarted/Succeeded/Failed/TimedOut/Aborted, Entered/Exited for supported states, LambdaFunctionScheduled/Started/Succeeded/Failed/TimedOut for direct integrations, and TaskScheduled/Started/Succeeded/Failed/TimedOut for optimized integrations. M2 adds official Parallel/Map/iteration event types and TaskSubmitted after an optimized .sync child is accepted, before its terminal result. Retry creates multiple task attempts within one state entry, not a fabricated AWS RetryScheduled event; internal retry scheduling belongs in tracing. + +Append history, update read views and settle terminal state atomically within the owning actor. Once DescribeExecution observes termination, history contains its terminal event. API input/output remain JSON strings. includeExecutionData=false omits execution data consistently without changing event linkage. + +Use random opaque pagination handles backed by a bounded server-side registry. Bind scope, operation, filters, ordering, includeExecutionData and a read high-water mark. TTL is 24h. Stable execution sorting uses start time and ARN; history reversal uses event ID. Newly appended records do not change the cursor's captured boundary. History maxResults defaults to 100 and caps at 1000; 0 selects the default. Other list operations follow their Smithy limits. Expired, modified, cross-scope or argument-mismatched tokens return InvalidToken. Reject capacity exhaustion rather than evict live tokens. + +Keep at most 25,000 events, also constrained by per-execution/global byte budgets. Reserve one event slot and byte allowance for termination. Imminent exhaustion fails the execution with local `States.HistoryLimitExceeded`; never truncate history and still report success. This locally identified error is advertised as a limitation. Clean up expired terminal histories only; preserve independent name tombstones. + +## 7. Rust engineering contract + +AGENTS.md § Error Handling: thiserror StepFunctionsError with source errors, mapped to protocol code/status without credentials, Docker internals or host paths. § Async & Concurrency: actor-owned registry/history, bounded mpsc/oneshot and supervised JoinSet; no Mutex-wrapped engine. § Type Design/Safety/Serialization: validated newtypes, checked arithmetic, forbid unsafe, immediate schema validation. § Testing: rstest error matrices and proptest ARN/idempotency/event-chain invariants. § Logging: typed IDs and redacted Debug, never input/definition/token. § Performance: Arc revisions and Bytes for original data. § Documentation: public API examples, Errors sections and ownership documentation. + +## 8. Cross-references + +- ← [PRD](./rustack-workflows-00-prd.md) +- → [ASL](./rustack-workflows-30-asl-design.md), [integrations](./rustack-workflows-40-integrations-design.md) +- ↔ [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md), [decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md new file mode 100644 index 00000000..bbf622ab --- /dev/null +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -0,0 +1,131 @@ +# Workflows 20: Docker Lambda Execution Environments + +Status: implementation contract; relay composition requires P0 validation · Depends on: [PRD](./rustack-workflows-00-prd.md), [Lambda hardening](./rustack-lambda-hardening-design.md) + +## 1. Ownership and execution contract + +Extend `rustack-lambda-core::executor` with DockerExecutor implementing the existing Executor interface. Lambda provider remains the only invocation entry point: HTTP Invoke, Step Functions and future event sources share version resolution, reserved concurrency, 32 global execution permits, default 8 permits per logical function, and 128 accepted asynchronous-work slots. Bridges must not bypass the provider by invoking an executor directly. + +Add a small Linux `rustack-lambda-runtime` relay artifact. It serves the standard Runtime API inside the execution environment and exchanges private control messages over Docker attach. Extract reusable RuntimeSession state logic: Native retains its host TCP adapter, while Docker uses the environment-local adapter. Do not embed RIE or rewrite user images. + +```text +┌─ Rustack host / Rustack server container ───────────────────────────┐ +│ Lambda provider → WorkSupervisor → DockerExecutor/PoolActor │ +│ immutable revision bounded commands / JoinSet │ +│ ▼ │ +│ Docker Engine API client: Unix socket, API negotiation │ +└─────────────────────────────┬──────────────────────────────────────┘ + Docker attach stdin/stdout: non-TTY, framed CBOR +┌─ Docker daemon trust boundary▼─────────────────────────────────────┐ +│ ┌─ one execution environment / independent network namespace ─────┐│ +│ │ relay container function container ││ +│ │ RuntimeSession actor original user image digest ││ +│ │ 127.0.0.1:9001 ◄── Runtime API ─ RIC / custom bootstrap ││ +│ │ owns netns network=container: ││ +│ └──────────────────────────────────────────────┬───────────────────┘│ +│ no published Runtime API ports │ SDK → Rustack │ +│ private user bridge / operator network ────────┘ gateway endpoint │ +└───────────────────────────────────────────────────────────────────┘ +``` + +The two containers share only the network namespace, not PID/mount namespaces, Docker sockets or host directories. Access to one's own Runtime API is intentional; other environments cannot reach that loopback listener. Do not publish Runtime API ports, use host networking or authenticate by NAT source address. Host and server-in-container modes use the same control path. Configure DNS/extra-host entries on the relay/netns owner; Docker rejects these options on a function using network=container:. [Docker networking](https://docs.docker.com/engine/network/) documents the foundation; the entire composition remains subject to actual platform tests. + +Cost: two containers per warm environment. Each relay has a separate 64 MiB memory, 64 PID and 0.1 CPU cap. A 32-environment limit therefore permits at most 64 environment containers, with aggregate memory admission including relay overhead. Record the tradeoff in D2. If P0 invalidates the composition, revise this design and rerun its gate; do not silently substitute an exposed host listener. + +## 2. Image resolution and deployment lifecycle + +### 2.1 Immutable artifacts + +Replace mutually inconsistent execution-request Option fields with `DeploymentArtifact::Zip(ValidatedArtifact)` or `Image(ResolvedImage)`, preserving public wire schemas. ExecutionRevision captures function resource UUID, resolved version, artifact/digest, architecture, ImageConfig, environment fingerprint, memory, timeout, ephemeral storage and backend policy. Use typed-builder for multi-field construction. + +Keep the caller's requested function ARN/reference separately from the resolved numeric version. Runtime Invoked-Function-Arn preserves the invoked qualifier, while ExecutedVersion identifies the actual version. Environment reuse keys use resolved identity rather than the mutable alias string. Image CodeSha256 derives from resolved content metadata, not the requested tag text; CodeSize uses inspected image metadata with the local representation documented, and never claims to be a ZIP byte count. + +ResolvedImage records requested reference, registry manifest digest when available, daemon content image ID, platform and inspection metadata hash. Invoke by content ID/digest, never by a freshly resolved tag. GetFunction.Code.ImageUri returns the requested reference; ResolvedImageUri returns repo@digest for registry images. A local unpushed image uses the explicitly local extension `local://sha256:`; never invent an ECR manifest digest. Snapshots preserve this distinction; restoring such an image on another daemon requires importing the same content ID. + +References are≤1,024 bytes and follow a bounded OCI/Docker repository/tag/digest grammar. Reject URL schemes, userinfo, controls and shell syntax. Local short names and non-ECR registries are advertised local extensions. Unchanged ECR references work with preloaded images or an allowed pull policy; Rustack does not emulate ECR. + +Pulling is disabled by default. Enabling it requires an explicit registry allowlist and total timeout. The daemon performs network pulls. Registry credentials must be supplied through operator configuration; never automatically read host AWS credentials or execute shell credential helpers. Allowlist enforcement governs references submitted by Rustack; daemon-side mirror/DNS/redirect policy remains an operator responsibility and is not an SSRF guarantee. + +### 2.2 Create, update and publication + +Create(Image) rejects Runtime/Handler, defaults omitted Architectures to [x86_64], requires exactly one architecture after normalization, and rejects explicitly empty or multi-element lists. It checks mutually exclusive Image/ZIP/S3 code sources, and validates ImageConfig/environment. Reserve function identity and a deployment-work slot, return Pending, then resolve/pull/inspect on a bounded worker. Validate Linux OS, architecture and executable entry configuration before atomically making the revision Active. Failed deployments retain Failed plus bounded reason/code for inspection; Invoke rejects Pending/Failed. Deployment resolution requires available Docker; disabled execution never yields a falsely executable image revision. + +Validate function MemorySize in 128..10,240 MiB, Timeout in 1..900 seconds, and EphemeralStorage in 512..10,240 MiB before allocation. User environment has at most 128 pairs; keys are 2..64 ASCII bytes matching `[A-Za-z][A-Za-z0-9_]+`, individual values are≤4,096 bytes and the total remains≤4 KiB. The key/count limits are local restrictions. Function name/qualifier/artifact validation remains governed by Lambda hardening. Parse error payloads only within their 32 KiB allowance; stackTrace has≤100 entries, each≤1,024 bytes, also constrained by that total. + +UpdateFunctionCode keeps the previous revision usable while LastUpdateStatus=InProgress. Success atomically replaces latest; failure preserves the old digest/config and exposes a failed-update reason. RevisionId compare-and-swap prevents concurrent overwrite. Initial release rejects Image Create/Update with Publish=true using InvalidParameterValueException: asynchronous resolution cannot safely return a published version within the control deadline. This is a documented local limitation. A separate PublishVersion succeeds after Active/Succeeded and captures the resolved image and configuration. + +Tag movement never changes deployed code. Published versions are immutable; aliases move references. Update/delete/recreate cannot reuse an old environment. In-flight calls keep their captured revision. Reject package-type changes. DryRun validates input, function/qualifier and permission shape without starting a container or executing code. + +### 2.3 Platform and entry configuration + +Map x86_64 to linux/amd64 and arm64 to linux/arm64. Guarantee native architecture only. Reject cross-architecture calls unless the operator explicitly enables emulation; do not promise emulation performance/parity. Registry manifest lists select exactly one matching platform and pin its child manifest digest. Inspect local image architecture rather than trusting the declaration. + +Inherit Dockerfile ENTRYPOINT, CMD and WORKDIR, then override only explicitly supplied ImageConfig fields. Missing and explicitly empty arrays differ. WorkingDirectory is an image-internal absolute path≤1,000 bytes, not a host path. Each argv list has≤1,500 elements, each≤4,096 bytes and total≤64 KiB locally. Pass JSON argv directly, never sh -c. Do not force CMD=bootstrap. P0 must prove official Lambda images enter their RIC path when AWS_LAMBDA_RUNTIME_API is set. + +## 3. Runtime API and relay transport + +Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. + +Inject `AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001` and revision-derived Lambda name/version/memory/region/log/task-root metadata. Reject reserved-variable overrides. User environment totals≤4 KiB with Lambda key grammar. Inject only explicitly configured local test credentials; do not inherit host AWS secret variables. + +Control frames are 4-byte big-endian length plus CBOR envelope, protocol version 1. Maximum frame is 6 MiB+64 KiB, with metadata≤64 KiB and payload encoded as CBOR bytes rather than base64. Message variants are Hello, Ready, BootstrapReady, Invoke, InvocationResult, InitError, Cancel, Close and Ack. Include environment UUID and invocation-attempt UUID. Hello advertises protocol version and architecture; mismatch retires the environment. EOF, malformed frames or attach failure invalidate it. Never reconnect and silently replay an invocation. Relay stdout carries frames only; stderr carries bounded redacted diagnostics. Collect function stdout/stderr separately from its own container. Enforce an independent subscriber filter that rejects all Bollard events more verbose than INFO regardless of RUST_LOG; the selected dependency logs HTTP request/response bodies and trailing attach bytes at trace/debug. Test secret-bearing create requests and truncated attach frames under RUST_LOG=trace. Application-level redacted Debug alone is insufficient. + +Create the relay with Tty=false, OpenStdin=true and StdinOnce=true. Establish bidirectional attach before starting it so the initial Hello cannot be lost; then start and await Hello/Ready before creating the function container. Demultiplex Docker stdout/stderr frames before decoding CBOR, preserve partial-frame buffering within the maximum length, and cap/control metadata before allocation. Relay protocol output never passes through a text logger. Shutdown closes the input only after Close/Ack or deadline expiry. P0 experimentally showed StdinOnce=false leaves the relay running after attach-input shutdown, while true delivers EOF and terminates the relay. Each environment owns exactly one attach connection; reconnect/replay remains forbidden. Keep attach IO in separately supervised read/write tasks so slow log collection cannot deadlock invocation completion or cancellation. + +Reject indefinite-length CBOR collections, nesting deeper than16 and more than128 metadata members before materialization; payload bytes are the only large field. Control-message variants validate permitted fields and reject duplicates/unknown fields. BootstrapReady is emitted once, only after the first valid RIC GET /next; relay Ready cannot satisfy the function initialization deadline. The bounded transport is not permission to deserialize arbitrary recursive CBOR structures. + +Runtime `/next` supplies Request-Id, Deadline-Ms, Invoked-Function-Arn and Trace-Id. Convert wall deadlines with checked arithmetic and enforce elapsed time using monotonic clocks. If the optional Invocation-Id header is supplied on completion, it must match; omission remains compatible with older RICs. Unknown, expired or duplicate completions cannot resolve another request. Validate method/path/header/body limits before touching pending state. See [Runtime API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html). + +```text +Provider Environment actor Docker/relay RIC + │1. admit + revision │ │ │ + │─────────────────────────▶│2. attach/start relay ───▶│ Hello/Ready │ + │ │3. start function ───────┼───────────────────▶│ + │ │ │◀── GET /next ────│ + │ │◀── bootstrap-ready ─────│ │ + │ │4. Invoke(id,deadline) ──▶│─── event ────────▶│ + │ │ │◀─ response/error ─│ + │ │◀── InvocationResult ────│ │ + │◀── typed result ─────────│5. settle once / pool │ │ + │ cancel/timeout ─────────▶│6. invalidate attempt │ │ + │ │ kill function → close relay → remove both │ + │◀── cleanup settlement ───│ release or quarantine allocation reservation │ +``` + +Ready means the relay is listening; bootstrap-ready means the RIC issued its first GET /next. They are different signals. Image deployment, relay start, function initialization, handler execution and cleanup have separate budgets. Synchronous Invoke's outer total includes cold start. Function handler timeout starts at event delivery and is additionally bounded by the caller's Task absolute deadline: cold start must not extend that Task deadline. Idle `/next` uses the environment idle/shutdown lease, not the short control-request timeout; this is bounded long polling. + +## 4. Environment states, pooling and cancellation + +State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. + +Warm identity includes resource UUID, resolved version, artifact/digest, configuration fingerprint, architecture and backend. Never share across functions. Preserve 32 global warm environments and default 1 retained warm environment per revision. Busy allocations have the same independent global cap; excess concurrent environments retire after completion instead of growing the warm pool. Wait for environment capacity only within bounded queue/deadline budgets. Default idle lifetime is 600s. Init/protocol failures, OOM, timeout and failed invocations retire their environments. + +Cleanup ordering: invalidate attempt → cancel relay job → stop/kill function → await exit → remove function → close/remove relay → release environment reservation. Normal execution permits survive through cleanup. If Docker becomes unreachable, retain container IDs and a quarantine allocation reservation against global capacity; the caller may receive failure while reconciliation remains owned by the supervisor. Retry cleanup after 1/2/4 seconds, capped at 30s between attempts. A dropped HTTP future cannot orphan containers. Unresolved cleanup at shutdown sets readiness=false, prevents successful consistency snapshot publication and produces nonzero exit. + +Labels include persistent data-directory ownerId, runId, environmentId and function resource UUID. Startup may reclaim old runs only for the same owner after acquiring its directory's exclusive lock. Never delete containers belonging to another Rustack instance based on a generic Lambda label; never delete user images. Keep revision/artifact references alive until invocation cleanup finishes. + +## 5. Isolation and resource limits + +- Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. +- Function containers use read-only rootfs, cap-drop ALL, no-new-privileges, default seccomp,≤256 PIDs, default 1 CPU and configured function memory subject to aggregate admission. Prohibit privileged mode, host PID/IPC/network, devices, Docker sockets and arbitrary host mounts. Preserve a nonroot image USER; override absent/root USER with the configured unprivileged UID/GID. Permission-incompatible images fail explicitly. +- `/tmp` is a size-capped tmpfs, 512–10,240 MiB. Its used memory counts toward the container memory cap, unlike AWS disk-backed ephemeral storage; advertise this limitation. ZIP code reaches a daemon-owned volume through a validated archive, not a host-path bind assumption. Mount `/var/task` read-only; never expose the project root. +- Docker runs operator-selected code; it is not a hostile multi-tenant isolation promise. Egress follows the configured Docker network. Without network policy, do not claim handler SSRF prevention. Registry pull allowlists do not restrict arbitrary handler networking. +- Continuously drain stdout/stderr into separate bounded rings,≤256 KiB each; LogType=Tail returns the final 4 KiB. Cap individual diagnostic lines at 16 KiB and errors at 32 KiB. Drop/count excess logs without blocking handlers. User stdout is intentional user output, not infrastructure tracing; do not promise automatic secret detection in arbitrary logs. + +## 6. Responses, errors and ZIP support + +`/response` yields original Bytes. `/error` yields Invoke HTTP200 plus FunctionError=Unhandled and bounded payload. Missing images, OOM, missing RIC, early exit, initialization and protocol failures produce distinct ExecutorError variants mapped to Lambda errors. Infrastructure failures must never become success payloads. ASL mappings are specified separately. + +M4 adds Docker execution of provided.al2023 ZIPs using an operator-pinned OS-only image digest. Safely populate the code volume and explicitly execute `/var/task/bootstrap`; an OS-only image's runtime launcher is not the user's bootstrap. Validate Linux ELF architecture. Preserve Native/Squib ZIP behavior. Managed Python/Node ZIP runtime mappings are excluded; deploy images containing their RICs instead. + +## 7. Rust engineering contract + +Apply AGENTS.md § Error Handling/Async/Type/Safety: thiserror sources; retain async-trait for the existing dyn Executor and document object safety; concrete internal actors use ordinary async methods. Use Bytes/Arc revisions, validated artifact enums, redacted Debug, checked memory/deadline arithmetic, forbid unsafe, IO deadlines and supervised JoinSet tasks. Serde control envelopes use camelCase; Runtime API uses AWS headers. Validate raw lengths before parsing. Tests use actual RIC images and cancellation/failure matrices. Tracing contains IDs, durations and classes, not payloads. Avoid per-invoke pull/inspect. Public APIs document Errors and examples; operational docs cover both deployment topologies. + +Use the surveyed bollard 0.21.1 API as a starting point and recheck before implementation; disable default features and enable pipe only. Reuse ciborium. TLS transport is N/A for this release; future TLS must use rustls + aws-lc-rs rather than enabling a ring-backed SSL default. + +## 8. Cross-references + +- ← [PRD](./rustack-workflows-00-prd.md), [existing runtime spike](../docs/research/spike-cargo-lambda-runtime-execution.md) +- → [integrations](./rustack-workflows-40-integrations-design.md), [operations](./rustack-workflows-60-operations-design.md) +- ↔ [verification](./rustack-workflows-70-verification-plan.md), [decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-30-asl-design.md b/specs/rustack-workflows-30-asl-design.md new file mode 100644 index 00000000..916bfba0 --- /dev/null +++ b/specs/rustack-workflows-30-asl-design.md @@ -0,0 +1,134 @@ +# Workflows 30: ASL Compiler and State Machine Engine + +Status: implementation contract · Depends on: [API/data model](./rustack-workflows-10-api-design.md) + +## 1. Compile before execution + +Create, Update and Validate share AslCompiler and produce immutable CompiledMachine objects. Never repeatedly parse definitions or resolve string Next targets during execution. Reject duplicate JSON keys and enforce byte/depth limits before constructing serde_json::Value. Business payloads are dynamic JSON; state definitions must become typed tagged enums rather than arbitrary maps interpreted throughout the engine. + +Pipeline: bounded bytes → strict AST → field/state validation → scoped graph validation → Path/Intrinsic AST → integration resolution → indexed IR. StateId is a checked internal index. Next/Catch/Choice targets stay within their machine scope: no branch jumps to a parent or sibling machine. Loops are valid; do not require a DAG. Require a reachable terminal state; unreachable states are warnings. + +Reject unsupported fields, JSONata, Assign, Credentials, dynamic Resource, Distributed Map and unavailable integrations during creation. Diagnostics carry JSON Pointer location, severity, stable Rustack code and bounded message without copying definition/input into logs. QueryLanguage may be omitted or explicitly JSONPath. + +## 2. Supported ASL profile + +The [ASL specification](https://states-language.net/spec.html) supplies language terminology. The table below is the release contract; supporting eight state types does not imply support for every recent language extension. + +| State | Supported fields beyond Type/Comment and applicable InputPath/OutputPath | Constraints | +|---|---|---| +| Pass | Result, Parameters, ResultPath, Next/End | Result and Parameters are mutually exclusive locally; no Retry/Catch | +| Task | Resource, Parameters, ResultSelector, ResultPath, Retry/Catch, TimeoutSeconds/Path, HeartbeatSeconds/Path, Next/End | Exact integration allowlist; heartbeat supported for callback only | +| Choice | Choices, Default | No top-level Next/End or ResultPath; first matching rule wins | +| Wait | Seconds, SecondsPath, Timestamp, TimestampPath, Next/End | Exactly one wait source | +| Succeed | InputPath, OutputPath | No Next/End or Retry/Catch | +| Fail | Error/Cause, ErrorPath/CausePath | Each literal/path pair is exclusive; no input/output transforms or Next/End | +| Parallel | Branches, Parameters, ResultSelector, ResultPath, Retry/Catch, Next/End | At least one independent branch machine | +| Map, INLINE | ItemsPath, ItemSelector, ItemProcessor, MaxConcurrency/Path, ResultSelector, ResultPath, Retry/Catch, Next/End | Normalize legacy Iterator/Parameters; reject simultaneous legacy/current equivalents | + +States supporting Next/End require exactly one, with End=true. Choice Next exists only inside rules. ProcessorConfig only permits Mode=INLINE. MaxConcurrency and MaxConcurrencyPath are exclusive. Compile-time constraints fail compilation; dynamically selected values are validated on entry and never replaced with defaults after an error. + +### 2.1 Paths + +Use winnow for an explicit ASL subset rather than treating a general JSONPath crate as the AWS contract. Support `$` and `$$`, `.identifier`, `['quoted-key']` with bounded quote/backslash escaping, nonnegative `[index]`, object/array `[*]`, nonnegative `[start:end]`, and consecutive wildcards such as `$[*][*]`. Reject filters/scripts, recursive descent, unions, negative indices/steps and dot-escape syntax at creation. ReferencePath permits only root/member/index; ResultPath must target `$`, never `$$`. + +Return Missing, Single or Selection, distinguishing JSON null from no match. Wildcard/slice selections remain arrays even with one match. Array ordering is stable; object wildcard uses deterministic key order as a local behavior, not an AWS object-order guarantee. Reference lookup failures do not panic or become null. + +### 2.2 Data processing + +```text +┌─ immutable raw state input S ─────────────────────────────────────┐ +│ InputPath(default $, null→{}) → effective input I │ +│ ▼ │ +│ Parameters template(I,$$), or I when absent │ +│ ▼ │ +│ Task / Pass / Parallel / Map produces raw result R │ +│ ▼ │ +│ ResultSelector(R,$$), or R when absent │ +│ │ │ +│ S ────────────────▶ ResultPath(default $, null→preserve S) → M │ +│ │ │ +│ OutputPath(default $, null→{}) ◀─────────────────────────────┘ │ +│ ▼ next state's raw input │ +└───────────────────────────────────────────────────────────────────┘ + Every expansion/merge checks the 256 KiB payload budget. +``` + +ResultPath merges into original S, not the Parameters result. Missing object members may create intermediate objects; writes through scalar/null/wrong container types fail, and arrays do not grow to arbitrary indices. ResultPath=null discards the result. Parameters/ResultSelector recursively process object/array `.$` fields. Dynamic fields require valid Path/Intrinsic strings; collision after removing `.$` is a compile error. ResultSelector `$` refers to raw result, not state input. + +Bounded writers enforce final UTF-8 JSON size without allocating unbounded output first. Connector results are size-checked before OutputPath can discard them. Catch.ResultPath merges `{Error,Cause}` into the failed state's raw input rather than a partially transformed result. + +### 2.3 Context and intrinsics + +Support `Execution.{Id,Input,Name,RoleArn,StartTime}`, `StateMachine.{Id,Name}`, and `State.{Name,EnteredTime,RetryCount}`. Map ItemSelector may read `Map.Item.{Index,Value,Source}` with Source=STATE_DATA; inner iteration states receive such data through ItemSelector rather than implicit outer context. Only callback tasks expose Task.Token. Timestamps use RFC3339 UTC and fixture-defined precision. Context is read-only; internal actor IDs cannot masquerade as AWS ARNs. + +Intrinsic allowlist: States.Format, StringToJson, JsonToString, Array, ArrayLength, ArrayGetItem, ArrayContains, ArrayPartition, StringSplit, MathAdd, UUID and JsonMerge. JsonMerge supports only `States.JsonMerge(a,b,false)` shallow merging. Reject all others at creation. Nesting is at most ten; each evaluation permits at most 10,000 AST operations and remains within the output budget. P0 fixtures pin escaping, Format, StringSplit delimiter behavior, MathAdd rounding/ranges, shallow merge and UUID format. Do not substitute Rust formatter/split semantics. Production UUIDs use secure randomness; deterministic tests inject a typed ID source rather than seed production randomness. + +### 2.4 Choice and Wait + +Choice supports short-circuit And/Or/Not, IsPresent/IsNull/IsString/IsNumeric/IsBoolean/IsTimestamp; String/Numeric/Timestamp Equals/LessThan/GreaterThan/LessThanEquals/GreaterThanEquals and their Path variants; BooleanEquals/Path and StringMatches. Do not coerce strings to numbers. IsPresent safely tests missing data; other missing-reference errors are runtime failures. Present values of inappropriate comparison types yield false. Distinguish missing, null and wrong type in fixtures. StringMatches implements AWS wildcard/escaping grammar, not user regex. Reject NaN/Infinity; preserve numeric comparison precision within the documented binary64 interoperability range. + +Wait Seconds is nonnegative; timestamps must be valid RFC3339. Dynamic type errors fail. Past timestamps finish immediately. Apply AWS second-level timestamp truncation, compute remaining duration using wall time once, then use a monotonic timer to avoid NTP jumps. Overall deadlines still bound long waits. No dedicated sleeping thread per state. + +## 3. Execution ownership and scheduling + +One ExecutionActor owns execution data, all branch frames, attempt generations, retry counters, timers and history. Branches are frames, not an unbounded tree of Tokio tasks. A ready queue drives pure transitions, yielding after at most 64 transitions per tick. Bounded supervised JoinSet futures perform external calls and return typed results; pure evaluation never performs network IO. + +Each frame holds state ID, entry generation, raw input, parent and phase: Entering, Calling, WaitingTimer, WaitingChildren, WaitingCallback or Completing. Timer entries carry generations. Reject stale timers/results before applying them. Completion, timeout, Stop and callback messages settle through the same owner exactly once. Results observed at or after the monotonic deadline cannot win a select race and become success. + +```text +┌─ Supervisor ───────────────────────────────────────────────────────┐ +│ open-execution permits │ JoinSet │ panic monitor │ cancellation │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ ExecutionActor: one mutable owner ────────────────────────────────┐ +│ ready frames → evaluate ≤64 transitions → history + read views │ +│ ▲ │ │ │ +│ timer heap Task dispatch ├─ terminal settle│ +│ └─ Wait/Retry ▼ bounded external calls└─ cancellation │ +│ IntegrationPort │ +│ frames ◀────────── result(attempt,generation) │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Pure loops are bounded by transition/history/total-duration limits and cannot starve Stop or health. Registry mailbox defaults to 256; per-execution control/completion lanes to 64 each, with an independent stop/watch lane. Acquire downstream admission before dispatch. Waiting for Lambda capacity must not hold a scheduler worker or downstream dispatch slot indefinitely. + +## 4. Retry, Catch and deadlines + +Retry applies to Task/Parallel/Map. Scan retriers in order and use the first matching one. Maintain retry counts per retrier within a state entry. MaxAttempts counts retries after the first failure; zero disables retries. Defaults: IntervalSeconds=1, MaxAttempts=3, BackoffRate=2.0, JitterStrategy=NONE. Retry k waits `min(IntervalSeconds × BackoffRate^(k-1), MaxDelaySeconds)`; FULL samples uniformly between zero and that delay. Use checked/saturating arithmetic and the remaining overall deadline. + +Validate IntervalSeconds in 1..99,999,999, MaxAttempts in 0..99,999,999, finite BackoffRate≥1, MaxDelaySeconds in 1..31,622,400, and NONE/FULL jitter. Effective work remains bounded by local transition/history budgets. States.ALL is the sole entry of the last matcher. States.TaskFailed matches task failures except States.Timeout, subject to uncatchable-error rules. + +Only after retries exhaust does Catch scan in order. Without a match, propagate to the parent frame or terminate root execution. Retry starts from the original state input and reevaluates Parameters with updated RetryCount; it does not consume the failed output. Generate fresh attempt, Lambda request ID and callback token. External side effects may repeat. See [AWS error handling](https://docs.aws.amazon.com/step-functions/latest/dg/concepts-error-handling.html). + +| Condition | Behavior | +|---|---| +| Task TimeoutSeconds/Path | Cancel resource, raise States.Timeout; Task Retry/Catch may handle it; reset per attempt | +| Machine TimeoutSeconds, Standard one-year ceiling or operator deadline | Execution TIMED_OUT; internal Catch cannot resume it | +| Callback heartbeat expiry | States.HeartbeatTimeout; starting point defined by callback integration | +| Lambda's own timeout | Preserve Sandbox.Timedout or the runtime-specific Lambda error | +| States.Runtime: unevaluable paths/types/expressions | Fatal, no Retry/Catch | +| States.DataLimitExceeded | Not matched by States.ALL; explicit matching may Retry/Catch it | +| Stop or parent cancellation | Abort/cancel; bypass user Retry/Catch | +| Actor panic or invariant failure | Supervised Rustack.InternalError, no automatic business replay | + +Explicit DataLimitExceeded handling follows current AWS documentation rather than older claims that it is always uncatchable. Keep failure data bounded; repeated oversized transformations fail again. Runtime diagnostics retain Path/Intrinsic/ResultPath categories without embedding inputs. P0 pins precise boundary errors and history details for the profile. + +## 5. Parallel and Inline Map + +Parallel starts each branch with the same effective input and orders results by branch definition. Map selects an array via ItemsPath, applies ItemSelector and orders outputs by item index, never completion order. Empty Map returns `[]`. Normalize Iterator/ItemProcessor and legacy Parameters/ItemSelector once in the compiler. + +Map MaxConcurrency=0 or omission selects the Inline limit of 40, further constrained by local task/frame budgets. Lower operational concurrency changes scheduling but not result order. MaxConcurrencyPath must resolve to a nonnegative integer. Static values above 40 are rejected by the local profile. Bound items to 10,000 and input to 256 KiB; active frames to 4,096 globally and 512 per execution. Create iterations lazily, not one spawn per item. Nested Map/Parallel share all budgets. + +An uncaught branch/iteration failure stops new work, cancels and joins in-flight siblings, then settles the enclosing state. External side effects are not rolled back. Parallel/Map Retry repeats all branches/items, not just failed ones. Invalidate old generations/tokens before the next round; do not reset the execution deadline. All branch events belong to the root execution's history sequence with correct causal links. + +## 6. Lifecycle and Rust engineering contract + +Start transfers ownership to the supervisor, independent of its HTTP request. Stop marks cancellation, invalidates attempts, cleans up resources and commits the terminal state. Quarantined external resources remain supervisor-owned and visible in diagnostics. Restarting a supervisor restores its ability to accept work; it never silently replays an uncertain invocation or restarts a failed execution. + +Apply AGENTS.md: thiserror/source errors; bounded actor messages and JoinSet; no Mutex-wrapped engine; non-exhaustive public DTOs and exhaustive internal state enums; forbidden unsafe/external panics and checked budgets; immediate serde validation; rstest/proptest/paused-time race tests; tracing skips payloads; Arc/Bytes and compiled IR avoid repeated parsing. Public compiler/runtime APIs provide examples and Errors sections. Configuration is centralized in the operations design, not embedded test shortcuts. + +## 7. Cross-references + +- ← [API/data model](./rustack-workflows-10-api-design.md) +- → [integrations](./rustack-workflows-40-integrations-design.md), [verification](./rustack-workflows-70-verification-plan.md) +- ↔ [operations](./rustack-workflows-60-operations-design.md), [decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-40-integrations-design.md b/specs/rustack-workflows-40-integrations-design.md new file mode 100644 index 00000000..c72f36f6 --- /dev/null +++ b/specs/rustack-workflows-40-integrations-design.md @@ -0,0 +1,134 @@ +# Workflows 40: Lambda Tasks, Nested `.sync` and Callbacks + +Status: implementation contract · Depends on: [API](./rustack-workflows-10-api-design.md), [Lambda runtime](./rustack-workflows-20-lambda-runtime-design.md), [ASL](./rustack-workflows-30-asl-design.md) + +## 1. Internal ports and dependency direction + +stepfunctions-core defines an object-safe LambdaTaskPort, a concrete channel-based ChildExecutionPort, and callback commands. The binary's `apps/rustack/src/stepfunctions_bridge.rs` injects a Lambda provider adapter. Core does not depend on lambda-core, an AWS SDK, HTTP loopback calls or Docker. Child execution uses a registry handle, avoiding a strong Arc cycle back to its own provider. + +LambdaTaskPort logically exposes `start(validated command, deadline) → InvocationHandle`, with completion and `cancel_and_wait(deadline)`. Dynamic adapter injection requires async-trait; document its object-safety reason per AGENTS.md. ChildExecutionPort submits scoped Start/Subscribe/Cancel messages with parent references. Pure Pass workflows work without Lambda; compiling a Lambda integration without an installed adapter produces a dependency-disabled diagnostic. + +Add a shared internal invocation-handle API to Lambda provider; HTTP Invoke delegates to it without changing wire behavior. Dropping the old response future cannot prove Docker cleanup. RAII may signal cancellation, but a supervisor must await cleanup; Drop cannot pretend to finish asynchronous IO. + +```text +┌─ stepfunctions-core ───────────────────────────────────────────────┐ +│ ExecutionActor → TaskAdapter enum │ +│ │ ├─ LambdaTaskPort (dyn) ───────────────────┐ │ +│ │ ├─ ChildExecutionPort → registry actor │ │ +│ │ └─ CallbackRegistry: token + generation │ │ +│ └─ cancellation / generation validation │ │ +└──────────────────────────────────────────────────────────────┼─────┘ +┌─ binary composition root ────────────────────────────────────▼─────┐ +│ StepFunctionsLambdaBridge → Lambda provider.start_invocation │ +│ version resolution + same admission │ +└───────────────────────────────────────────────┬───────────────────┘ + Executor → actual runtime +``` + +## 2. Integration allowlist + +| Resource | Pattern | Milestone | Successful Task result | +|---|---|---|---| +| `arn::lambda:::function:[:qualifier]` | Direct Lambda RequestResponse | M1 | Function JSON payload itself | +| `arn::states:::lambda:invoke` | Optimized RequestResponse | M1 | AWS envelope with Payload/StatusCode/ExecutedVersion/metadata | +| `...:states:::states:startExecution` | Await child start acceptance | M2 | ExecutionArn, StartDate; no wait for child termination | +| `...:states:::states:startExecution.sync` | Job wait | M2 | Child execution envelope; Output is a JSON string | +| `...:states:::states:startExecution.sync:2` | Job wait | M2 | Same envelope; Output is a JSON value | +| `...:states:::lambda:invoke.waitForTaskToken` | Callback | M3 | SendTaskSuccess JSON output | + +Partition must match scope; initially region/account must also match. Reject `lambda:invoke.sync`, `aws-sdk:*`, other services' `.sync`, and direct-Lambda callback at validation. Match the entire service/action/pattern allowlist, never arbitrary resources ending in `.sync`. [AWS integration patterns](https://docs.aws.amazon.com/step-functions/latest/dg/integrate-optimized.html). + +## 3. Lambda RequestResponse + +Direct ARN passes Parameters, if supplied, otherwise effective input, as the function event; the result is the function payload. Optimized Parameters supports FunctionName, Payload, Qualifier and InvocationType. Omitted/RequestResponse is supported; Event/DryRun is rejected in this release to avoid confusing acceptance with function completion. FunctionName may be a supported name or ARN. Reject conflicting explicit and embedded qualifiers rather than silently choosing one. Dynamic values use the same Lambda resolver after evaluation. + +Resolve aliases/latest at each attempt's invocation time and capture that Lambda revision for the call. Machine revision capture does not freeze all future Lambda aliases. Projects needing fixed code across retries use published version ARNs. Retry resolves alias/latest again; Rustack must not invent execution-wide alias pinning. + +Optimized success contains parsed JSON Payload, StatusCode=200, ExecutedVersion, SdkHttpMetadata.HttpStatusCode and SdkResponseMetadata.RequestId. Only include meaningful bounded metadata; do not invent cloud tracing capabilities. Never JSON-encode Payload twice. Raw Lambda Invoke supports arbitrary Bytes, but ASL requires JSON: invalid UTF-8/JSON or oversized connector output fails explicitly without lossy conversion. + +FunctionError causes Task failure even when Invoke returns HTTP200. Extract a bounded errorType and cause from the error payload; use Lambda.Unknown when no valid type exists. Do not infer failure from ordinary success payload keys such as error/statusCode. Adapter mappings are: + +| Origin | ASL behavior | +|---|---| +| Lambda reserved/global concurrency rejection | Lambda.TooManyRequestsException | +| Known Lambda service exception | `Lambda.`, without host internals | +| Function `/error` | User errorType; reserved `States.*` names become Lambda.Unknown | +| Function's own timeout | Sandbox.Timedout or supported runtime-specific error | +| Initialization/entrypoint failure | Runtime.InvalidEntrypoint or appropriate Runtime.* class | +| Step Functions Task deadline | States.Timeout; cancel handle and reject late completion | +| Docker transport failure with uncertain result | Lambda.ServiceException; internal deliveryUnknown diagnostic; no automatic replay | + +P0 fixtures pin exact exception/event details. Only explicit ASL Retry repeats business calls; adapters must not add hidden invocation retries. Retrying deliveryUnknown may repeat external side effects, so retain attempt IDs for diagnosis. + +## 4. Child workflows and `.sync` + +Parameters supports StateMachineArn, Input and Name. Normalize omitted/null Input to `{}`. An Input string is already serialized JSON: validate and preserve its bytes rather than encoding it again. Compact-serialize other JSON values. Passing the serialized string `null` therefore differs from a null Parameters value. Users may place `AWS_STEP_FUNCTIONS_STARTED_BY_EXECUTION_ID.$: $$.Execution.Id` inside child Input. Do not inject fields into business payloads. Generate an independent UUID name when absent and store parent linkage internally. + +RequestResponse waits only for acceptance. Such a detached child is not cancelled when its parent subsequently fails/stops, but remains owned by the global supervisor. `.sync`/`.sync:2` establish an owned relationship. Subscribe atomically with Start or use a versioned subscribe-then-read protocol; a fast child must not terminate before its parent can observe completion. Deliver termination once. Use internal events, not HTTP polling or EventBridge rules. + +```text +Parent actor Registry actor Child actor + │1. StartChild + subscribe │ │ + │─────────────────────────────▶│2. name/revision/admission │ + │ │──────────────────────────▶│ + │◀── executionArn + lease ─────│ │ + │3. release dispatch slot │ │ + │ await child terminal │ │ + │ │◀── terminal/read view ────│ + │◀──4. child result ───────────│ │ + │ .sync: Output string │ │ + │ .sync:2: Output JSON │ │ + │ Stop/timeout ────────────────▶│5. cancel owned child ─────▶│ + │◀── cleanup settlement ───────│◀── child cleanup/terminal ─│ +``` + +Use the optimized integration's PascalCase ExecutionArn/StateMachineArn/Name/Status/Input/Output/StartDate/StopDate envelope. Dates in optimized child-start and child-completion envelopes are integer epoch milliseconds, unlike HTTP API fractional epoch seconds. Do not reuse the camelCase HTTP DescribeExecution serializer. `.sync` retains Input/Output as JSON strings; `.sync:2` decodes both Input and Output into JSON values. Include InputDetails/OutputDetails.Included where applicable. The P0 oracle memo pins the upstream AWS-validated evidence and its historical-date limitation. [AWS nested workflows](https://docs.aws.amazon.com/step-functions/latest/dg/connect-stepfunctions.html). + +FAILED/TIMED_OUT/ABORTED children fail the parent Task with States.TaskFailed and bounded child ARN/error/status cause. A child's timeout is not the parent's own States.Timeout. Parent Task timeout, Stop or sibling-branch failure requests cancellation of an owned child. Single terminal settlement resolves concurrent completion; successful children and external side effects are never rolled back. + +The default 128 open-execution limit includes parents and children; nesting depth≤16. **A waiting parent holds no dispatch slot or Lambda permit**, preventing parents from exhausting all capacity required by children. If open-execution admission is exhausted, return ExecutionLimitExceeded rather than queue a child forever. Recursive workflows are allowed within depth/history/duration limits. Retry creates another child attempt; a user-supplied fixed Name still obeys normal idempotency/name-retention rules. + +### 4.1 Example: wait for a real child workflow + +The child can be the image-Lambda machine from the PRD. The parent definition is: + +```json +{ + "StartAt": "RunChild", + "States": { + "RunChild": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:ImageChild", + "Input.$": "$" + }, + "TimeoutSeconds": 30, + "OutputPath": "$.Output", + "End": true + } + } +} +``` + +The parent must remain RUNNING until the child's container handler finishes. A failed child cannot produce a successful parent output. + +## 5. Callback and heartbeat, M3 + +Support optimized Lambda `.waitForTaskToken` only. Before dispatch, generate a 256-bit CSPRNG token, register its hash/scope/execution/frame/generation, and then evaluate Parameters so `$$.Task.Token` exists. Invoke Lambda synchronously. After Lambda succeeds, await callback; the Lambda return payload is not the Task output. Lambda failure revokes the token and follows ordinary failure handling. + +A callback can arrive before the Lambda invocation returns. Store it provisionally in the same owner, commit it after successful Lambda return, or discard it if Lambda fails. A fast callback is neither lost nor allowed to hide invocation failure. + +Task timeout includes invocation and callback waiting. Heartbeat timing starts only after successful synchronous Lambda return. SendTaskHeartbeat extends the heartbeat window, not the absolute Task deadline. Retry/timeout revokes old tokens and creates a fresh token for the next attempt. Without a heartbeat field, do not invent a short heartbeat timeout. [AWS Lambda callback timing](https://docs.aws.amazon.com/step-functions/latest/dg/connect-lambda.html). + +SendTaskSuccess.output is a JSON string≤256 KiB. Failure.error/cause are≤256/32,768 bytes. Token input is≤2,048 bytes. Malformed, unknown or cross-scope tokens return InvalidToken; known revoked/completed/timed-out callback tasks return TaskTimedOut. TaskDoesNotExist is reserved for the nonexistent-activity API case, outside this callback-only profile. Retain bounded token tombstones for execution retention to preserve classification. + +Treat tokens as credentials. Infrastructure tracing/Debug/metrics/causes never contain raw tokens; the registry stores hashes. Before history persistence, redact values injected from Task.Token in materialized Parameters and mark execution-data truncation. Advertise this local history difference; never save raw tokens and hope a later logger redacts them. This does not promise scanning arbitrary business data for secrets. Use constant-time comparison where token material is compared and a production security-grade random source. + +## 6. Rust engineering contract and cross-references + +Follow AGENTS.md Error/Async/Type/Safety/Serialization: typed thiserror mappings, documented dyn object safety, supervisor-owned handles, bounded IO/channel budgets, validated private scope/ARN/token types, redacted Debug and camelCase internal envelopes. Tests cover absent adapters, HTTP200 function errors, alias updates, child subscription races, early callbacks, cancellation and deadlines. Tracing records IDs; internal calls avoid HTTP serialization and reparsing. Public APIs need runnable SDK examples and Errors documentation. + +- ← [API](./rustack-workflows-10-api-design.md), [Lambda](./rustack-workflows-20-lambda-runtime-design.md), [ASL](./rustack-workflows-30-asl-design.md) +- → [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md) +- ↔ [key decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md new file mode 100644 index 00000000..0866a5ee --- /dev/null +++ b/specs/rustack-workflows-60-operations-design.md @@ -0,0 +1,142 @@ +# Workflows 60: Configuration, Operations and Snapshot Boundaries + +Status: implementation contract · Depends on: [API](./rustack-workflows-10-api-design.md), [Lambda](./rustack-workflows-20-lambda-runtime-design.md), [ASL](./rustack-workflows-30-asl-design.md), [integrations](./rustack-workflows-40-integrations-design.md) + +## 1. Extend the existing configuration system + +Use `RUSTACK_CONFIG` and the existing YAML `environment`, `budgets`, `advertisedEndpoint` schema. Do not add a parallel configuration loader or mutate process environment. Add keys to the validated settings allowlist and add typed, camelCase budget fields. Process environment overrides YAML; defaults apply last. Reject unknown keys, malformed scalars, zero/excessive budgets and conflicting executor settings before listening. Configuration structs use typed-builder and validated newtypes. + +The following is a **proposed configuration after implementation**, not a statement that the current binary supports these keys: + +```yaml +environment: + SERVICES: "lambda,stepfunctions" + LAMBDA_EXECUTOR: "docker" + LAMBDA_DOCKER_SOCKET: "unix:///var/run/docker.sock" + LAMBDA_DOCKER_RELAY_IMAGE: "rustack/lambda-runtime:integration" + LAMBDA_DOCKER_NETWORK: "rustack-integration" + LAMBDA_DOCKER_PULL_POLICY: "never" + LAMBDA_DOCKER_GATEWAY_ENDPOINT: "http://host.docker.internal:4566" +budgets: + workflowOpenExecutions: 128 + workflowDispatches: 32 + workflowMaxDurationSeconds: 3600 + workflowHistoryBytes: 33554432 + workflowTotalHistoryBytes: 268435456 + lambdaDockerEnvironments: 32 + lambdaDockerMemoryBytes: 8589934592 +``` + +Start with `RUSTACK_CONFIG=./rustack.yaml rustack`; the current CLI does not have a `--config` flag. Existing `--snapshot ` remains the persistence entry point. Installing this feature must not silently enable code execution: absent explicit selection, Lambda remains disabled. + +### 1.1 Docker configuration + +| Key | Default and validation | +|---|---| +| LAMBDA_DOCKER_SOCKET | `unix:///var/run/docker.sock`; operator may select Desktop/rootless socket; absolute local socket path≤4,096 bytes; no TCP/SSH fallback | +| LAMBDA_DOCKER_RELAY_IMAGE | Required when Docker enabled; bounded image reference, resolved and pinned at startup; no implicit pull | +| LAMBDA_DOCKER_NETWORK | Operator-owned user bridge name/ID, or a per-owner private bridge created by Rustack;≤128 ASCII bytes; never host/container arbitrary input | +| LAMBDA_DOCKER_PULL_POLICY | never / ifMissing; default never; missing image gives actionable error | +| LAMBDA_DOCKER_REGISTRIES | Required for ifMissing; comma-separated≤32 approved registry hosts, each≤253 bytes; exact host[:port] matching, no suffix matching | +| LAMBDA_DOCKER_REGISTRY_AUTH_FILE | Optional operator-owned YAML credential map by approved registry host;≤64 KiB,≤32 entries, secret fields redacted and permissions checked; parse as a secret-bearing type, never execute credential helpers | +| LAMBDA_DOCKER_GATEWAY_ENDPOINT | Optional operator-trusted HTTP(S) endpoint reachable by functions;≤2,048 bytes; no userinfo/fragment; never derive from a request Host header | +| LAMBDA_DOCKER_UID / GID | Default1000; nonzero bounded numeric IDs; only overrides absent/root image USER | +| LAMBDA_DOCKER_CPU | Default1; finite decimal CPU units in 0.1..16; no zero/unlimited value | +| LAMBDA_DOCKER_LOG_RING_BYTES | Default262,144;4,096..1,048,576 per stdout/stderr ring | +| LAMBDA_DOCKER_CROSS_ARCHITECTURE | false; true permits an explicitly configured daemon emulation path, outside guaranteed performance | +| LAMBDA_DOCKER_ZIP_IMAGE | Required only for Docker ZIP; provided.al2023 image pinned by content/digest; no implicit runtime catalog | + +Retain LAMBDA_DOCKER_ENABLED as the existing alias for explicit docker selection; conflicts remain errors. Add configured local test credentials through the existing secret-bearing configuration mechanism; never print them or infer real AWS credentials from the host. The advertised gateway endpoint and the function-visible endpoint are independent; the former cannot automatically solve Docker networking. + +Registry credential records accept only approved host, username≤256 bytes, password≤4,096 bytes or identityToken≤8,192 bytes, with explicit mutual-exclusion rules and secret wrappers. The credential-file path is an operator-controlled local path, loaded with a timeout and restrictive file-ownership/permission checks. The configuration file must not be included in generated artifacts, snapshots or user-code mounts. + +### 1.2 Tunable budgets + +All budgets are validated once; limits below are local defaults and safe configuration maxima. Static AWS payload/definition quotas are compile-time constants and cannot be raised by YAML. + +| Budget field | Default | Allowed range / relationship | +|---|---:|---| +| workflowOpenExecutions |128|1..4,096, including children | +| workflowDispatches |32|1..1,024; released while waiting on child/callback | +| workflowFrames / workflowFramesPerExecution |4,096 /512|1..65,536 /1..4,096; per-execution≤global | +| workflowMaxDurationSeconds |3,600|1..31,536,000; independent local ceiling even without ASL TimeoutSeconds | +| workflowTransitions |100,000|1..1,000,000; history may exhaust first | +| workflowHistoryBytes / workflowTotalHistoryBytes |32 MiB /256 MiB|1..256 MiB /8 MiB..4 GiB; per-execution≤total | +| workflowCompletedExecutions |10,000|1..100,000; do not evict unexpired retained records | +| workflowRetentionSeconds |86,400|60..7,776,000; terminal history retention | +| workflowNameTombstones |100,000|1..1,000,000; always retain names 90 days | +| workflowPaginationTokens |10,000|1..100,000; token TTL 24h | +| workflowCallbackTokens / workflowCallbackTombstones |4,096 /100,000|1..65,536 /1..1,000,000; closed-token TTL follows execution retention | +| lambdaDockerEnvironments |32|1..32 initially, including busy/idle/quarantined environments | +| lambdaDockerMemoryBytes |8 GiB|256 MiB..1 TiB; includes relay caps and all allocated environments | +| lambdaDockerDeploymentJobs |8|1..32; queued plus running deployments | +| lambdaDockerPullSeconds / lambdaDockerStartSeconds |120 /15|1..600 /1..60 | +| lambdaDockerCleanupSeconds |10|1..60; bounded by global shutdown remaining time | +| workflowDrainSeconds |5|0..60; bounded by global shutdown remaining time | + +Registry/execution mailbox capacities and per-tick quantum are compile-time implementation constants from the ASL design, not runtime tuning knobs. Function init/idle settings reuse existing validated Lambda configuration. CPU and log limits use the exact environment keys above. Effective memory admission may allow fewer than 32 environments. Lower quotas must reject admission or fail the affected execution explicitly, never silently drop data/work. + +## 2. Deployment topology and readiness + +| Topology | Control path | Handler SDK path | Required test | +|---|---|---|---| +| Host Rustack + Linux Docker | Local socket + attach | Configured host-gateway address; bind gateway on an operator-approved reachable interface | Linux amd64/arm64 | +| Host Rustack + Docker Desktop | Desktop socket + attach | host.docker.internal with reachable gateway bind | macOS arm64 | +| Rustack server container + sibling functions | Mounted daemon socket + attach | Shared user bridge and stable Rustack service alias | Linux container lane | + +The host example requires a suitable gateway listener; the default host loopback listener is not universally reachable from containers. Do not change the default listener merely because Docker is enabled. Startup diagnostics explain the chosen endpoint and its probe status. With no function-visible endpoint, pure handlers may run, but capabilities state that SDK callbacks are unconfigured. rootless Docker requires its limit capabilities to be probed; unsupported resource enforcement fails explicit Docker readiness rather than removing limits. + +Expose capability fields for workflow type/query profile/state types/integrations, local quotas, snapshot coverage, Docker platform, backend probe result, relay protocol version and limitations. Distinguish available adapter from successfully executed test fixture. Health endpoints remain free of ordinary business admission starvation. Metrics contain bounded error classes/counts/durations, not function names, input, image credentials or task tokens as unbounded labels. + +## 3. Shutdown, cancellation and snapshots + +```text +┌─ admission boundary ───────────────────────────────────────────────┐ +│ mark draining; reject new external work │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ producers and workflow workers ───────────────────────────────────┐ +│ Events/SNS quiesce → Step Functions drain/cancel → Lambda quiesce │ +│ owned children / tokens / invocations cleaned │ +│ DDB and other existing workers retain their required dependencies │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ SQS/final downstream quiescence +┌─ consistency cut ──────────────────────────────────────────────────┐ +│ no RUNNING execution; no unresolved container cleanup │ +│ success → existing atomic snapshot save → workers shutdown │ +│ failure → retain prior snapshot; diagnostics; nonzero exit │ +└───────────────────────────────────────────────────────────────────┘ +``` + +The workflow drain interval permits already accepted executions to finish. Then cancel remaining executions with ABORTED and error=Rustack.Shutdown, preserving terminal history. This is deliberate local shutdown behavior, not cloud-style pause/resume. Revoke callback tokens, cancel owned children and await Lambda cleanup before Lambda itself quiesces. Quarantine or unfinished cleanup prevents publishing a successful consistency snapshot. Quiescence must be idempotent and share one global absolute deadline, not restart a full timeout per worker. + +Snapshot coverage: + +| Resource | Persist | Restore behavior | +|---|---|---| +| Machine definitions/revisions, role/config/tags | Yes | Validate, compile and stage all records before replacing live state | +| Terminal executions/history and captured machine revision | Yes, within retention/byte budgets | Preserve outcomes, causality, pagination source data and DescribeStateMachineForExecution | +| Name tombstones | Yes | Retain remaining90-day reservations; reject overflow rather than shorten retention | +| RUNNING records in imported/unexpected archives | No resumption | Convert to ABORTED/Rustack.ExecutionInterrupted before serving; never replay uncertain side effects | +| Callback tokens/live handles/pagination handles | No | Fresh runtime epoch invalidates old handles; no plaintext token archive | +| Lambda image content references/config | Yes | Validate same digest/content availability before image execution; no tag re-resolution | +| Docker image layers, containers, tmpfs and code-process state | No | Images must be preloaded/allowed pulls must fetch exact digest; environments start cold | + +Use the existing versioned service archives: `services/stepfunctions/meta.ss.zst` and `data.ss.zst` for definitions/indices and bounded history payloads. Store durable DTOs, not Arc/compiler heap/channel layouts. Recompile definitions with an archived profile version. Unknown profile versions or invalid references fail import atomically. Add versioned Lambda image metadata to its existing archive, preserving the existing artifact protections. Imported run records do not authorize deletion of containers from another owner. + +Reserve terminal-record and name-tombstone capacity at Start admission so accepting a run never depends on finding retention capacity after its side effects finish. Reserve the terminal history allowance independently of ordinary event append capacity. History deletion cannot release a still-live name reservation. + +There is no WAL: SIGKILL may lose all requests since the previous completed snapshot. A running-record conversion can explain an imported historical record; it does not recover executions that were never saved. Old completed snapshots remain recoverable under the existing replacement protocol. Restored pagination handles are invalid, and this epoch boundary is advertised. + +## 4. Debugging and engineering contract + +Diagnostics identify deployment Pending/Failed, backend unavailable, invalid image architecture, missing Runtime API polling, handler error, Task timeout, callback expiry, child failure and cleanup quarantine separately. Payload-bearing execution history is an intentional API capability; infrastructure logs are redacted. Operator log access remains a trust boundary. + +Follow AGENTS.md Error/Async/Type/Safety/Serialization/Testing/Logging/Performance/Documentation: config errors identify key/source without secret values; resource ownership remains in actors; validation is repeated on archive import; decompression has byte/count/depth limits; shutdown tasks are joined; config and snapshot fixtures test rejection/atomicity; use existing archive mechanisms rather than per-record files. Authentication configuration reuses current gateway behavior; no new public Runtime API listener is created. + +Explicit exception to AGENTS.md § Injection Prevention, URL/SSRF: an operator-configured local gateway may use HTTP and private/loopback addressing because local AWS emulation requires it. It is not an ASL-controlled outbound URL, and request Host headers cannot configure it. Function networking and Docker daemon registry networking remain separate operator trust boundaries. Local cancellation actively terminates a function container; this is resource hygiene, not a promise that AWS Lambda would stop code when a Step Functions Task times out. Already completed side effects survive both paths. + +## 5. Cross-references + +- ← [runtime hardening](./rustack-runtime-hardening-design.md), [binary archives](./ruststack-snapshot-binary-archive-design.md) +- ↔ [verification](./rustack-workflows-70-verification-plan.md), [implementation](./rustack-workflows-90-impl-plan.md), [decisions D8/D10](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md new file mode 100644 index 00000000..9f16a773 --- /dev/null +++ b/specs/rustack-workflows-70-verification-plan.md @@ -0,0 +1,105 @@ +# Workflows 70: Verification, Security and Performance Gates + +Status: planned acceptance criteria; no new runtime tests executed · Applies to M0–M4 + +## 1. Evidence and oracle policy + +Use independent expected fixtures from official AWS API/language documentation and sanitized, provenance-recorded AWS observations where available. Each fixture records source/date, SDK/model version, supported profile and normalized nondeterministic fields. Do not calculate expected results with the same evaluator under test. Existing bootstrap fixtures and the [Cargo Lambda spike](../docs/research/spike-cargo-lambda-runtime-execution.md) are useful evidence, not proof of Docker or ASL execution. + +Ordinary CI is offline after image provisioning and never contacts AWS. Optional AWS differential tests require an explicitly configured isolated test account and an intentional command; they are not run as part of this specification task. If no AWS account is available in P0, use official protocol fixtures and a documented evidence limitation; do not claim cloud differential parity. Any unresolved behavior that changes acceptance semantics blocks the corresponding release gate until a documented fixture or narrower profile resolves it. + +## 2. Required test matrix + +| ID | Scenario | Observable acceptance | Gate | +|---|---|---|---| +| L01 | Real image handler returns a computed value and invocation counter | Correct value and increasing counter; no echo path; warm reuse visible | M0 | +| L02 | Official Python/Node RIC image plus a custom Rust RIC image | At least one official managed-language image and custom RIC pass; both Python and Node pass by M4 | M0/M4 | +| L03 | ImageConfig overrides ENTRYPOINT/CMD/WORKDIR, including explicit empty arrays | Correct argv/cwd; absent differs from empty; no shell evaluation | M0 | +| L04 | Tag moves A→B, publish A, update latest B, alias movement | Published A stays A, latest uses B only after update, old in-flight revision completes A | M0 | +| L05 | Wrong OS/architecture, no RIC, init/error, early exit, OOM, malformed Runtime API | Distinct failures, no successful payload and no retained unhealthy environment | M0 | +| L06 | Two concurrent functions and a third forged completion | No cross-invocation response, stale/duplicate/foreign request rejected | M0 | +| L07 | Timeout/HTTP cancellation during allocating/init/busy/cleanup | Terminal failure once; function/relay removed or quarantined under retained ownership | M0 | +| L08 | Docker unavailable, interrupted pull, missing image, deployment saturation | Bounded failure; previous revision preserved; no Active placeholder | M0 | +| L09 | Reserved=0, global/per-function admission, memory/environment caps | Explicit throttling; permits not released before cleanup; no unbounded allocation | M0 | +| L10 | Container SDK calls Rustack S3/DDB | Real local result, no host credential inheritance; test enables those services explicitly | M4 | +| L11 | provided.al2023 Cargo Lambda ZIP in Docker | Same Linux bootstrap artifact executes on its native architecture | M4 | +| A01 | Create/update/delete/list/tag/describe through Rust SDK and boto3 | Correct JSON protocol, ARNs, timestamp types and errors | M1 | +| A02 | Concurrent Start with same name/input and mismatched input | One execution; same original response for RUNNING duplicate; correct collision error otherwise | M1 | +| A03 | Definition update while execution waits | Running execution uses old revision; new execution uses new revision | M1 | +| A04 | Pass/Task/Choice/Wait/Succeed/Fail | Correct transitions, outputs and terminal status | M1 | +| A05 | Full InputPath→Parameters→result→ResultSelector→ResultPath→OutputPath | Merge uses original raw input; missing/null/selection distinctions match fixtures | M1 | +| A06 | Path quoting/index/wildcard/slice, invalid grammar and duplicate .$ output keys | Correct bounded results or creation diagnostics | M1 | +| A07 | Every allowed intrinsic and nesting/operation/output boundary | Documented AWS-compatible fixture outputs; unsupported functions rejected | M1 | +| A08 | Choice short-circuit, missing/null/wrong-type, StringMatches and timestamps | Guarded missing path safe; no coercion/regex injection; NoChoiceMatched diagnosed | M1 | +| A09 | Retry ordered matchers, zero/default counts, backoff/jitter, Catch merge | Exact attempt count and bounded delay; no hidden adapter retries | M1 | +| A10 | Lambda timeout versus Task timeout versus execution timeout | Distinct errors/terminal states; execution timeout cannot be caught internally | M1 | +| A11 | States.Runtime and explicit DataLimitExceeded matching | Fatal runtime bypasses Catch; ALL does not catch data limit; explicit matching does | M1 | +| A12 | History pagination, reverseOrder, data omission, forged/expired tokens | Stable snapshot boundary and causal IDs; no cross-scope access | M1 | +| A13 | History/byte/transition budget exhaustion | Explicit failure with reserved terminal record; never successful truncated execution | M1 | +| I01 | Direct versus optimized Lambda with FunctionError/HTTP200 | Correct result envelope and Task failure mapping | M1 | +| I02 | Child finishes before subscription consumer runs | Parent always receives terminal result, no missed wakeup | M2 | +| I03 | startExecution versus .sync versus .sync:2 | Acceptance-only detached child; waiting owned child; string/JSON Output distinction | M2 | +| I04 | Parent Stop/timeout, child timeout/failure, simultaneous completion | Correct ownership/cancellation/status; one settlement | M2 | +| I05 | Nested parents saturate open/dispatch limits | Rejection or progress, no parent-holds-child-slot deadlock | M2 | +| P01 | Parallel completes out of order, Map completes out of order | Outputs preserve definition/item order | M2 | +| P02 | Map empty input, ItemProcessor/Iterator, MaxConcurrencyPath, nested Map | Correct empty/legacy/current behavior, bounded frames/concurrency | M2 | +| P03 | Branch failure plus state-level Retry/Catch | Siblings cancelled/joined; whole container state reruns, not failed branch alone | M2 | +| C01 | Callback before/after Lambda response, Lambda failure after early callback | Provisional result settles only after successful invoke; errors are not hidden | M3 | +| C02 | Heartbeat reset, overall timeout, retry and old token reuse | Task deadline unchanged; old token cannot complete new attempt | M3 | +| C03 | Wrong-scope/forged/expired/duplicate tokens | Defined API errors; no plaintext in tracing/history/archive | M3 | +| O01 | Graceful shutdown during Wait/Retry/Map/.sync/callback | Correct ABORTED cutoff, downstream cleanup, no false snapshot success | M4 | +| O02 | Restore terminal history/definitions/names; import RUNNING record | Atomic restore; RUNNING becomes interrupted, never replayed | M4 | +| O03 | Corrupt archive, changed tag, missing pinned image, owner mismatch | No partial restore or unsafe cleanup; no tag substitution | M4 | +| O04 | Two Rustack instances share daemon; one crashes | Restart removes only owned abandoned containers; other instance survives | M4 | +| O05 | Infinite ASL loop, log flood, large nested payload, oversized frame | Bounded memory/CPU/output; health and Stop remain responsive | M4 | + +Every row needs positive and failure-path assertions where applicable. Tests involving Docker must assert container ownership/cleanup after completion and failure; checking only response JSON is insufficient. + +## 3. Test layers and automation + +1. Unit/parameterized tests: compiler, Path/intrinsic evaluator, retry matcher, data pipeline and protocol error mapping. Use descriptive `test_should_` names and rstest. +2. Property tests: no panic on bounded malformed input; valid graph transitions stay in scope; generation invalidation prevents double completion; result ordering independent of completion order; history causes refer only to earlier events. +3. Paused-time tests: deadlines, backoff/jitter ranges, token lifetimes and completion/cancel races using Tokio test clocks. Use real clock tests only for integration boundaries. +4. In-process integration: actual providers and binary adapters; controlled external ports only for forcing failures. Avoid replacing the entire engine with mocks. +5. SDK/HTTP integration: AWS SDK for Rust plus boto3, exact endpoint and test credentials; verify headers, field casing, JSON strings and errors. +6. Docker integration: native-architecture official/custom images, fault injection and ownership checks on all required platform lanes. Missing Docker may skip a local optional suite, but a required CI lane must fail if prerequisites are missing. +7. IaC smoke: basic Pulumi aws.sfn.StateMachine create/refresh/update/destroy with supported fields and a deployed image Lambda; versions/logging/encryption outside profile must reject explicitly. + +Add future Makefile targets: `codegen-stepfunctions`, `test-stepfunctions`, `test-lambda-docker`, `smoke-workflows`, and `verify-workflows`. Add image provisioning/build targets instead of shell scripts. Pin fixture-image digests/content IDs and expected test manifest entries. `verify-workflows` runs the exact required suites and checks that none were silently skipped. Reuse the existing build/fmt/clippy/audit/deny targets. + +For Rust-relevant implementation work run cargo build, cargo test, cargo +nightly fmt, and cargo clippy -- -D warnings, plus the repository's stricter pedantic/boundary lint selection and existing applicable targets. Run cargo audit and cargo deny check when dependencies land. This documents future implementation gates; the current spec-only task changes no executable/toolchain behavior and does not require Rust compilation. + +## 4. Security acceptance + +| Boundary | Required proof | +|---|---| +| HTTP/ASL/archives | Byte/count/depth bounds before expansion; duplicate-key rejection; invalid values cannot reach unchecked engine indexing | +| Docker control | Local trusted socket only, no user-selected host mounts/socket/devices, no Runtime API port publish | +| Environment | Unprivileged identity, limits actually enforced, read-only code/rootfs, bounded tmpfs/logs | +| Invocation identity | One active attempt per environment; duplicate/stale/foreign completion cannot settle another call | +| Secrets | Redacted Debug/tracing, no host credentials, token hashes/tombstones, injected token redaction before history persistence | +| Cross-service | Scope validated at compiler and dynamic invocation; adapter cannot bypass Lambda admission | +| Shutdown | No forgotten resources; cleanup uncertainty blocks successful consistency snapshot | + +Apply AGENTS.md boundary lints: unwrap_used, expect_used, indexing_slicing and panic. Dependency security audit is required evidence, not a claim derived from popularity. This local executor does not claim hostile multi-tenant containment or IAM policy parity. + +## 5. Performance and resource acceptance + +Use a recorded reference machine: native-architecture Linux, 4 dedicated CPUs, 8 GiB RAM and local Docker; also record Desktop results separately. Provision images before timing. These are provisional budgets to measure during implementation, not existing benchmarks: + +| Measurement | Budget / method | +|---|---| +| Pure 50-state Pass/Choice execution, 1 KiB input | p95≤100 ms over 100 sequential executions, no Docker | +| Warm image Lambda adapter overhead | p95≤50 ms beyond handler time over 100 calls, concurrency 1 | +| Preloaded-image cold environment | p95≤5s over 20 environments; report relay and RIC init separately | +| Stop observability | Terminal/readiness response within cleanup budget; no health starvation under pure loops/log flood | +| Bounded storage |≤configured history/frames/containers/log limits; record allocator/RSS overhead separately rather than treating quotas as a total RSS claim | +| Steady-state cleanup | No growing container/task count after 100 invoke/timeout cycles | + +Do not introduce a benchmark crate early. First measure these integration workloads and profile deviations; use current criterion only when a stable hot-path benchmark is justified. A missed provisional budget requires recorded analysis and an explicit spec amendment, not deleting the measurement. + +## 6. Cross-references + +- ← [all component contracts](./rustack-workflows-00-prd.md#7-reading-order-and-cross-references) +- → [roadmap exits](./rustack-workflows-80-roadmap.md), [implementation gates](./rustack-workflows-90-impl-plan.md) +- ↔ [operations](./rustack-workflows-60-operations-design.md), [decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-80-roadmap.md b/specs/rustack-workflows-80-roadmap.md new file mode 100644 index 00000000..7c02bef9 --- /dev/null +++ b/specs/rustack-workflows-80-roadmap.md @@ -0,0 +1,64 @@ +# Workflows 80: User-Facing Delivery Roadmap + +Status: proposed milestone commitments · Date: 2026-10-25 + +## 1. Delivery principles + +Each milestone is usable end to end and preserves bounded execution, honest errors and explicit capability reporting. Unfinished capabilities remain rejected. M0 is useful independently, but the requested workflow/runtime feature is complete only after M4. The [implementation plan](./rustack-workflows-90-impl-plan.md) orders engineering work by dependency; this roadmap groups outcomes a consuming project can use. + +```text +┌─ P0: evidence, contracts and risk retirement ───────────────────────┐ +│ official RIC + Docker relay proof │ semantic fixtures / model pin │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M0: run real image Lambdas ────────────────────────────────────────┐ +│ deploy → immutable digest/version → actual Invoke → cleanup │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M1: test sequential serverless workflows ─────────────────────────┐ +│ SDK → ASL/data flow → Docker Lambda → Retry/Catch → history │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M2: coordinate fan-out and nested jobs ────────────────────────────┐ +│ Parallel/Inline Map │ child execution │ .sync/.sync:2 cancellation │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M3: test callback workflows ──────────────────────────────────────┐ +│ Task.Token → real Lambda → SendTask* → heartbeat/deadline │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ M4: use in consuming-project CI ──────────────────────────────────┐ +│ provided ZIP │ snapshots │ platform matrix │ fault/load/IaC gates │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## 2. Milestones + +| Milestone | What another project can do | Exit evidence | Engineering phase | +|---|---|---|---| +| M0 — Actual image execution | Create/update an Image Lambda, publish/alias it and receive the handler's actual result using a normal SDK | L01–L09; official RIC and custom Rust fixture; digest/version isolation and owned cleanup | P1 | +| M1 — Sequential workflows | Create STANDARD definitions and test six state types, transformations, real Lambda tasks, retries, catching, waiting and timeouts | A01–A13, I01; Rust SDK+boto3; PRD scenario succeeds and deliberate failures fail | P2 | +| M2 — Fan-out and nested completion | Execute all eight state types, bound Inline Map concurrency, wait for real child workflows and stop owned subwork | I02–I05, P01–P03; correct order/types and no missed completion/deadlock | P3 | +| M3 — Callback workflows | Pass Task.Token to a real function and complete/fail/heartbeat its task safely | C01–C03; early callbacks, retries and revoked tokens covered without token disclosure | P4 | +| M4 — Reusable local/CI dependency | Use supported image/ZIP artifacts, snapshots and a basic IaC lifecycle across the platform matrix | L02 completed, L10/L11, O01–O05; complete verification manifest passes without required skips | P5 | + +All milestone evidence refers to the [verification plan](./rustack-workflows-70-verification-plan.md). Each exit also includes the applicable repository quality gates and fresh capability output. A demo without negative cases does not close a milestone. + +## 3. Calendar and confidence + +Assume one experienced Rust engineer, a working Docker environment, preexisting gateway/Lambda hardening, no distributed persistence requirement and access to both Linux architectures plus Docker Desktop. Estimate 56–83 engineering days including P0, then add 25% for review, integration and platform troubleshooting: approximately **14–21 calendar weeks**. These are planning estimates based on identified work, not measured delivery velocity. + +| User-visible checkpoint | Approximate cumulative calendar window | +|---|---| +| M0 | Weeks 4–6 | +| M1 | Weeks 8–12 | +| M2 | Weeks 10–15 | +| M3 | Weeks 12–18 | +| M4 | Weeks 14–21 | + +P0 is the first re-estimation point. If relay portability fails, or semantic evidence forces a broader expression engine, revise this roadmap before implementation commitments. Image execution, ASL semantics and shutdown correctness are substantive work, not a small API addition. Optional future scope—EXPRESS, JSONata, ECS/Batch, WAL recovery—has no implicit delivery date in this estimate. + +## 4. Cross-references + +- ← [PRD](./rustack-workflows-00-prd.md) +- ↔ [implementation plan](./rustack-workflows-90-impl-plan.md), [verification](./rustack-workflows-70-verification-plan.md), [decisions](./rustack-workflows-99-decisions.md) diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md new file mode 100644 index 00000000..fb04bd3f --- /dev/null +++ b/specs/rustack-workflows-90-impl-plan.md @@ -0,0 +1,145 @@ +# Workflows 90: Dependency-Ordered Implementation Plan + +Status: P0 complete; P1 next; complete M0–M4 implementation objective remains active · Date: 2026-10-25 + +## 1. Readiness and dependencies + +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, host Runtime API, immutable artifact hardening, invocation admission, binary-owned cross-service adapters, validated YAML settings, runtime worker ownership and service snapshots. Missing: Docker executor/relay and image resolution, full ImageConfig propagation, Step Functions protocol/model/core, ASL engine, integration handles, workflow snapshots and test fixtures. + +The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. + +```text +┌─ P0 evidence ──────────────────────────────────────────────────────┐ +│ relay/RIC/platform spike │ protocol fixtures │ source/deps audit │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ +┌─ shared contracts, landed before consumers ────────────────────────┐ +│ revision/artifact enum │ invocation handle/cancel │ scope/errors │ +│ RuntimeSession + transport │ supervisor ownership │ budget schema │ +└───────────────┬──────────────────────────────┬─────────────────────┘ + ▼ ▼ +┌─ P1 / M0 ────────────────────┐ ┌─ P2 foundations ────────────────┐ +│ image resolver + Docker pool │ │ Smithy API + registry + compiler│ +│ relay + provider lifecycle │ │ actor/timer/history spine │ +└───────────────┬──────────────┘ └──────────────┬───────────────────┘ + └────────────────┬───────────────┘ + ▼ P2 closes M1: real Lambda tasks +┌─ P3 / M2 ──────────────────────────────────────────────────────────┐ +│ frame scheduler → Parallel/Map → child start/subscription/.sync │ +└─────────────────────────────┬──────────────────────────────────────┘ + ▼ P4 / M3: token registry + callbacks +┌─ P5 / M4 ──────────────────────────────────────────────────────────┐ +│ ZIP containers │ snapshots/shutdown │ platform + fault + IaC gates │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## 2. Effort model + +| Phase | Outcome | Engineering days | +|---|---|---:| +| P0 | Resolve load-bearing risks |4–6| +| P1 | Close M0 |12–17| +| P2 | Close M1 |15–22| +| P3 | Close M2 |10–15| +| P4 | Close M3 |6–9| +| P5 | Close M4 |9–14| +| Total | Complete this specification set |56–83| + +One developer plus 25% calendar overhead gives 14–21 weeks. Estimates explicitly include test/error/cleanup work; they are not benchmarks or guarantees. Independent fixture preparation can overlap design work, but API/adapter contracts and cancellation semantics must stabilize before consumers. Do not count theoretical parallelism as completed evidence. + +## 3. P0 — Evidence and risk retirement + +| Task | Deliverable | Contract | Days | +|---|---|---|---:| +| 0.1 | Reproducible isolated relay spike; Docker attach framing/EOF, netns sharing, official RIC, unprivileged limits and owned cleanup | [Lambda §§1–5](./rustack-workflows-20-lambda-runtime-design.md) |1–2| +| 0.2 | Semantic fixture catalog and memo: input equality, data null/missing, intrinsic/Choice edge cases, history event shapes, nested result/date types, boundary error matching | [ASL](./rustack-workflows-30-asl-design.md), [verification §1](./rustack-workflows-70-verification-plan.md#1-evidence-and-oracle-policy) |1–2| +| 0.3 | Pin official SFN model commit/hash and chosen SDK/fixture-image versions; prove numeric timestamp generator compatibility and dependency feature/security status | [API §1](./rustack-workflows-10-api-design.md#1-protocol-and-crate-boundaries) |1| +| 0.4 | Record measured spike outcomes, amend decisions/design/profile and recalibrate estimates; resolve incompatible assumptions | [decisions](./rustack-workflows-99-decisions.md) |1| + +Artifacts: [relay spike](../docs/research/spike-lambda-container-relay.md) and [semantic oracle](../docs/research/spike-stepfunctions-semantic-oracle.md), with retained runnable probes, fixtures and exact commands/environment. Both memos now exist and are indexed. Spike-only automation belongs in Makefile if retained in the repository; no production workaround is hidden in the experiment. + +**Exit:** actual relay+RIC execution and failure cleanup pass on at least Linux amd64 and macOS arm64; Linux arm64 lane is provisioned; source/fixture provenance is recorded; no unresolved question can change the selected public contract. Cloud differential evidence may remain unavailable only under the verification plan's explicit narrower-evidence policy. A failed spike requires a design amendment before P1. + +## 4. P1 — Runtime contracts and real Image Lambda, closes M0 + +| Task | Strict dependency order | Contract | Days | +|---|---|---|---:| +| 1.1 | Land validated artifact/revision enum, full ImageConfig/ephemeral propagation and cancellable invocation handles; adapt HTTP and existing backends without behavior regressions | [Lambda §§1–2](./rustack-workflows-20-lambda-runtime-design.md), [ports](./rustack-workflows-40-integrations-design.md#1-internal-ports-and-dependency-direction) |3–4| +| 1.2 | Extract RuntimeSession state; add bounded relay codec/server artifact, socket client and startup probes | [Lambda §3](./rustack-workflows-20-lambda-runtime-design.md#3-runtime-api-and-relay-transport) |3–4| +| 1.3 | Add image resolution/digest records, Pending/update lifecycle, deployment workers, revision publication and pool/environment actors | [Lambda §§2–5](./rustack-workflows-20-lambda-runtime-design.md), [configuration](./rustack-workflows-60-operations-design.md) |3–4| +| 1.4 | Wire provider/config/capabilities, remove obsolete Docker rejection branches, implement cleanup/quarantine/reconciliation and L01–L09 | [verification §2](./rustack-workflows-70-verification-plan.md#2-required-test-matrix) |3–5| + +Add appropriate Makefile fixture-image/runtime build and `test-lambda-docker` targets. Update workspace dependencies/features once, auditing bollard and the relay artifact. Remove obsolete fallback branches rather than deprecating unavailable execution paths. Keep default disabled, Auto and Native/Squib behavior explicit. + +**Exit:** ordinary SDK invokes a real image on the P0 lanes, returns computed output, distinguishes failures, preserves published code across updates, and cleans/quarantines every failed allocation under bounded ownership. Existing Lambda quality gates pass. No requirement is satisfied using a noop/echo executor. + +## 5. P2 — API, compiled semantics and sequential engine, closes M1 + +| Task | Strict dependency order | Contract | Days | +|---|---|---|---:| +| 2.1 | Implement protocol-aware numeric timestamp generation with required/optional/fractional/negative tests; import pinned model/config, generate model/HTTP, add service routing and feature/settings/health registration; registry/revisions/names/tags/read views | [API §§1–5](./rustack-workflows-10-api-design.md) |3–4| +| 2.2 | Build bounded strict compiler, scoped IR, Path/intrinsic evaluators and exact data transformations against fixtures | [ASL §§1–2](./rustack-workflows-30-asl-design.md) |4–6| +| 2.3 | Execution actor/timer/generation/history spine, six sequential states, Retry/Catch/deadlines and shutdown ownership | [ASL §§3–4](./rustack-workflows-30-asl-design.md), [API §6](./rustack-workflows-10-api-design.md#6-history-and-pagination) |4–6| +| 2.4 | Binary Lambda bridge, direct/optimized results/errors, Rust SDK+boto3 coverage and complete A01–A13/I01 | [integrations §3](./rustack-workflows-40-integrations-design.md#3-lambda-requestresponse) |4–6| + +Add `codegen-stepfunctions`, `test-stepfunctions` and a sequential `smoke-workflows` target. Compiling a not-yet-implemented Parallel/Map/callback definition returns a precise unavailable-profile diagnostic until its milestone. Model presence alone does not advertise execution support. + +**Exit:** PRD example executes its real image through ASL; failure, Retry/Catch, timeout and history semantics pass independent fixtures through both SDK clients. Duplicate Start/update races and forged pagination are tested. Stop cannot be starved by a tight loop. + +## 6. P3 — Structured fan-out and nested jobs, closes M2 + +| Task | Dependency order | Contract | Days | +|---|---|---|---:| +| 3.1 | Extend frame scheduler for lazy branches/items, stable result ordering and global/per-execution frame budgets | [ASL §5](./rustack-workflows-30-asl-design.md#5-parallel-and-inline-map) |3–4| +| 3.2 | Parallel/Map Retry/Catch, legacy normalization, sibling cleanup and P01–P03 | [ASL §5](./rustack-workflows-30-asl-design.md#5-parallel-and-inline-map) |4–6| +| 3.3 | Child start/subscribe/cancel contracts, .sync/.sync:2 output mapping, admission/depth rules and I02–I05 | [integrations §4](./rustack-workflows-40-integrations-design.md#4-child-workflows-and-sync) |3–5| + +**Exit:** all eight state types run; nested workflow example waits for actual child Lambda completion; string/JSON result distinction, fast-child races, cancellation, deadlock saturation and out-of-order fan-out tests pass. Unsupported service suffixes still reject. + +## 7. P4 — Callback tasks, closes M3 + +| Task | Dependency order | Contract | Days | +|---|---|---|---:| +| 4.1 | Token hashes/scope/generation/tombstones, provisional callback protocol and injected-token history redaction | [integrations §5](./rustack-workflows-40-integrations-design.md#5-callback-and-heartbeat-m3) |2–3| +| 4.2 | Lambda callback adapter, SendTask* routing, heartbeat/Task timers and cancel/retry invalidation | [integrations §5](./rustack-workflows-40-integrations-design.md#5-callback-and-heartbeat-m3) |2–3| +| 4.3 | Actual callback-capable image fixture, C01–C03, token exposure scans and race coverage | [verification](./rustack-workflows-70-verification-plan.md) |2–3| + +**Exit:** real handler callbacks work before/after Lambda return; Lambda failure cannot be hidden; old tokens never settle a new attempt; tracing/history persistence do not contain injected plaintext token values. + +## 8. P5 — Persistence and consuming-project qualification, closes M4 + +| Task | Dependency order | Contract | Days | +|---|---|---|---:| +| 5.1 | provided.al2023 Docker ZIP execution and same-artifact fixture | [Lambda §6](./rustack-workflows-20-lambda-runtime-design.md#6-responses-errors-and-zip-support) |2–3| +| 5.2 | Workflow quiescence ordering, versioned service archives, image reference metadata and atomic restore validation | [operations §3](./rustack-workflows-60-operations-design.md#3-shutdown-cancellation-and-snapshots) |3–4| +| 5.3 | Three-platform lanes, server-in-container topology, fault/load cases, SDK service callbacks, IaC lifecycle and L10/L11/O01–O05 | [verification §§2–5](./rustack-workflows-70-verification-plan.md) |3–5| +| 5.4 | Final documentation/capability reconciliation, exact required-suite manifest and independent implementation review against this spec | [PRD goals](./rustack-workflows-00-prd.md#3-users-and-measurable-goals) |1–2| + +Add `verify-workflows` and extend existing snapshot/smoke targets rather than introducing parallel scripts. Record measured performance and platform limitations. Update existing service documentation only with features actually verified. + +**Exit:** every required verification row passes, no required lane silently skips Docker, all repository Rust/dependency checks relevant to the implementation pass, and unresolved correctness/security findings are fixed. Snapshot behavior and local compatibility limitations are user-visible. This closes the requested work. + +## 9. Why this order is necessary + +Cancellation/revision contracts precede both the Docker executor and workflow bridge; otherwise stopping a Task leaks resources or reuses mutable code. Compiler and history semantics precede Parallel/Map so branch machinery has one data/error contract. Child subscription/admission rules precede `.sync` so fast completion cannot be missed and parents cannot block child capacity. Snapshot integration follows fully settled lifecycle semantics, avoiding archives that appear resumable while silently replaying side effects. + +For implementation, invoke the repository's impl workflow one phase at a time and its independent review requirement; this specification task itself performs no production implementation or runtime acceptance. + +## 10. Cross-references + +- ← [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md), [PRD](./rustack-workflows-00-prd.md) +- ↔ [roadmap](./rustack-workflows-80-roadmap.md), [verification](./rustack-workflows-70-verification-plan.md), [decisions](./rustack-workflows-99-decisions.md) + +## 11. Implementation execution record + +The complete implementation objective includes P0–P5. This ledger records evidence, not reduced scope. TaskCreate is unavailable in this environment; task rows below provide the phase tracking required by the impl workflow. + +| Task | State | Evidence / next action | +|---|---|---| +| 0.1 | Evidence complete | Real official RIC image execution, warm reuse, handler failures, cgroup/nonroot/rootfs checks, busy-handler cancellation, attach EOF comparison and label reconciliation passed on arm64 VZ and Linux amd64/QEMU. Native x86/Desktop/rootless qualification remains M4. | +| 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | +| 0.3 | Evidence complete | Model commit/hash and37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task2.1. Rust updated to current stable1.99.1. | +| 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | +| P1–P5 | Pending | Remain required; no runtime implementation or acceptance is claimed yet. | + +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task1.1; P0 does not advertise Docker execution as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md new file mode 100644 index 00000000..edfd20fc --- /dev/null +++ b/specs/rustack-workflows-93-review.md @@ -0,0 +1,42 @@ +# Workflows 93: Implementation Review and Required Follow-ups + +Status: active implementation record · Date: 2026-10-25 + +This is the canonical review backlog for P0–P5. A corrected specification is not evidence that its runtime consumer has been implemented. All required in-scope entries must close before the complete feature is declared done. + +## P0 independent review + +| ID | Priority / source | Finding and resolution | Remaining implementation gate | +|---|---|---|---| +| W01 | P1; `apps/rustack/src/main.rs:373`, runtime transport contract | Bollard logs request/response/attach bytes at debug/trace. Specification now requires an independent mandatory dependency filter, even with RUST_LOG=trace | P1: implement filter and captured-log secret/payload regression | +| W02 | P1; runtime protocol contract | BootstrapReady was missing from the message variants despite being required by initialization. Specification fixed; real relay probe uses the separate signal | P1: implement production typed signal and initialization tests | +| W03 | P1; runtime create contract | Omitted Architectures must default to x86_64, unlike an explicit empty list. Specification corrected | P1: normalize before validation and cover omission/empty/multiple | +| W04 | P1; `codegen/src/shapes.rs:180`, `codegen/src/codegen.rs:263` | Generated chrono fields do not accept numeric AWS JSON timestamps. Independently reproduced; required/optional/fractional/negative codec behavior pinned | P2 task2.1: generator fix and executable generated round trips before service generation | +| W05 | P2; callback error contract | Known completed/revoked callback tasks use TaskTimedOut, not TaskDoesNotExist. Specification and fixtures corrected | P4: implement token tombstone classification and retry/cancel cases | +| W06 | P1; pinned AWS-validated `test_callback.snapshot.json:1251` | `.sync:2` decodes Input as well as Output. Root's subsequent oracle inspection corrected the specification and added TaskSubmitted | P3: validate complete optimized envelope, dates and history sequence | + +All six changes preserve the intended scope. No review finding has been dismissed as invalid. P0 probe review produced the following additional fixes. + +## P0 retained-probe review + +| ID | Priority | Finding and fix | Evidence | +|---|---|---|---| +| W07 | P2 | Create-response cancellation can lose an ID. Allocate a run UUID before requests, label every resource, reconcile labels under a bounded cleanup deadline, and deliberately omit one create result from the ID registry | Both retained architecture runs report labelReconciliation=true and independent daemon queries find no P0 containers | +| W08 | P2 | Close/Ack did not establish EOF behavior. Add clean/truncated-prefix/truncated-body EOF tests and compare real attach-input shutdown with StdinOnce false/true | Seven unit tests pass; both architecture runs show false leaves the relay running and true exits before cleanup. Production spec now requires true | +| W09 | P3 | Offline Python tests generated untracked bytecode | Probe target sets PYTHONDONTWRITEBYTECODE=1; generated test bytecode removed and excluded from staged files | + +The independent final recheck confirmed W07–W09 addressed, queried both daemon run labels and found zero containers, and reported no further in-P0 correctness issue. Production consumers remain subject to W01–W06. + +## Existing baseline issues affecting final gates + +| ID | Priority / source | Verified evidence | Required disposition | +|---|---|---|---| +| B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | +| B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | + +The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. + +## Cross-references + +- ← [implementation plan](./rustack-workflows-90-impl-plan.md) +- ↔ [runtime spike](../docs/research/spike-lambda-container-relay.md), [semantic oracle](../docs/research/spike-stepfunctions-semantic-oracle.md), [verification gates](./rustack-workflows-70-verification-plan.md) diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md new file mode 100644 index 00000000..3d49184e --- /dev/null +++ b/specs/rustack-workflows-99-decisions.md @@ -0,0 +1,107 @@ +# Workflows 99: Key Decisions and Glossary + +Status: accepted for specification; implementation requires P0 evidence · Date: 2026-10-25 + +Supersede decisions with a new ID rather than rewriting their rationale after implementation. Component documents own detailed behavior; this document owns the reasons. + +## D1 — Own a bounded STANDARD/JSONPath interpreter + +- Context: other projects need actual state transitions and fault behavior, not only API-shaped responses. +- Alternatives: wrap Step Functions Local/LocalStack; embed a general durable-workflow product; implement every ASL dialect immediately. +- Decision: implement typed ASL compilation and an actor-based interpreter, initially STANDARD and the documented JSONPath profile. +- Why: Rustack owns correctness, shared-service integration and lifecycle without external emulator dependencies. Explicit limits avoid accidental JSONata or EXPRESS claims. +- Pinned by: [PRD scope](./rustack-workflows-00-prd.md#4-release-scope), [compiler/profile](./rustack-workflows-30-asl-design.md#1-compile-before-execution). + +## D2 — Keep Docker Runtime API inside each execution environment + +- Alternatives: publish a wildcard host Runtime API; assume host.docker.internal can reach host loopback; forward invokes to RIE; rewrite user-image entrypoints with an injected launcher. +- Decision: use a small relay plus the unchanged function image sharing a network namespace; control traffic uses framed Docker attach. +- Why: host loopback is not container loopback; publishing Runtime API complicates isolation; RIE is unnecessary when Rustack owns the Runtime API. A relay preserves arbitrary valid RIC image entrypoints and works without host-address assumptions for control traffic. +- Cost/evidence: two containers per environment and a versioned relay artifact. Full composition is unproven until P0; this decision does not waive that gate. +- Pinned by: [Lambda architecture/transport](./rustack-workflows-20-lambda-runtime-design.md#1-ownership-and-execution-contract), [P0](./rustack-workflows-90-impl-plan.md#3-p0--evidence-and-risk-retirement). + +## D3 — Pin content at deployment, not on each invocation + +- Alternatives: pull by tag per invocation; freeze only published numeric versions; treat local content IDs as registry digests. +- Decision: latest and published revisions both capture resolved image content and full execution configuration; local content identity is explicitly distinguished from registry manifest identity. +- Why: tag movement and warm reuse must not change code silently; version tests need real immutable artifacts, including ImageConfig and architecture. +- Pinned by: [image resolution](./rustack-workflows-20-lambda-runtime-design.md#2-image-resolution-and-deployment-lifecycle), [snapshot image references](./rustack-workflows-60-operations-design.md#3-shutdown-cancellation-and-snapshots). + +## D4 — Preserve one Lambda admission and execution path + +- Alternatives: call Docker directly from ASL; use an HTTP AWS SDK client against Rustack itself. +- Decision: binary-owned LambdaTaskPort adapter calls the existing provider through a cancellable invocation-handle contract. +- Why: reserved concurrency, versions, backend failures and cleanup must match ordinary Invoke; an internal port avoids protocol overhead and recursive HTTP request-permit deadlock. +- Pinned by: [integration ports](./rustack-workflows-40-integrations-design.md#1-internal-ports-and-dependency-direction). + +## D5 — Compile definitions and keep execution mutation in one owner + +- Alternatives: interpret serde_json maps each step; share mutable execution state through Mutex/RwLock; spawn one task per Map item. +- Decision: immutable indexed IR plus one ExecutionActor with bounded frames, timers and external-call futures. +- Why: validation happens before side effects, generations settle races consistently, and fan-out cannot become an unbounded task tree. +- Pinned by: [ASL compiler/scheduler](./rustack-workflows-30-asl-design.md), [history](./rustack-workflows-10-api-design.md#6-history-and-pagination). + +## D6 — Implement `.sync` only for a real supported job integration + +- Alternatives: accept `.sync` on Lambda/arbitrary tasks; equate child start acceptance with child success; implement ECS-shaped responses without ECS execution. +- Decision: ordinary Lambda invocation awaits its result; `.sync` and `.sync:2` apply to nested Step Functions in this release. +- Why: production AWS ASL compatibility matters more than permissive local syntax. Nested child completion is a real job boundary; string-versus-JSON Output and ownership cancellation are observable contracts. +- Pinned by: [allowlist](./rustack-workflows-40-integrations-design.md#2-integration-allowlist), [nested execution](./rustack-workflows-40-integrations-design.md#4-child-workflows-and-sync). + +## D7 — Expose uncertain side effects rather than replay automatically + +- Alternatives: retry every Docker transport failure; claim exactly-once function execution; release permits when the caller stops waiting. +- Decision: no hidden business-call retries; explicit ASL Retry may repeat work; generation invalidation and supervised cleanup retain uncertain resources. +- Why: a broken connection cannot establish whether user code already changed another service. Resource ownership must outlive response futures. +- Pinned by: [Lambda cleanup](./rustack-workflows-20-lambda-runtime-design.md#4-environment-states-pooling-and-cancellation), [error mapping](./rustack-workflows-40-integrations-design.md#3-lambda-requestresponse). + +## D8 — Preserve snapshot semantics without claiming durable workflow recovery + +- Alternatives: add a new database/WAL immediately; serialize live handles/timers; replay all RUNNING tasks on restart. +- Decision: persist definitions, terminal histories and independent name reservations; quiesce/cancel active work before normal saves; never replay imported RUNNING calls. +- Why: existing snapshots are graceful-save archives. Exactly-once recovery requires a separate durable command/outcome protocol and external-side-effect strategy; serializing structs cannot provide it. +- Pinned by: [snapshot contract](./rustack-workflows-60-operations-design.md#3-shutdown-cancellation-and-snapshots). + +## D9 — Support callbacks with scoped credentials and provisional completion + +- Alternatives: accept only callbacks arriving after Lambda returns; let an early callback hide Lambda failure; persist raw task tokens in diagnostics. +- Decision: register tokens before dispatch, stage early callbacks, commit only after successful invocation, and redact injected tokens before history persistence. +- Why: real handlers can callback immediately; old attempts must not complete new work, and task tokens authorize state transitions. +- Pinned by: [callback protocol](./rustack-workflows-40-integrations-design.md#5-callback-and-heartbeat-m3). + +## D10 — Make local operational limits visible + +- Alternatives: inherit unlimited Docker resources; enable code execution by default; silently discard history or unsupported fields. +- Decision: explicit Docker opt-in, startup probes, bounded quotas, local-only transport and detailed capabilities; retain existing YAML settings. +- Why: honest local restrictions are testable. Docker permissions, tmpfs memory accounting, native-architecture coverage and non-IAM role metadata cannot be mistaken for AWS parity. +- Pinned by: [operations](./rustack-workflows-60-operations-design.md), [security/performance gates](./rustack-workflows-70-verification-plan.md). + +## D11 — Apply P0 protocol evidence before implementing consumers + +- Context: independent review and actual transport/SDK probes exposed gaps in the original draft. +- Decision: require BootstrapReady, default omitted Lambda architecture to x86_64, use callback TaskTimedOut for known closed tokens, generate numeric epoch-second HTTP timestamps, and decode both nested Input/Output for `.sync:2`. Optimized nested dates use epoch milliseconds. Use StdinOnce=true for the environment's single attach lifetime. +- Why: actual EOF tests distinguish detach from process stdin closure; ordinary SDK behavior and pinned AWS-validated snapshots establish observable wire distinctions. Current official errors take precedence over older snapshots. +- Pinned by: [runtime spike](../docs/research/spike-lambda-container-relay.md), [semantic evidence](../docs/research/spike-stepfunctions-semantic-oracle.md), [required review fixes](./rustack-workflows-93-review.md). +- Date:2026-10-25. These amendments refine D2/D6/D9 without changing the complete M0–M4 scope. + +## Glossary + +| Term | Meaning in this specification | +|---|---| +| State machine revision | Immutable ASL/config snapshot captured when an execution starts; not a public Step Functions version/alias | +| Lambda execution revision | Resolved function code plus all execution configuration captured per invocation attempt | +| Execution | One STANDARD state-machine run, not one Lambda invocation | +| Frame | Engine-owned state/branch/iteration context inside an execution; not necessarily a Tokio task | +| State entry | One visit to a state; its retries share raw input and retrier counters | +| Attempt/generation | One dispatch and its validity epoch; stale results/timers cannot complete newer work | +| Execution environment | One function container plus its relay/netns; processes one invocation at a time and may be warm-reused | +| RIC / RIE | Runtime Interface Client inside the function versus Runtime Interface Emulator; Rustack needs RIC compatibility, not RIE forwarding | +| RequestResponse / `.sync` / callback | Await API/function response versus job termination versus token-authorized completion; distinct integration patterns | +| Owned / detached child | A `.sync` child is cancelled with its parent; start-only child remains independently supervised | +| Quarantine | A tracked environment whose cleanup is uncertain and still consumes allocation capacity | +| Snapshot / recovery log | A completed graceful-save archive versus a crash-consistent record of accepted commands/outcomes; only the former exists in this release | + +## Cross-references + +- ← [PRD](./rustack-workflows-00-prd.md) +- ↔ [roadmap](./rustack-workflows-80-roadmap.md), [implementation plan](./rustack-workflows-90-impl-plan.md), [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) diff --git a/specs/ruststack-lambda-design.md b/specs/ruststack-lambda-design.md index bcf7abec..dd8b61eb 100644 --- a/specs/ruststack-lambda-design.md +++ b/specs/ruststack-lambda-design.md @@ -1,5 +1,7 @@ # Rustack Lambda: Invoke-Focused Native Rust Implementation Design +> Current execution target: [Workflows Lambda runtime](./rustack-workflows-20-lambda-runtime-design.md) supersedes this document's RIE-forwarding/Docker execution architecture. Retain unaffected control-plane details and [Lambda hardening](./rustack-lambda-hardening-design.md). This is a design update, not a claim that Docker execution is implemented. + **Date:** 2026-03-06 **Status:** Draft / RFC **Depends on:** [smithy-s3-redesign-design.md](./smithy-s3-redesign-design.md), [rustack-dynamodb-design.md](./rustack-dynamodb-design.md), [rustack-sqs-design.md](./rustack-sqs-design.md) diff --git a/specs/ruststack-lambda-executor-design.md b/specs/ruststack-lambda-executor-design.md index 779efb1f..6e3568d4 100644 --- a/specs/ruststack-lambda-executor-design.md +++ b/specs/ruststack-lambda-executor-design.md @@ -1,5 +1,7 @@ # Rustack Lambda: Real Function Execution (Native + Docker Backends) +> Current execution target: [Workflows Lambda runtime](./rustack-workflows-20-lambda-runtime-design.md) supersedes the historical Docker networking, dependency versions and cleanup plan below. Preserve the applicable Native/Runtime API contracts and [Lambda hardening](./rustack-lambda-hardening-design.md). Docker remains unimplemented until the new phase gates pass. + **Date:** 2026-04-19 **Status:** Draft / Implementation Plan **Depends on:** [ruststack-lambda-design.md](./ruststack-lambda-design.md) diff --git a/tests/workflows/evidence/botocore-wire.json b/tests/workflows/evidence/botocore-wire.json new file mode 100644 index 00000000..fb308d8f --- /dev/null +++ b/tests/workflows/evidence/botocore-wire.json @@ -0,0 +1,22 @@ +{ + "boto3": "1.43.118", + "botocore": "1.43.118", + "protocol": "json", + "target": "AWSStepFunctions.StartExecution", + "preservesRawInput": true, + "timestamps": [ + { + "epoch": 0.125, + "iso": "1970-01-01T00:00:00.125000+00:00" + }, + { + "epoch": -0.125, + "iso": "1969-12-31T23:59:59.875000+00:00" + } + ], + "errorCodes": [ + "TaskTimedOut", + "InvalidToken" + ], + "awsRequests": 0 +} diff --git a/tests/workflows/evidence/relay-amd64.json b/tests/workflows/evidence/relay-amd64.json new file mode 100644 index 00000000..eb93b56e --- /dev/null +++ b/tests/workflows/evidence/relay-amd64.json @@ -0,0 +1,45 @@ +{ + "architecture": "amd64", + "attachEof": [ + { + "runningAfterShutdown": true, + "stdinOnce": false + }, + { + "runningAfterShutdown": false, + "stdinOnce": true + } + ], + "cancelledBusyFunction": true, + "closeAck": true, + "coldMs": 2526, + "errorType": "ValueError", + "labelReconciliation": true, + "noPublishedRuntimePorts": true, + "outputs": [ + { + "computed": 42, + "counter": 1, + "memoryMax": "201326592", + "pidsMax": "256", + "requestId": "426463ee-6a63-4a99-ac42-b67075694b96", + "rootReadOnly": true, + "uid": 1000 + }, + { + "computed": 7, + "counter": 2, + "memoryMax": "201326592", + "pidsMax": "256", + "requestId": "2c470034-bda1-4414-97cf-edeeec4110ce", + "rootReadOnly": true, + "uid": 1000 + } + ], + "runId": "19e31db5-376e-4149-b362-af0210f98a8e", + "qualification": "x86_64 Linux guest under QEMU on an arm64 host; functional evidence, not native x86 performance", + "rustVersion": "1.99.1", + "dockerServerVersion": "29.5.2", + "host": "macOS 27 / Apple M5", + "observedAt": "2026-10-25" +} diff --git a/tests/workflows/evidence/relay-arm64.json b/tests/workflows/evidence/relay-arm64.json new file mode 100644 index 00000000..7dff6dca --- /dev/null +++ b/tests/workflows/evidence/relay-arm64.json @@ -0,0 +1,45 @@ +{ + "architecture": "arm64", + "attachEof": [ + { + "runningAfterShutdown": true, + "stdinOnce": false + }, + { + "runningAfterShutdown": false, + "stdinOnce": true + } + ], + "cancelledBusyFunction": true, + "closeAck": true, + "coldMs": 122, + "errorType": "ValueError", + "labelReconciliation": true, + "noPublishedRuntimePorts": true, + "outputs": [ + { + "computed": 42, + "counter": 1, + "memoryMax": "201326592", + "pidsMax": "256", + "requestId": "26d86ab5-d555-4030-afd6-bf070ef8cb55", + "rootReadOnly": true, + "uid": 1000 + }, + { + "computed": 7, + "counter": 2, + "memoryMax": "201326592", + "pidsMax": "256", + "requestId": "731ec28f-ff9b-40cf-8d4a-07e28478b053", + "rootReadOnly": true, + "uid": 1000 + } + ], + "runId": "b51a309e-613c-475d-b647-07e2257f88b1", + "qualification": "macOS arm64 host, native aarch64 VZ guest; not Docker Desktop/rootless qualification", + "rustVersion": "1.99.1", + "dockerServerVersion": "29.5.2", + "host": "macOS 27 / Apple M5", + "observedAt": "2026-10-25" +} diff --git a/tests/workflows/provenance.json b/tests/workflows/provenance.json new file mode 100644 index 00000000..5da63cd1 --- /dev/null +++ b/tests/workflows/provenance.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": 1, + "retrievedAt": "2026-10-25", + "awsModelCommit": "18e485ce1c8dbf29d47e5ce00d9b8513da95912c", + "awsModelSha256": "38dd64c074f5c05320978970dafa6e0f199ab2e33b6065232002e8a603c5f6ed", + "rustSdkSfn": "1.123.0", + "boto3": "1.43.118", + "botocore": "1.43.118", + "bollard": "0.21.1", + "winnow": "1.0.4", + "cloudDifferentialEvidence": false, + "vendorLocalstackCommit": "45dfee8ce27df00fcac3f91b8071c8156eb5c5c7", + "upstreamAwsValidationDates": { + "nestedSync": "2023-06-30", + "nestedSync2": "2023-08-12" + } +} diff --git a/tests/workflows/requirements.txt b/tests/workflows/requirements.txt new file mode 100644 index 00000000..52c9e9d0 --- /dev/null +++ b/tests/workflows/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.43.118 +botocore==1.43.118 diff --git a/tests/workflows/semantic-cases.json b/tests/workflows/semantic-cases.json new file mode 100644 index 00000000..b7ead24c --- /dev/null +++ b/tests/workflows/semantic-cases.json @@ -0,0 +1,659 @@ +{ + "schemaVersion": 1, + "profile": "rustack-standard-jsonpath-v1", + "evidence": "Official-document-derived expectations; not AWS-observed executions and not an interpreter test result.", + "retrievedAt": "2026-10-25", + "sources": { + "asl": "https://states-language.net/spec.html", + "pipeline": "https://docs.aws.amazon.com/step-functions/latest/dg/input-output-inputpath-params.html", + "paths": "https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-paths.html", + "choice": "https://docs.aws.amazon.com/step-functions/latest/dg/state-choice.html", + "intrinsics": "https://docs.aws.amazon.com/step-functions/latest/dg/intrinsic-functions.html", + "errors": "https://docs.aws.amazon.com/step-functions/latest/dg/concepts-error-handling.html", + "nested": "https://docs.aws.amazon.com/step-functions/latest/dg/connect-stepfunctions.html", + "start": "https://docs.aws.amazon.com/step-functions/latest/apireference/API_StartExecution.html", + "success": "https://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskSuccess.html", + "smithy": "https://smithy.io/2.0/aws/protocols/aws-json-1_0-protocol.html#shape-serialization", + "nestedSnapshots": "vendors/localstack/tests/aws/services/stepfunctions/v2/callback/test_callback.snapshot.json@45dfee8ce27df00fcac3f91b8071c8156eb5c5c7", + "nestedDates": "https://github.com/aws/aws-cdk/issues/5474" + }, + "cases": [ + { + "id": "path-member-index", + "kind": "path", + "source": "paths", + "path": "$.orders[0].total", + "input": { + "orders": [ + { + "total": 23 + } + ] + }, + "expected": 23 + }, + { + "id": "path-quoted-member", + "kind": "path", + "source": "paths", + "path": "$['odd.key']", + "input": { + "odd.key": 17 + }, + "expected": 17 + }, + { + "id": "path-slice", + "kind": "path", + "source": "pipeline", + "path": "$.items[0:2]", + "input": { + "items": [ + 3, + 5, + 8 + ] + }, + "expected": [ + 3, + 5 + ] + }, + { + "id": "path-single-selection", + "kind": "path", + "source": "paths", + "path": "$.items[*].id", + "input": { + "items": [ + { + "id": 7 + } + ] + }, + "expected": [ + 7 + ] + }, + { + "id": "path-null", + "kind": "path", + "source": "asl", + "path": "$.value", + "input": { + "value": null + }, + "expected": null + }, + { + "id": "path-missing", + "kind": "path", + "source": "errors", + "path": "$.missing", + "input": {}, + "expectedError": "States.Runtime" + }, + { + "id": "path-context", + "kind": "path", + "source": "asl", + "path": "$$.Execution.Id", + "input": {}, + "context": { + "Execution": { + "Id": "arn:aws:states:us-east-1:000000000000:execution:P0:one" + } + }, + "expected": "arn:aws:states:us-east-1:000000000000:execution:P0:one" + }, + { + "id": "pipeline-original-input-merge", + "kind": "pipeline", + "source": "pipeline", + "input": { + "keep": 9, + "request": { + "number": 4 + } + }, + "inputPath": "$.request", + "parameters": { + "value.$": "$.number" + }, + "rawResult": { + "status": 200, + "payload": { + "doubled": 8 + } + }, + "resultSelector": { + "value.$": "$.payload.doubled" + }, + "resultPath": "$.result", + "outputPath": "$", + "expectedTaskInput": { + "value": 4 + }, + "expected": { + "keep": 9, + "request": { + "number": 4 + }, + "result": { + "value": 8 + } + } + }, + { + "id": "pipeline-null-input", + "kind": "pipeline", + "source": "pipeline", + "input": { + "unused": 4 + }, + "inputPath": null, + "rawResult": { + "answer": 7 + }, + "resultPath": "$", + "outputPath": "$", + "expectedTaskInput": {}, + "expected": { + "answer": 7 + } + }, + { + "id": "pipeline-discard-result", + "kind": "pipeline", + "source": "asl", + "input": { + "keep": 4 + }, + "inputPath": "$", + "rawResult": { + "discard": 9 + }, + "resultPath": null, + "outputPath": "$", + "expectedTaskInput": { + "keep": 4 + }, + "expected": { + "keep": 4 + } + }, + { + "id": "pipeline-null-output", + "kind": "pipeline", + "source": "asl", + "input": { + "keep": 4 + }, + "inputPath": "$", + "rawResult": { + "discard": 9 + }, + "resultPath": "$", + "outputPath": null, + "expectedTaskInput": { + "keep": 4 + }, + "expected": {} + }, + { + "id": "intrinsic-format", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.Format('job-{}-{}', $.kind, $.id)", + "input": { + "kind": "image", + "id": 7 + }, + "expected": "job-image-7" + }, + { + "id": "intrinsic-string-to-json", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.StringToJson($.text)", + "input": { + "text": "{\"count\":7}" + }, + "expected": { + "count": 7 + } + }, + { + "id": "intrinsic-json-to-string", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.JsonToString($.value)", + "input": { + "value": { + "count": 7 + } + }, + "expected": "{\"count\":7}" + }, + { + "id": "intrinsic-array", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.Array($.first, $.second)", + "input": { + "first": 2, + "second": 5 + }, + "expected": [ + 2, + 5 + ] + }, + { + "id": "intrinsic-array-length", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.ArrayLength($.items)", + "input": { + "items": [ + 3, + 7, + 11 + ] + }, + "expected": 3 + }, + { + "id": "intrinsic-array-get-item", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.ArrayGetItem($.items, 1)", + "input": { + "items": [ + 3, + 7, + 11 + ] + }, + "expected": 7 + }, + { + "id": "intrinsic-array-contains", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.ArrayContains($.items, 7)", + "input": { + "items": [ + 3, + 7, + 11 + ] + }, + "expected": true + }, + { + "id": "intrinsic-array-partition", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.ArrayPartition($.items, 2)", + "input": { + "items": [ + 3, + 7, + 11 + ] + }, + "expected": [ + [ + 3, + 7 + ], + [ + 11 + ] + ] + }, + { + "id": "intrinsic-string-split", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.StringSplit($.text, '+=:')", + "input": { + "text": "red+green=blue:gold" + }, + "expected": [ + "red", + "green", + "blue", + "gold" + ] + }, + { + "id": "intrinsic-math-add", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.MathAdd($.left, $.right)", + "input": { + "left": 2.6, + "right": 3.4 + }, + "expected": 6 + }, + { + "id": "intrinsic-json-merge", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.JsonMerge($.left, $.right, false)", + "input": { + "left": { + "nested": { + "a": 1 + }, + "keep": 7 + }, + "right": { + "nested": { + "b": 2 + } + } + }, + "expected": { + "nested": { + "b": 2 + }, + "keep": 7 + } + }, + { + "id": "intrinsic-uuid", + "kind": "intrinsic", + "source": "intrinsics", + "expression": "States.UUID()", + "input": {}, + "expectedPattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + { + "id": "choice-missing-guard", + "kind": "choice", + "source": "choice", + "input": {}, + "rule": { + "And": [ + { + "Variable": "$.optional", + "IsPresent": true + }, + { + "Variable": "$.optional", + "StringEquals": "yes" + } + ] + }, + "expected": false + }, + { + "id": "choice-wrong-type", + "kind": "choice", + "source": "asl", + "input": { + "value": "7" + }, + "rule": { + "Variable": "$.value", + "NumericEquals": 7 + }, + "expected": false + }, + { + "id": "choice-is-null", + "kind": "choice", + "source": "asl", + "input": { + "value": null + }, + "rule": { + "Variable": "$.value", + "IsNull": true + }, + "expected": true + }, + { + "id": "choice-string-matches", + "kind": "choice", + "source": "choice", + "input": { + "value": "build-123.json" + }, + "rule": { + "Variable": "$.value", + "StringMatches": "build-*.json" + }, + "expected": true + }, + { + "id": "retry-zero", + "kind": "retry", + "source": "errors", + "retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "MaxAttempts": 0 + } + ], + "failures": [ + "ExampleError" + ], + "expectedAttempts": 1, + "expectedDelays": [] + }, + { + "id": "retry-backoff", + "kind": "retry", + "source": "errors", + "retry": [ + { + "ErrorEquals": [ + "ExampleError" + ], + "IntervalSeconds": 2, + "BackoffRate": 3, + "MaxAttempts": 2 + } + ], + "failures": [ + "ExampleError", + "ExampleError", + "ExampleError" + ], + "expectedAttempts": 3, + "expectedDelays": [ + 2, + 6 + ] + }, + { + "id": "retry-capped", + "kind": "retry", + "source": "errors", + "retry": [ + { + "ErrorEquals": [ + "ExampleError" + ], + "IntervalSeconds": 2, + "BackoffRate": 3, + "MaxDelaySeconds": 4, + "MaxAttempts": 2 + } + ], + "failures": [ + "ExampleError", + "ExampleError", + "ExampleError" + ], + "expectedAttempts": 3, + "expectedDelays": [ + 2, + 4 + ] + }, + { + "id": "matcher-timeout", + "kind": "errorMatcher", + "source": "errors", + "error": "States.Timeout", + "matcher": "States.TaskFailed", + "expected": false + }, + { + "id": "matcher-runtime", + "kind": "errorMatcher", + "source": "errors", + "error": "States.Runtime", + "matcher": "States.ALL", + "expected": false + }, + { + "id": "matcher-data-limit-all", + "kind": "errorMatcher", + "source": "errors", + "error": "States.DataLimitExceeded", + "matcher": "States.ALL", + "expected": false + }, + { + "id": "matcher-data-limit-explicit", + "kind": "errorMatcher", + "source": "errors", + "error": "States.DataLimitExceeded", + "matcher": "States.DataLimitExceeded", + "expected": true + }, + { + "id": "nested-sync-string", + "kind": "nestedResult", + "source": "nestedSnapshots", + "resource": "arn:aws:states:::states:startExecution.sync", + "childOutput": { + "computed": 42 + }, + "expectedOutput": "{\"computed\":42}", + "childInput": { + "value": 7 + }, + "expectedInput": "{\"value\":7}" + }, + { + "id": "nested-sync-json", + "kind": "nestedResult", + "source": "nestedSnapshots", + "resource": "arn:aws:states:::states:startExecution.sync:2", + "childOutput": { + "computed": 42 + }, + "expectedOutput": { + "computed": 42 + }, + "childInput": { + "value": 7 + }, + "expectedInput": { + "value": 7 + } + }, + { + "id": "wire-input-string", + "kind": "wire", + "source": "start", + "operation": "StartExecution", + "request": { + "stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:P0", + "name": "one", + "input": "{ \"value\": 7 }" + }, + "expectedTarget": "AWSStepFunctions.StartExecution" + }, + { + "id": "callback-completed", + "kind": "callbackError", + "source": "success", + "tokenState": "completed", + "expectedError": "TaskTimedOut" + }, + { + "id": "callback-cancelled", + "kind": "callbackError", + "source": "success", + "tokenState": "cancelled", + "expectedError": "TaskTimedOut" + }, + { + "id": "callback-expired", + "kind": "callbackError", + "source": "success", + "tokenState": "expired", + "expectedError": "TaskTimedOut" + }, + { + "id": "callback-retried", + "kind": "callbackError", + "source": "success", + "tokenState": "revoked", + "expectedError": "TaskTimedOut" + }, + { + "id": "callback-unknown", + "kind": "callbackError", + "source": "success", + "tokenState": "unknown", + "expectedError": "InvalidToken" + }, + { + "id": "timestamp-fractional", + "kind": "wireTimestamp", + "source": "smithy", + "epochSeconds": 0.125, + "expectedWireType": "number" + }, + { + "id": "timestamp-negative", + "kind": "wireTimestamp", + "source": "smithy", + "epochSeconds": -0.125, + "expectedWireType": "number" + }, + { + "id": "nested-dates-milliseconds", + "kind": "nestedTimestamp", + "source": "nestedDates", + "epochSeconds": 1.125, + "expectedStartDate": 1125, + "evidenceLimitation": "Historic direct AWS result; current live AWS differential not performed." + }, + { + "id": "nested-null-input-default", + "kind": "nestedInput", + "source": "nestedSnapshots", + "input": null, + "expectedApiInput": "{}" + }, + { + "id": "nested-string-input-preserved", + "kind": "nestedInput", + "source": "start", + "input": "{ \"value\": 7 }", + "expectedApiInput": "{ \"value\": 7 }" + }, + { + "id": "nested-object-input", + "kind": "nestedInput", + "source": "nestedSnapshots", + "input": { + "value": 7 + }, + "expectedApiInput": "{\"value\":7}" + } + ] +} diff --git a/tests/workflows/test_botocore_wire.py b/tests/workflows/test_botocore_wire.py new file mode 100644 index 00000000..2c744f75 --- /dev/null +++ b/tests/workflows/test_botocore_wire.py @@ -0,0 +1,65 @@ +"""Offline SDK wire evidence; these tests do not execute an ASL interpreter.""" + +import json +from datetime import datetime +from pathlib import Path +import unittest + +import boto3 +import botocore +import botocore.parsers +import botocore.serialize +import botocore.session + + +class StepFunctionsWireEvidence(unittest.TestCase): + """Check independently implemented SDK serialization and parsing.""" + + @classmethod + def setUpClass(cls): + root = Path(__file__).parent + cls.provenance = json.loads((root / "provenance.json").read_text()) + cls.fixtures = json.loads((root / "semantic-cases.json").read_text()) + cls.service = botocore.session.get_session().get_service_model("stepfunctions") + + def test_should_match_pinned_sdk_versions(self): + self.assertEqual(boto3.__version__, self.provenance["boto3"]) + self.assertEqual(botocore.__version__, self.provenance["botocore"]) + + def test_should_preserve_json_string_input_and_target(self): + fixture = next(c for c in self.fixtures["cases"] if c["id"] == "wire-input-string") + serializer = botocore.serialize.create_serializer(self.service.metadata["protocol"]) + request = serializer.serialize_to_request( + fixture["request"], self.service.operation_model(fixture["operation"]) + ) + self.assertEqual(request["headers"]["X-Amz-Target"], fixture["expectedTarget"]) + self.assertEqual(json.loads(request["body"]), fixture["request"]) + + def test_should_parse_fractional_and_negative_epoch_seconds(self): + parser = botocore.parsers.ResponseParserFactory().create_parser("json") + output = self.service.operation_model("CreateStateMachine").output_shape + for fixture in self.fixtures["cases"]: + if fixture["kind"] != "wireTimestamp": + continue + with self.subTest(case=fixture["id"]): + value = fixture["epochSeconds"] + body = json.dumps({ + "creationDate": value, + "stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:P0", + }).encode() + parsed = parser.parse({"body": body, "headers": {}, "status_code": 200}, output) + self.assertIsInstance(parsed["creationDate"], datetime) + self.assertEqual(parsed["creationDate"].timestamp(), value) + + def test_should_parse_callback_error_codes_without_network(self): + parser = botocore.parsers.ResponseParserFactory().create_parser("json") + output = self.service.operation_model("SendTaskSuccess").output_shape + for code in {"TaskTimedOut", "InvalidToken"}: + with self.subTest(code=code): + body = json.dumps({"__type": code, "message": "fixture"}).encode() + parsed = parser.parse({"body": body, "headers": {}, "status_code": 400}, output) + self.assertEqual(parsed["Error"]["Code"], code) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/workflows-spikes/.dockerignore b/tools/workflows-spikes/.dockerignore new file mode 100644 index 00000000..81ad77ad --- /dev/null +++ b/tools/workflows-spikes/.dockerignore @@ -0,0 +1,2 @@ +** +!handler.py diff --git a/tools/workflows-spikes/Cargo.lock b/tools/workflows-spikes/Cargo.lock new file mode 100644 index 00000000..b4d7d869 --- /dev/null +++ b/tools/workflows-spikes/Cargo.lock @@ -0,0 +1,1076 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bollard" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.53.1-rc.29.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +dependencies = [ + "serde", + "serde_json", + "serde_repr", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustack-workflows-spikes" +version = "0.1.0" +dependencies = [ + "anyhow", + "bollard", + "bytes", + "ciborium", + "futures", + "http-body-util", + "hyper", + "hyper-util", + "serde", + "serde_json", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/workflows-spikes/Cargo.toml b/tools/workflows-spikes/Cargo.toml new file mode 100644 index 00000000..68dbfb54 --- /dev/null +++ b/tools/workflows-spikes/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "rustack-workflows-spikes" +version = "0.1.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +anyhow = "1.0" +thiserror = "2.0" +bollard = { version = "=0.21.1", default-features = false, features = ["pipe"] } +bytes = { version = "1.12", features = ["serde"] } +ciborium = "0.2" +futures = "0.3" +hyper = { version = "1.11", features = ["server", "http1"] } +hyper-util = { version = "0.1", features = ["tokio"] } +http-body-util = "0.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "net", "sync", "time", "io-util", "io-std"] } +uuid = { version = "1.26", features = ["v4"] } + +[lints.rust] +rust_2024_compatibility = "warn" +missing_docs = "warn" +missing_debug_implementations = "warn" diff --git a/tools/workflows-spikes/Handler.Dockerfile b/tools/workflows-spikes/Handler.Dockerfile new file mode 100644 index 00000000..95d432e4 --- /dev/null +++ b/tools/workflows-spikes/Handler.Dockerfile @@ -0,0 +1,4 @@ +FROM public.ecr.aws/lambda/python:3.13@sha256:e0237850eecd760896365e768bac1526ea01098e325fa590e5bc735b26d5dbb9 +COPY handler.py /var/task/handler.py +USER 1000:1000 +CMD ["handler.handler"] diff --git a/tools/workflows-spikes/Relay.Dockerfile b/tools/workflows-spikes/Relay.Dockerfile new file mode 100644 index 00000000..cb9baccb --- /dev/null +++ b/tools/workflows-spikes/Relay.Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/lambda/provided:al2023@sha256:612c98f126c2a95e77a016d56fed50a83055e11d636132ee6ebdd461f01b9617 +COPY bootstrap /relay +USER 1000:1000 +ENTRYPOINT ["/relay"] +CMD [] diff --git a/tools/workflows-spikes/handler.py b/tools/workflows-spikes/handler.py new file mode 100644 index 00000000..c44fa764 --- /dev/null +++ b/tools/workflows-spikes/handler.py @@ -0,0 +1,27 @@ +"""Real official-RIC handler used by the isolated P0 transport experiment.""" + +import os +from pathlib import Path +import time + +counter = 0 + + +def handler(event, context): + global counter + counter += 1 + if event.get("fail"): + raise ValueError("intentional P0 failure") + if event.get("sleep"): + print("P0_BUSY_START=" + context.aws_request_id, flush=True) + time.sleep(60) + cgroup = Path("/sys/fs/cgroup") + return { + "computed": event["value"] + 7, + "counter": counter, + "uid": os.getuid(), + "requestId": context.aws_request_id, + "memoryMax": (cgroup / "memory.max").read_text().strip(), + "pidsMax": (cgroup / "pids.max").read_text().strip(), + "rootReadOnly": bool(os.statvfs("/").f_flag & os.ST_RDONLY), + } diff --git a/tools/workflows-spikes/src/bin/driver.rs b/tools/workflows-spikes/src/bin/driver.rs new file mode 100644 index 00000000..34304c0c --- /dev/null +++ b/tools/workflows-spikes/src/bin/driver.rs @@ -0,0 +1,524 @@ +//! Docker attach + official RIC feasibility driver. Never contacts AWS APIs. +#![forbid(unsafe_code)] + +use std::{ + collections::{BTreeSet, HashMap}, + env, + time::Duration, +}; + +use anyhow::{Context, Result, bail, ensure}; +use bollard::{ + API_DEFAULT_VERSION, Docker, + container::{AttachContainerResults, LogOutput}, + errors::Error as DockerError, + models::{ContainerCreateBody, HostConfig}, + query_parameters::{ + AttachContainerOptionsBuilder, ListContainersOptionsBuilder, LogsOptionsBuilder, + RemoveContainerOptionsBuilder, + }, +}; +use bytes::{Bytes, BytesMut}; +use futures::StreamExt; +use rustack_workflows_spikes::{Frame, MAX_FRAME, decode, write_frame}; +use serde_json::{Value, json}; +use tokio::{ + io::AsyncWriteExt, + time::{Instant, sleep, timeout}, +}; +use uuid::Uuid; + +async fn next_frame(attach: &mut AttachContainerResults, buffer: &mut BytesMut) -> Result { + timeout(Duration::from_secs(45), async { + loop { + if buffer.len() >= 4 { + let prefix: [u8; 4] = buffer.get(..4).context("length prefix")?.try_into()?; + let n = usize::try_from(u32::from_be_bytes(prefix))?; + ensure!(n <= MAX_FRAME, "frame exceeded cap"); + if buffer.len() >= n + 4 { + let complete = buffer.split_to(n + 4); + return Ok(decode(complete.get(4..).context("body slice")?)?); + } + } + match attach.output.next().await { + Some(Ok(LogOutput::StdOut { message })) => { + ensure!( + buffer + .len() + .checked_add(message.len()) + .context("buffer overflow")? + <= MAX_FRAME + 4096, + "buffer exceeded cap" + ); + buffer.extend_from_slice(&message); + } + Some(Ok(LogOutput::StdErr { message })) => { + bail!("relay diagnostic: {}", String::from_utf8_lossy(&message)) + } + Some(Ok(_)) => bail!("unexpected Docker stream"), + Some(Err(error)) => return Err(error.into()), + None => bail!("attach stream ended"), + } + } + }) + .await + .context("relay frame deadline")? +} + +fn limits(memory: i64, cpus: i64) -> HostConfig { + HostConfig { + memory: Some(memory), + nano_cpus: Some(cpus), + pids_limit: Some(64), + readonly_rootfs: Some(true), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + tmpfs: Some(HashMap::from([( + "/tmp".into(), + "rw,noexec,nosuid,size=536870912".into(), + )])), + ..Default::default() + } +} + +#[derive(Debug)] +struct Environment { + identity: String, + relay: String, + function: String, + attach: AttachContainerResults, + buffer: BytesMut, +} + +fn owner_labels(identity: &str, run_id: &str) -> HashMap { + HashMap::from([ + ("rustack.p0.environment".into(), identity.into()), + ("rustack.p0.run".into(), run_id.into()), + ]) +} + +async fn create_relay( + docker: &Docker, + architecture: &str, + identity: &str, + run_id: &str, + stdin_once: bool, +) -> Result { + Ok(docker + .create_container( + None, + ContainerCreateBody { + image: Some(format!("rustack-p0-relay:{architecture}")), + user: Some("1000:1000".into()), + env: Some(vec![format!("SPIKE_ENVIRONMENT_UUID={identity}")]), + labels: Some(owner_labels(identity, run_id)), + attach_stdin: Some(true), + attach_stdout: Some(true), + attach_stderr: Some(true), + open_stdin: Some(true), + stdin_once: Some(stdin_once), + tty: Some(false), + host_config: Some(limits(64 * 1024 * 1024, 100_000_000)), + ..Default::default() + }, + ) + .await? + .id) +} + +async fn create_function( + docker: &Docker, + architecture: &str, + identity: &str, + relay: &str, + run_id: &str, +) -> Result { + let mut host = limits(192 * 1024 * 1024, 1_000_000_000); + host.network_mode = Some(format!("container:{relay}")); + host.pids_limit = Some(256); + Ok(docker + .create_container( + None, + ContainerCreateBody { + image: Some(format!("rustack-p0-handler:{architecture}")), + user: Some("1000:1000".into()), + labels: Some(owner_labels(identity, run_id)), + env: Some(vec![ + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001".into(), + "AWS_LAMBDA_FUNCTION_NAME=spike".into(), + "AWS_LAMBDA_FUNCTION_VERSION=$LATEST".into(), + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192".into(), + "AWS_REGION=us-east-1".into(), + "AWS_DEFAULT_REGION=us-east-1".into(), + ]), + host_config: Some(host), + ..Default::default() + }, + ) + .await? + .id) +} + +async fn expect_kind( + attach: &mut AttachContainerResults, + buffer: &mut BytesMut, + kind: &str, +) -> Result { + let frame = next_frame(attach, buffer) + .await + .with_context(|| format!("await {kind}"))?; + ensure!( + frame.kind == kind, + "unexpected control variant awaiting {kind}" + ); + Ok(frame) +} + +async fn create_environment( + docker: &Docker, + ids: &mut Vec, + architecture: &str, + run_id: &str, +) -> Result { + let identity = Uuid::new_v4().to_string(); + let relay = create_relay(docker, architecture, &identity, run_id, true).await?; + ids.push(relay.clone()); + let mut attach = docker + .attach_container( + &relay, + Some( + AttachContainerOptionsBuilder::default() + .stdin(true) + .stdout(true) + .stderr(true) + .stream(true) + .logs(false) + .build(), + ), + ) + .await?; + docker.start_container(&relay, None).await?; + let mut buffer = BytesMut::new(); + expect_kind(&mut attach, &mut buffer, "hello").await?; + expect_kind(&mut attach, &mut buffer, "ready").await?; + let function = create_function(docker, architecture, &identity, &relay, run_id).await?; + ids.push(function.clone()); + docker.start_container(&function, None).await?; + let ready = expect_kind(&mut attach, &mut buffer, "bootstrapReady").await?; + ensure!(ready.environment == identity, "foreign bootstrap readiness"); + Ok(Environment { + identity, + relay, + function, + attach, + buffer, + }) +} + +async fn invoke(environment: &mut Environment, payload: Bytes, kind: &str) -> Result { + let request = Uuid::new_v4().to_string(); + write_frame( + &mut environment.attach.input, + &Frame::new(&environment.identity, "invoke", &request, payload), + ) + .await?; + let result = expect_kind(&mut environment.attach, &mut environment.buffer, kind).await?; + ensure!( + result.request_id == request && result.environment == environment.identity, + "foreign invocation result" + ); + Ok(result) +} + +fn check_output(body: &Value, value: i32, count: i32) -> Result<()> { + for (field, expected) in [ + ("computed", json!(value + 7)), + ("counter", json!(count)), + ("uid", json!(1000)), + ("memoryMax", json!("201326592")), + ("pidsMax", json!("256")), + ("rootReadOnly", json!(true)), + ] { + ensure!( + body.get(field) == Some(&expected), + "unexpected fixture output: {field}" + ); + } + Ok(()) +} + +async fn wait_handler_started(docker: &Docker, function: &str, request: &str) -> Result<()> { + let marker = format!("P0_BUSY_START={request}"); + let mut stream = docker.logs( + function, + Some( + LogsOptionsBuilder::default() + .stdout(true) + .stderr(true) + .follow(true) + .tail("20") + .build(), + ), + ); + timeout(Duration::from_secs(45), async { + let mut bytes = Vec::new(); + while let Some(message) = stream.next().await { + match message? { + LogOutput::StdOut { message } | LogOutput::StdErr { message } => { + ensure!( + bytes + .len() + .checked_add(message.len()) + .context("log size overflow")? + <= 16 * 1024, + "fixture logs exceeded cap" + ); + bytes.extend_from_slice(&message); + if bytes + .windows(marker.len()) + .any(|chunk| chunk == marker.as_bytes()) + { + return Ok(()); + } + } + _ => bail!("unexpected function log stream"), + } + } + bail!("handler exited before busy marker") + }) + .await + .context("handler start deadline")? +} + +async fn exercise( + docker: &Docker, + ids: &mut Vec, + architecture: &str, + run_id: &str, +) -> Result { + let start = Instant::now(); + let mut environment = create_environment(docker, ids, architecture, run_id).await?; + let cold_ms = start.elapsed().as_millis(); + let mut outputs = Vec::new(); + for (value, count) in [(35, 1), (0, 2)] { + let result = invoke( + &mut environment, + Bytes::from(serde_json::to_vec(&json!({"value":value}))?), + "result", + ) + .await?; + let body: Value = serde_json::from_slice(&result.payload)?; + check_output(&body, value, count)?; + outputs.push(body); + } + let error = invoke( + &mut environment, + Bytes::from_static(br#"{"fail":true}"#), + "error", + ) + .await?; + let failure: Value = serde_json::from_slice(&error.payload)?; + ensure!( + failure.get("errorType") == Some(&json!("ValueError")), + "RIC error type lost" + ); + let inspected = docker.inspect_container(&environment.relay, None).await?; + if let Some(ports) = inspected + .network_settings + .and_then(|settings| settings.ports) + { + ensure!( + ports.values().all(Option::is_none), + "Runtime API ports published" + ); + } + let request = Uuid::new_v4().to_string(); + write_frame( + &mut environment.attach.input, + &Frame::new( + &environment.identity, + "invoke", + &request, + Bytes::from_static(br#"{"sleep":true}"#), + ), + ) + .await?; + wait_handler_started(docker, &environment.function, &request).await?; + docker + .remove_container( + &environment.function, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await?; + write_frame( + &mut environment.attach.input, + &Frame::new(&environment.identity, "close", "", Bytes::new()), + ) + .await?; + expect_kind(&mut environment.attach, &mut environment.buffer, "ack").await?; + Ok( + json!({"architecture":architecture,"coldMs":cold_ms,"outputs":outputs,"errorType":failure.get("errorType"),"noPublishedRuntimePorts":true,"closeAck":true,"cancelledBusyFunction":true}), + ) +} + +async fn relay_is_running(docker: &Docker, relay: &str) -> Result { + docker + .inspect_container(relay, None) + .await? + .state + .and_then(|state| state.running) + .context("missing container state") +} + +async fn probe_attach_eof( + docker: &Docker, + architecture: &str, + run_id: &str, + ids: &mut Vec, +) -> Result { + let mut observations = Vec::new(); + for stdin_once in [false, true] { + let identity = Uuid::new_v4().to_string(); + let relay = create_relay(docker, architecture, &identity, run_id, stdin_once).await?; + ids.push(relay.clone()); + let mut attach = docker + .attach_container( + &relay, + Some( + AttachContainerOptionsBuilder::default() + .stdin(true) + .stdout(true) + .stderr(true) + .stream(true) + .build(), + ), + ) + .await?; + docker.start_container(&relay, None).await?; + let mut buffer = BytesMut::new(); + expect_kind(&mut attach, &mut buffer, "hello").await?; + expect_kind(&mut attach, &mut buffer, "ready").await?; + attach.input.shutdown().await?; + if stdin_once { + timeout(Duration::from_secs(20), async { + while relay_is_running(docker, &relay).await? { + sleep(Duration::from_millis(25)).await; + } + Ok::<(), anyhow::Error>(()) + }) + .await + .context("relay did not exit after stdin EOF")??; + } else { + sleep(Duration::from_millis(500)).await; + } + observations.push(json!({"stdinOnce":stdin_once,"runningAfterShutdown":relay_is_running(docker, &relay).await?})); + } + Ok(json!(observations)) +} + +async fn owned_containers(docker: &Docker, run_id: &str) -> Result> { + let filter: HashMap> = + HashMap::from([("label".into(), vec![format!("rustack.p0.run={run_id}")])]); + let listed = docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filter) + .build(), + )) + .await?; + ensure!(listed.len() <= 8, "unexpected probe resource count"); + listed + .into_iter() + .map(|record| record.id.context("owned container lacks ID")) + .collect() +} + +async fn cleanup_owned(docker: &Docker, run_id: &str, ids: &[String]) -> Result<()> { + timeout(Duration::from_secs(20), async { + let mut candidates: BTreeSet = ids.iter().cloned().collect(); + let mut errors = Vec::new(); + for _ in 0..3 { + candidates.extend(owned_containers(docker, run_id).await?); + for id in &candidates { + match docker + .remove_container( + id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) + | Err(DockerError::DockerResponseServerError { + status_code: 404, .. + }) => {} + Err(error) => errors.push(error), + } + } + sleep(Duration::from_millis(200)).await; + } + ensure!( + owned_containers(docker, run_id).await?.is_empty(), + "probe-owned containers remain: run {run_id}" + ); + ensure!( + errors.is_empty(), + "cleanup had uncertain failures for run {run_id}" + ); + Ok::<(), anyhow::Error>(()) + }) + .await + .context("owned cleanup deadline")? +} + +#[tokio::main] +async fn main() -> Result<()> { + let socket = env::args().nth(1).context("socket argument required")?; + let architecture = env::args().nth(2).context("architecture required")?; + ensure!( + matches!(architecture.as_str(), "arm64" | "amd64"), + "invalid architecture" + ); + let docker = Docker::connect_with_unix(&socket, 60, API_DEFAULT_VERSION)? + .negotiate_version() + .await?; + let run_id = Uuid::new_v4().to_string(); + let mut ids = Vec::new(); + let outcome = timeout(Duration::from_secs(180), async { + let mut evidence = exercise(&docker, &mut ids, &architecture, &run_id).await?; + let eof = probe_attach_eof(&docker, &architecture, &run_id, &mut ids).await?; + evidence + .as_object_mut() + .context("evidence object")? + .insert("attachEof".into(), eof); + // Deliberately omit this ID from the registry to simulate losing a create response. + let _unrecorded = create_relay( + &docker, + &architecture, + &Uuid::new_v4().to_string(), + &run_id, + true, + ) + .await?; + evidence + .as_object_mut() + .context("evidence object")? + .insert("runId".into(), json!(run_id)); + Ok::(evidence) + }) + .await + .context("spike deadline") + .and_then(|value| value); + cleanup_owned(&docker, &run_id, &ids).await?; + match outcome { + Ok(mut evidence) => { + evidence + .as_object_mut() + .context("evidence object")? + .insert("labelReconciliation".into(), json!(true)); + println!("{}", serde_json::to_string_pretty(&evidence)?); + Ok(()) + } + Err(error) => Err(error), + } +} diff --git a/tools/workflows-spikes/src/bin/relay.rs b/tools/workflows-spikes/src/bin/relay.rs new file mode 100644 index 00000000..91a46d10 --- /dev/null +++ b/tools/workflows-spikes/src/bin/relay.rs @@ -0,0 +1,203 @@ +//! Temporary experiment relay: real HTTP Runtime API with framed CBOR stdio. +#![forbid(unsafe_code)] + +use std::{ + convert::Infallible, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::{ + Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, +}; +use hyper_util::rt::TokioIo; +use rustack_workflows_spikes::{Frame, read_frame, write_frame}; +use tokio::{ + net::TcpListener, + sync::{mpsc, oneshot, watch}, + task::JoinSet, + time::timeout, +}; + +#[derive(Debug)] +struct ApiRequest { + method: String, + path: String, + body: Bytes, + reply: oneshot::Sender>>, +} + +fn response(status: StatusCode, body: Bytes) -> Response> { + let mut response = Response::new(Full::new(body)); + *response.status_mut() = status; + response +} + +async fn serve( + request: Request, + tx: mpsc::Sender, +) -> Result>, Infallible> { + let method = request.method().to_string(); + let path = request.uri().path().to_owned(); + let result = timeout( + Duration::from_secs(5), + Limited::new(request.into_body(), 6 * 1024 * 1024).collect(), + ) + .await; + let body = match result { + Ok(Ok(body)) => body.to_bytes(), + _ => return Ok(response(StatusCode::BAD_REQUEST, Bytes::new())), + }; + let (reply, rx) = oneshot::channel(); + if tx + .send(ApiRequest { + method, + path, + body, + reply, + }) + .await + .is_err() + { + return Ok(response(StatusCode::SERVICE_UNAVAILABLE, Bytes::new())); + } + Ok(match rx.await { + Ok(value) => value, + Err(_) => response(StatusCode::SERVICE_UNAVAILABLE, Bytes::new()), + }) +} + +async fn http_server( + listener: TcpListener, + tx: mpsc::Sender, + mut stop: watch::Receiver, +) -> Result<()> { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + _ = stop.changed() => break, + connection = listener.accept() => { + let (socket, _) = connection?; + ensure!(connections.len() < 8, "connection cap"); + let tx = tx.clone(); + connections.spawn(async move { + http1::Builder::new().serve_connection(TokioIo::new(socket), service_fn(move |request| serve(request, tx.clone()))).await + }); + } + completed = connections.join_next(), if !connections.is_empty() => { + if let Some(completed) = completed { completed??; } + } + } + } + connections.abort_all(); + while connections.join_next().await.is_some() {} + Ok(()) +} + +async fn run() -> Result<()> { + let environment = std::env::var("SPIKE_ENVIRONMENT_UUID").context("environment identity")?; + ensure!(environment.len() == 36, "invalid environment identity"); + let listener = TcpListener::bind(("127.0.0.1", 9001)).await?; + let (api_tx, mut api_rx) = mpsc::channel(8); + let (control_tx, mut control_rx) = mpsc::channel(8); + let (stop_tx, stop_rx) = watch::channel(false); + let mut workers = JoinSet::new(); + workers.spawn(http_server(listener, api_tx, stop_rx)); + workers.spawn(async move { + let mut input = tokio::io::stdin(); + loop { + let frame = read_frame(&mut input).await?; + let close = frame.kind == "close"; + control_tx.send(frame).await.context("control receiver")?; + if close { + return std::future::pending::>().await; + } + } + }); + let mut output = tokio::io::stdout(); + write_frame( + &mut output, + &Frame::new(&environment, "hello", "", Bytes::new()), + ) + .await?; + write_frame( + &mut output, + &Frame::new(&environment, "ready", "", Bytes::new()), + ) + .await?; + let mut poll: Option>>> = None; + let mut job: Option = None; + let mut delivered = false; + let mut bootstrap_seen = false; + let outcome: Result<()> = async { + loop { + tokio::select! { + frame = control_rx.recv() => { + let Some(frame) = frame else { bail!("control EOF") }; + ensure!(frame.environment == environment, "foreign environment"); + match frame.kind.as_str() { + "invoke" => { ensure!(job.is_none(), "concurrent invocation"); job = Some(frame); delivered = false; } + "close" => { write_frame(&mut output, &Frame::new(&environment, "ack", "", Bytes::new())).await?; break; } + _ => bail!("unexpected control variant"), + } + } + request = api_rx.recv() => { + let Some(request) = request else { bail!("API channel closed") }; + if request.method == "GET" && request.path == "/2018-06-01/runtime/invocation/next" { + if poll.is_some() || delivered { + let _ = request.reply.send(response(StatusCode::CONFLICT, Bytes::new())); + } else { + poll = Some(request.reply); + if !bootstrap_seen { + write_frame(&mut output, &Frame::new(&environment, "bootstrapReady", "", Bytes::new())).await?; + bootstrap_seen = true; + } + } + } else if request.method == "POST" && request.path == "/2018-06-01/runtime/init/error" { + let _ = request.reply.send(response(StatusCode::ACCEPTED, Bytes::new())); + write_frame(&mut output, &Frame::new(&environment, "initError", "", request.body)).await?; + } else if let Some(active) = &job { + let success = format!("/2018-06-01/runtime/invocation/{}/response", active.request_id); + let failure = format!("/2018-06-01/runtime/invocation/{}/error", active.request_id); + if request.method == "POST" && delivered && (request.path == success || request.path == failure) { + let kind = if request.path == success { "result" } else { "error" }; + let _ = request.reply.send(response(StatusCode::ACCEPTED, Bytes::new())); + write_frame(&mut output, &Frame::new(&environment, kind, &active.request_id, request.body)).await?; + job = None; delivered = false; + } else { let _ = request.reply.send(response(StatusCode::BAD_REQUEST, Bytes::new())); } + } else { let _ = request.reply.send(response(StatusCode::BAD_REQUEST, Bytes::new())); } + } + worker = workers.join_next(), if !workers.is_empty() => { + if let Some(worker) = worker { worker??; } + bail!("relay worker terminated"); + } + () = tokio::time::sleep(Duration::from_secs(120)) => bail!("idle lease expired"), + } + if !delivered && let Some(active) = &job && let Some(reply) = poll.take() { + let deadline = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis().checked_add(10_000).context("deadline overflow")?; + let mut event = response(StatusCode::OK, active.payload.clone()); + for (key, value) in [ + ("lambda-runtime-aws-request-id", active.request_id.clone()), + ("lambda-runtime-deadline-ms", deadline.to_string()), + ("lambda-runtime-invoked-function-arn", "arn:aws:lambda:us-east-1:000000000000:function:spike".into()), + ("lambda-runtime-trace-id", "Root=1-00000000-000000000000000000000000;Sampled=0".into()), + ("content-type", "application/json".into()), + ] { event.headers_mut().insert(key, value.parse()?); } + reply.send(event).map_err(|_| anyhow::anyhow!("RIC disconnected before delivery"))?; + delivered = true; + } + } + Ok(()) + }.await; + let _ = stop_tx.send(true); + workers.abort_all(); + while workers.join_next().await.is_some() {} + outcome +} + +#[tokio::main] +async fn main() -> Result<()> { + run().await +} diff --git a/tools/workflows-spikes/src/lib.rs b/tools/workflows-spikes/src/lib.rs new file mode 100644 index 00000000..266149cc --- /dev/null +++ b/tools/workflows-spikes/src/lib.rs @@ -0,0 +1,277 @@ +//! Bounded control framing for the isolated Docker feasibility experiment. +//! +//! This tool verifies transport feasibility with trusted fixture peers. It is not +//! the production Runtime API or a hardened external CBOR deserializer. +#![forbid(unsafe_code)] + +use std::{ + fmt, + io::{self, Cursor}, + num::TryFromIntError, +}; + +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +/// Maximum frame in this feasibility experiment. +pub const MAX_FRAME: usize = 6 * 1024 * 1024 + 64 * 1024; + +/// Framing failures without payload-bearing diagnostics. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum FrameError { + /// Encoding the envelope failed. + #[error("control envelope encoding failed")] + Encode(#[from] ciborium::ser::Error), + /// Decoding the envelope failed. + #[error("control envelope decoding failed")] + Decode(#[from] ciborium::de::Error), + /// Reading or writing the transport failed. + #[error("control transport IO failed")] + Io(#[from] io::Error), + /// A length does not fit the target integer type. + #[error("control length conversion failed")] + Length(#[from] TryFromIntError), + /// A frame exceeds the allocation budget. + #[error("oversized control frame")] + TooLarge, + /// More than one CBOR item was present. + #[error("trailing control data")] + TrailingData, + /// The control version is unsupported. + #[error("unsupported control protocol")] + UnsupportedVersion, +} + +/// Fixed message schema for experiment-owned peers only. +#[derive(Serialize, Deserialize)] +#[non_exhaustive] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Frame { + /// Protocol version. + pub version: u32, + /// Environment identity. + pub environment: String, + /// Message variant. + pub kind: String, + /// Invocation identity, empty outside invocation messages. + pub request_id: String, + /// Raw event/result bytes, omitted from Debug. + pub payload: Bytes, +} + +impl Frame { + /// Build a control message from trusted fixture metadata. + /// + /// ``` + /// use bytes::Bytes; + /// use rustack_workflows_spikes::Frame; + /// let frame = Frame::new("fixture", "ready", "", Bytes::new()); + /// assert_eq!(frame.version, 1); + /// ``` + #[must_use] + pub fn new(environment: &str, kind: &str, request_id: &str, payload: Bytes) -> Self { + Self { + version: 1, + environment: environment.into(), + kind: kind.into(), + request_id: request_id.into(), + payload, + } + } +} + +/// Encode a complete length-prefixed frame. +/// +/// # Errors +/// Returns an encoding or length error, or rejects an oversized frame. +/// +/// ``` +/// use bytes::Bytes; +/// use rustack_workflows_spikes::{Frame, encode}; +/// # fn main() -> anyhow::Result<()> { +/// let bytes = encode(&Frame::new("fixture", "ready", "", Bytes::new()))?; +/// assert!(bytes.len() > 4); +/// # Ok(()) } +/// ``` +pub fn encode(frame: &Frame) -> Result, FrameError> { + let mut body = Vec::new(); + ciborium::into_writer(frame, &mut body)?; + if body.len() > MAX_FRAME { + return Err(FrameError::TooLarge); + } + let length = u32::try_from(body.len())?.to_be_bytes(); + let capacity = body.len().checked_add(4).ok_or(FrameError::TooLarge)?; + let mut out = Vec::with_capacity(capacity); + out.extend_from_slice(&length); + out.extend_from_slice(&body); + Ok(out) +} + +/// Decode one already bounded message body from a trusted fixture peer. +/// +/// # Errors +/// Rejects oversized, malformed, trailing or unsupported-version data. +/// +/// ``` +/// use rustack_workflows_spikes::decode; +/// assert!(decode(&[]).is_err()); +/// ``` +pub fn decode(body: &[u8]) -> Result { + if body.len() > MAX_FRAME { + return Err(FrameError::TooLarge); + } + let mut cursor = Cursor::new(body); + let frame: Frame = ciborium::from_reader(&mut cursor)?; + if usize::try_from(cursor.position())? != body.len() { + return Err(FrameError::TrailingData); + } + if frame.version != 1 { + return Err(FrameError::UnsupportedVersion); + } + Ok(frame) +} + +/// Read a bounded frame from a dedicated reader task. +/// +/// # Errors +/// Propagates transport, decoding and length failures. +/// +/// ```no_run +/// # async fn example() -> anyhow::Result<()> { +/// let mut stdin = tokio::io::stdin(); +/// let _frame = rustack_workflows_spikes::read_frame(&mut stdin).await?; +/// # Ok(()) } +/// ``` +pub async fn read_frame(reader: &mut R) -> Result { + let length = usize::try_from(reader.read_u32().await?)?; + if length > MAX_FRAME { + return Err(FrameError::TooLarge); + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).await?; + decode(&body) +} + +/// Write and flush a complete frame. +/// +/// # Errors +/// Propagates encoding and transport failures. +/// +/// ```no_run +/// # async fn example(frame: rustack_workflows_spikes::Frame) -> anyhow::Result<()> { +/// let mut stdout = tokio::io::stdout(); +/// rustack_workflows_spikes::write_frame(&mut stdout, &frame).await?; +/// # Ok(()) } +/// ``` +pub async fn write_frame( + writer: &mut W, + frame: &Frame, +) -> Result<(), FrameError> { + writer.write_all(&encode(frame)?).await?; + writer.flush().await?; + Ok(()) +} + +impl fmt::Debug for Frame { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Frame") + .field("version", &self.version) + .field("environment", &self.environment) + .field("kind", &self.kind) + .field("request_id", &self.request_id) + .field("payload_bytes", &self.payload.len()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::io::ErrorKind; + + use anyhow::Result; + use bytes::Bytes; + use tokio::io::AsyncWriteExt; + + use super::{Frame, FrameError, MAX_FRAME, decode, encode, read_frame}; + + #[tokio::test] + async fn test_should_round_trip_fragmented_binary_frame() -> Result<()> { + let payload = Bytes::from((0_u8..=255).collect::>()); + let frame = Frame::new( + "test-environment", + "invoke", + "test-attempt", + payload.clone(), + ); + let encoded = encode(&frame)?; + let (mut writer, mut reader) = tokio::io::duplex(16); + let task = tokio::spawn(async move { + for chunk in encoded.chunks(3) { + writer.write_all(chunk).await?; + } + Ok::<(), std::io::Error>(()) + }); + let decoded = read_frame(&mut reader).await?; + task.await??; + assert_eq!(decoded.payload, payload); + Ok(()) + } + + #[test] + fn test_should_reject_trailing_cbor_data() -> Result<()> { + let mut body = Vec::new(); + ciborium::into_writer(&Frame::new("env", "ready", "", Bytes::new()), &mut body)?; + body.push(0); + assert!(decode(&body).is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_oversized_prefix_before_body_read() -> Result<()> { + let prefix = u32::try_from(MAX_FRAME + 1)?.to_be_bytes(); + let mut reader = prefix.as_slice(); + assert!(read_frame(&mut reader).await.is_err()); + Ok(()) + } + + #[test] + fn test_should_redact_payload_in_debug() { + let debug = format!( + "{:?}", + Frame::new( + "env", + "invoke", + "attempt", + Bytes::from_static(b"secret-marker") + ) + ); + assert!(!debug.contains("secret-marker")); + assert!(debug.contains("payload_bytes")); + } + #[tokio::test] + async fn test_should_detect_clean_eof() { + let mut input = b"".as_slice(); + assert!( + matches!(read_frame(&mut input).await, Err(FrameError::Io(error)) if error.kind() == ErrorKind::UnexpectedEof) + ); + } + + #[tokio::test] + async fn test_should_detect_truncated_prefix_eof() { + let mut input = b"\0\0\0".as_slice(); + assert!( + matches!(read_frame(&mut input).await, Err(FrameError::Io(error)) if error.kind() == ErrorKind::UnexpectedEof) + ); + } + + #[tokio::test] + async fn test_should_detect_truncated_body_eof() { + let mut input = b"\0\0\0\x02\xa0".as_slice(); + assert!( + matches!(read_frame(&mut input).await, Err(FrameError::Io(error)) if error.kind() == ErrorKind::UnexpectedEof) + ); + } +} From 79152811dd93f7bdb573a453ada42674b46e6d62 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 15:56:56 -0700 Subject: [PATCH 002/106] phase 1: capture stable Lambda resource and invocation identity Land the first P1 task1.1 foundation toward M0: stable nonnil resource IDs with snapshot preservation, immutable identity updates, duplicate-restore rejection, captured revision/ImageConfig/ephemeral storage, invoked qualifier metadata, warm-key isolation and redacted invocation diagnostics. Artifact enums, cooperative cancellation and Docker remain pending. Workspace build/test/fmt/Clippy/rustdoc pass: 1703 tests, 154 Lambda core tests and two new doctests. The native target validates six native/DryRun cases; its separately gated Squib case is not execution evidence. Independent read-only review found no in-slice defect. Existing advisory and strict-boundary-lint failures remain explicitly recorded as B01/B03, without ignores. --- Cargo.lock | 1 + crates/rustack-lambda-core/Cargo.toml | 1 + .../rustack-lambda-core/src/executor/auto.rs | 4 + .../src/executor/instance.rs | 62 ++++++-- .../rustack-lambda-core/src/executor/squib.rs | 8 ++ .../rustack-lambda-core/src/executor/types.rs | 100 ++++++++++++- crates/rustack-lambda-core/src/identity.rs | 133 ++++++++++++++++++ crates/rustack-lambda-core/src/lib.rs | 1 + crates/rustack-lambda-core/src/provider.rs | 127 +++++++++++++---- crates/rustack-lambda-core/src/resolver.rs | 1 + crates/rustack-lambda-core/src/storage.rs | 109 +++++++++++++- crates/rustack-lambda-core/src/work.rs | 4 + ...tack-workflows-20-lambda-runtime-design.md | 10 +- specs/rustack-workflows-90-impl-plan.md | 11 +- specs/rustack-workflows-93-review.md | 5 + specs/rustack-workflows-99-decisions.md | 8 ++ 16 files changed, 537 insertions(+), 48 deletions(-) create mode 100644 crates/rustack-lambda-core/src/identity.rs diff --git a/Cargo.lock b/Cargo.lock index 885ed16c..a458b3e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3758,6 +3758,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "typed-builder", "uuid", "zip", ] diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 9e932cf7..8605e6db 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -18,6 +18,7 @@ rustack-auth = { workspace = true } squib = { workspace = true } http = { workspace = true } bytes = { workspace = true } +typed-builder = { workspace = true } dashmap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/rustack-lambda-core/src/executor/auto.rs b/crates/rustack-lambda-core/src/executor/auto.rs index 2ace648b..4137940a 100644 --- a/crates/rustack-lambda-core/src/executor/auto.rs +++ b/crates/rustack-lambda-core/src/executor/auto.rs @@ -68,6 +68,10 @@ mod tests { fn request(package_type: PackageType) -> InvokeRequest { InvokeRequest { + resource_id: crate::identity::FunctionResourceId::default(), + revision_id: "test-revision".into(), + image_config: None, + ephemeral_storage_mb: 512, function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), function_name: "echo".to_owned(), qualifier: "$LATEST".to_owned(), diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 573e287f..8c713e6c 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -123,15 +123,12 @@ impl InstancePool { let identity = format!( "{:?}", ( - &req.qualifier, - &req.code_root, - &req.image_uri, - &req.runtime, - &req.handler, - &req.architectures, + (req.resource_id, &req.revision_id, &req.qualifier), + (&req.code_root, &req.image_uri), + (&req.runtime, &req.handler, &req.architectures), environment, - req.timeout, - req.memory_mb + (req.timeout, req.memory_mb, req.ephemeral_storage_mb), + &req.image_config, ) ); ( @@ -313,3 +310,52 @@ pub(crate) fn spawn_reaper( } }) } + +#[cfg(test)] +mod tests { + use rustack_lambda_model::types::ImageConfig; + + use super::*; + + #[test] + fn test_should_key_warm_environment_by_resource_and_captured_configuration() { + let request = InvokeRequest::fixture(); + let original = InstancePool::key(&request); + let mut alias = request.clone(); + alias.function_arn.push_str(":prod"); + assert_eq!(InstancePool::key(&alias), original); + + let mut recreated = request.clone(); + recreated.resource_id = crate::identity::FunctionResourceId::default(); + assert_ne!(InstancePool::key(&recreated), original); + let mut updated = request.clone(); + updated.revision_id = "new-revision".into(); + assert_ne!(InstancePool::key(&updated), original); + let mut storage = request.clone(); + storage.ephemeral_storage_mb = 1024; + assert_ne!(InstancePool::key(&storage), original); + let mut image = request.clone(); + image.image_config = Some(ImageConfig { + command: Some(vec![]), + ..ImageConfig::default() + }); + assert_ne!(InstancePool::key(&image), original); + let explicitly_empty = InstancePool::key(&image); + image.image_config = Some(ImageConfig::default()); + assert_ne!(InstancePool::key(&image), explicitly_empty); + } + + #[test] + fn test_should_fingerprint_environment_independently_of_hashmap_iteration_order() { + let mut first = InvokeRequest::fixture(); + first.environment.insert("FIRST".into(), "one".into()); + first.environment.insert("SECOND".into(), "two".into()); + let mut second = first.clone(); + second.environment.clear(); + second.environment.insert("SECOND".into(), "two".into()); + second.environment.insert("FIRST".into(), "one".into()); + assert_eq!(InstancePool::key(&first), InstancePool::key(&second)); + second.environment.insert("SECOND".into(), "changed".into()); + assert_ne!(InstancePool::key(&first), InstancePool::key(&second)); + } +} diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index fd656489..d77b7973 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -701,6 +701,10 @@ mod tests { #[test] fn test_should_encode_request_payload_base64() { let req = InvokeRequest { + resource_id: crate::identity::FunctionResourceId::default(), + revision_id: "test-revision".into(), + image_config: None, + ephemeral_storage_mb: 512, function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), function_name: "echo".to_owned(), qualifier: "$LATEST".to_owned(), @@ -786,6 +790,10 @@ mod tests { #[test] fn test_should_reject_non_arm64_request() { let req = InvokeRequest { + resource_id: crate::identity::FunctionResourceId::default(), + revision_id: "test-revision".into(), + image_config: None, + ephemeral_storage_mb: 512, function_arn: String::new(), function_name: "echo".to_owned(), qualifier: "$LATEST".to_owned(), diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index 5ad04574..8f61417e 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -1,8 +1,12 @@ //! Shared executor request/response/config types. -use std::{collections::HashMap, path::PathBuf, str::FromStr, time::Duration}; +use std::{collections::HashMap, fmt, path::PathBuf, str::FromStr, time::Duration}; use bytes::Bytes; +use rustack_lambda_model::types::ImageConfig; +use typed_builder::TypedBuilder; + +use crate::identity::FunctionResourceId; /// Selects which execution backend the provider should construct at startup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -58,8 +62,17 @@ impl PackageType { } /// Everything an executor needs to run one invocation. -#[derive(Debug, Clone)] +#[derive(Clone, TypedBuilder)] +#[non_exhaustive] pub struct InvokeRequest { + /// Function lifetime identity; deletion/recreation never reuses it. + pub resource_id: FunctionResourceId, + /// Captured configuration revision, independent of the invoked alias. + pub revision_id: String, + /// Complete Docker entry configuration; omitted and empty argv differ. + pub image_config: Option, + /// Ephemeral storage size in MiB, captured with this invocation's revision. + pub ephemeral_storage_mb: u32, /// Qualified ARN — used for `Lambda-Runtime-Invoked-Function-Arn`. pub function_arn: String, /// Bare function name. @@ -92,9 +105,57 @@ pub struct InvokeRequest { pub capture_logs: bool, } +impl fmt::Debug for InvokeRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvokeRequest") + .field("resource_id", &self.resource_id) + .field("revision_id", &self.revision_id) + .field("function_arn", &self.function_arn) + .field("qualifier", &self.qualifier) + .field("package_type", &self.package_type) + .field("architectures", &self.architectures) + .field("environment", &"[REDACTED]") + .field("image_config", &"[REDACTED]") + .field("code_zip", &"[REDACTED]") + .field("payload", &"[REDACTED]") + .field("timeout", &self.timeout) + .field("memory_mb", &self.memory_mb) + .field("ephemeral_storage_mb", &self.ephemeral_storage_mb) + .field("capture_logs", &self.capture_logs) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +impl InvokeRequest { + pub(crate) fn fixture() -> Self { + Self::builder() + .resource_id(FunctionResourceId::default()) + .revision_id("fixture-revision".into()) + .image_config(None) + .ephemeral_storage_mb(512) + .function_arn("arn:aws:lambda:us-east-1:000000000000:function:fixture".into()) + .function_name("fixture".into()) + .qualifier("$LATEST".into()) + .runtime(Some("provided.al2023".into())) + .handler(Some("bootstrap".into())) + .architectures(vec!["arm64".into()]) + .package_type(PackageType::Zip) + .code_root(Some(PathBuf::from("/fixture/code"))) + .code_zip(Some(Bytes::from_static(b"fixture-archive"))) + .image_uri(None) + .environment(HashMap::new()) + .timeout(Duration::from_secs(3)) + .memory_mb(128) + .payload(Bytes::new()) + .capture_logs(false) + .build() + } +} + /// Outcome of a successful round-trip with the user code (which may itself /// have reported a function error). -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct InvokeResponse { /// HTTP status to mirror back to the caller (typically 200). pub status: u16, @@ -108,6 +169,18 @@ pub struct InvokeResponse { pub executed_version: String, } +impl fmt::Debug for InvokeResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvokeResponse") + .field("status", &self.status) + .field("function_error", &self.function_error.is_some()) + .field("executed_version", &self.executed_version) + .field("payload", &"[REDACTED]") + .field("log_tail", &"[REDACTED]") + .finish() + } +} + impl InvokeResponse { /// Convenience constructor for a plain JSON success response. #[must_use] @@ -126,6 +199,27 @@ impl InvokeResponse { mod tests { use super::*; + #[test] + fn test_should_redact_invocation_request_and_response_debug() { + let secret = "fixture-secret-should-never-be-rendered"; + let mut request = InvokeRequest::fixture(); + request.environment.insert("SECRET".into(), secret.into()); + request.payload = Bytes::from(secret); + request.code_zip = Some(Bytes::from(secret)); + request.image_config = Some(ImageConfig { + entry_point: Some(vec![secret.into()]), + command: Some(vec![secret.into()]), + working_directory: Some(format!("/{secret}")), + }); + let mut response = InvokeResponse::success(Bytes::from(secret), "1".into()); + response.function_error = Some(secret.into()); + response.log_tail = Some(secret.into()); + for rendered in [format!("{request:?}"), format!("{response:?}")] { + assert!(!rendered.contains(secret)); + assert!(rendered.contains("[REDACTED]")); + } + } + #[test] fn test_should_parse_executor_backend_strings() { assert_eq!( diff --git a/crates/rustack-lambda-core/src/identity.rs b/crates/rustack-lambda-core/src/identity.rs new file mode 100644 index 00000000..747496e4 --- /dev/null +++ b/crates/rustack-lambda-core/src/identity.rs @@ -0,0 +1,133 @@ +//! Stable function resource identity, independent of mutable names and revisions. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use uuid::Uuid; + +/// Identity of one function's lifetime, preserved across updates and snapshots. +/// +/// Deleting and recreating a name must allocate a new identity. +/// +/// ``` +/// use rustack_lambda_core::identity::FunctionResourceId; +/// let identity = FunctionResourceId::default(); +/// assert_eq!(identity.to_string().parse::()?, identity); +/// # Ok::<(), rustack_lambda_core::identity::ResourceIdError>(()) +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct FunctionResourceId(Uuid); + +/// A saved resource identity is malformed or absent of meaningful identity. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ResourceIdError { + /// Only the 36-byte hyphenated UUID representation is accepted. + #[error("function resource identity must be a 36-byte UUID")] + Length, + /// The input is not a UUID. + #[error("invalid function resource UUID")] + Invalid { + /// UUID parser failure, without the original input. + #[source] + source: uuid::Error, + }, + /// A nil UUID cannot identify a function lifetime. + #[error("function resource UUID must not be nil")] + Nil, +} + +impl Default for FunctionResourceId { + fn default() -> Self { + Self::new() + } +} + +impl FunctionResourceId { + /// Allocate a random identity for a new function resource. + /// + /// ``` + /// use rustack_lambda_core::identity::FunctionResourceId; + /// let first = FunctionResourceId::new(); + /// assert_ne!(first, FunctionResourceId::new()); + /// ``` + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl fmt::Display for FunctionResourceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl FromStr for FunctionResourceId { + type Err = ResourceIdError; + + fn from_str(value: &str) -> Result { + if value.len() != 36 { + return Err(ResourceIdError::Length); + } + let uuid = Uuid::parse_str(value).map_err(|source| ResourceIdError::Invalid { source })?; + if uuid.is_nil() { + return Err(ResourceIdError::Nil); + } + Ok(Self(uuid)) + } +} + +impl Serialize for FunctionResourceId { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for FunctionResourceId { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + impl de::Visitor<'_> for Visitor { + type Value = FunctionResourceId; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a nonnil hyphenated function resource UUID") + } + + fn visit_str(self, value: &str) -> Result { + value.parse().map_err(E::custom) + } + } + deserializer.deserialize_str(Visitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_generate_distinct_nonnil_serializable_identities() + -> Result<(), Box> { + let first = FunctionResourceId::default(); + let second = FunctionResourceId::default(); + assert_ne!(first, second); + let saved = serde_json::to_string(&first)?; + assert_eq!(serde_json::from_str::(&saved)?, first); + Ok(()) + } + + #[test] + fn test_should_reject_nil_malformed_and_oversized_resource_identity() { + assert!(matches!( + "00000000-0000-0000-0000-000000000000".parse::(), + Err(ResourceIdError::Nil) + )); + for value in ["x".repeat(36), "x".repeat(4096), String::new()] { + assert!(value.parse::().is_err()); + let saved = serde_json::Value::String(value); + assert!(serde_json::from_value::(saved).is_err()); + } + } +} diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 1ed63610..9ba7edc1 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -15,6 +15,7 @@ pub mod config; pub mod error; pub mod executor; pub mod handler; +pub mod identity; pub mod provider; pub mod resolver; pub mod storage; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 7db04184..589a0b3d 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -59,6 +59,7 @@ use crate::{ AutoExecutor, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, NoopExecutor, PackageType, SquibExecutor, }, + identity::FunctionResourceId, resolver::{ alias_arn, function_arn, function_version_arn, layer_arn, layer_version_arn, parse_layer_version_arn, resolve_function_ref, resolve_version, @@ -505,6 +506,7 @@ impl RustackLambda { }; let record = FunctionRecord { + resource_id: FunctionResourceId::new(), name: name.clone(), arn: arn.clone(), latest: version_record, @@ -933,7 +935,7 @@ impl RustackLambda { if let Some(path) = &version.code_path { self.store.validate_artifact(path).await?; } - let req = self.build_invoke_request(&record, version, payload); + let req = self.build_invoke_request(&record, version, qualifier, payload); let (response, handle) = self .work @@ -963,18 +965,13 @@ impl RustackLambda { &self, record: &FunctionRecord, version: &VersionRecord, + invoked_qualifier: Option<&str>, payload: &[u8], ) -> InvokeRequest { - let function_arn = if version.version == "$LATEST" { - record.arn.clone() - } else { - function_version_arn( - &self.config.default_region, - &self.config.account_id, - &record.name, - &version.version, - ) - }; + let function_arn = invoked_qualifier.map_or_else( + || record.arn.clone(), + |qualifier| format!("{}:{qualifier}", record.arn), + ); let mut env = version.environment.clone(); // Always make the rustack region/account visible inside the runtime so // SDK calls back to the same rustack instance work without extra setup. @@ -982,23 +979,27 @@ impl RustackLambda { .or_insert_with(|| self.config.default_region.clone()); env.entry("AWS_DEFAULT_REGION".to_owned()) .or_insert_with(|| self.config.default_region.clone()); - InvokeRequest { - function_arn, - function_name: record.name.clone(), - qualifier: version.version.clone(), - runtime: version.runtime.clone(), - handler: version.handler.clone(), - architectures: version.architectures.clone(), - package_type: PackageType::from_wire(&version.package_type), - code_root: version.code_path.clone(), - code_zip: version.zip_bytes.clone(), - image_uri: version.image_uri.clone(), - environment: env, - timeout: Duration::from_secs(u64::from(version.timeout)), - memory_mb: version.memory_size, - payload: Bytes::copy_from_slice(payload), - capture_logs: false, - } + InvokeRequest::builder() + .resource_id(record.resource_id) + .revision_id(version.revision_id.clone()) + .image_config(version.image_config.clone()) + .ephemeral_storage_mb(version.ephemeral_storage_size) + .function_arn(function_arn) + .function_name(record.name.clone()) + .qualifier(version.version.clone()) + .runtime(version.runtime.clone()) + .handler(version.handler.clone()) + .architectures(version.architectures.clone()) + .package_type(PackageType::from_wire(&version.package_type)) + .code_root(version.code_path.clone()) + .code_zip(version.zip_bytes.clone()) + .image_uri(version.image_uri.clone()) + .environment(env) + .timeout(Duration::from_secs(u64::from(version.timeout))) + .memory_mb(version.memory_size) + .payload(Bytes::copy_from_slice(payload)) + .capture_logs(false) + .build() } // --------------------------------------------------------------- @@ -3048,6 +3049,76 @@ mod tests { } } + #[tokio::test] + async fn test_should_preserve_invoked_alias_separately_from_resolved_version() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("qualified")) + .await?; + provider.publish_version("qualified", &PublishVersionInput::default())?; + provider.create_alias( + "qualified", + CreateAliasInput { + name: "prod".into(), + function_version: "1".into(), + ..CreateAliasInput::default() + }, + )?; + let record = provider.get_record("qualified")?; + for (qualifier, resolved) in [ + (None, "$LATEST"), + (Some("$LATEST"), "$LATEST"), + (Some("prod"), "1"), + (Some("1"), "1"), + ] { + let version = resolve_version(&record, qualifier)?; + let request = provider.build_invoke_request(&record, version, qualifier, b"{}"); + let expected_arn = + qualifier.map_or_else(|| record.arn.clone(), |q| format!("{}:{q}", record.arn)); + assert_eq!(request.function_arn, expected_arn); + assert_eq!(request.qualifier, resolved); + assert_eq!(request.resource_id, record.resource_id); + assert_eq!(request.revision_id, version.revision_id); + } + Ok(()) + } + + #[tokio::test] + async fn test_should_capture_image_overrides_and_ephemeral_storage_by_value() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("captured")) + .await?; + let record = provider.get_record("captured")?; + let mut version = record.latest.clone(); + version.ephemeral_storage_size = 2048; + version.image_config = Some(rustack_lambda_model::types::ImageConfig { + entry_point: Some(vec!["/custom/entrypoint".into()]), + command: Some(vec![]), + working_directory: Some("/function".into()), + }); + let request = provider.build_invoke_request(&record, &version, None, b"{\"value\":1}"); + version.ephemeral_storage_size = 4096; + version.image_config = None; + assert_eq!(request.ephemeral_storage_mb, 2048); + let image = request + .image_config + .as_ref() + .ok_or_else(|| invalid_parameter("Image configuration was lost"))?; + assert_eq!( + image.entry_point.as_deref(), + Some(["/custom/entrypoint".to_owned()].as_slice()) + ); + assert_eq!(image.command.as_deref(), Some([].as_slice())); + assert_eq!(image.working_directory.as_deref(), Some("/function")); + assert_eq!(request.payload, Bytes::from_static(b"{\"value\":1}")); + assert_ne!(request.ephemeral_storage_mb, version.ephemeral_storage_size); + assert!(version.image_config.is_none()); + Ok(()) + } + #[tokio::test] async fn test_should_create_and_get_function() { let provider = test_provider(); diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index 01afab34..d21dc523 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -339,6 +339,7 @@ mod tests { ); FunctionRecord { + resource_id: crate::identity::FunctionResourceId::default(), name: "my-func".to_owned(), arn: "arn:aws:lambda:us-east-1:000000000000:function:my-func".to_owned(), latest: make_version("$LATEST"), diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index a9b392d8..d0b1b220 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -5,7 +5,7 @@ //! SHA-256 hashes computed on ingestion. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, HashSet}, io::ErrorKind, path::{Path, PathBuf}, }; @@ -20,7 +20,7 @@ use rustack_lambda_model::types::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::error::LambdaServiceError; +use crate::{error::LambdaServiceError, identity::FunctionResourceId}; /// In-memory store for Lambda functions. #[derive(Debug)] @@ -67,6 +67,8 @@ impl Drop for Artifact { /// Complete record for a Lambda function. #[derive(Debug, Clone)] pub struct FunctionRecord { + /// Stable identity for this resource's lifetime, independent of its name. + pub resource_id: FunctionResourceId, /// Function name. pub name: String, /// Function ARN. @@ -601,6 +603,9 @@ pub struct FunctionStoreSnapshot { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FunctionRecordSnapshot { + /// Stable identity; older snapshots allocate one when first decoded. + #[serde(default)] + pub resource_id: FunctionResourceId, /// Function name. pub name: String, /// Function ARN. @@ -688,6 +693,7 @@ pub struct VersionRecordSnapshot { impl FunctionRecordSnapshot { fn from_record(record: FunctionRecord) -> Self { Self { + resource_id: record.resource_id, name: record.name, arn: record.arn, latest: VersionRecordSnapshot::from_record(record.latest), @@ -719,6 +725,7 @@ impl FunctionRecordSnapshot { versions.insert(version, record.into_record(store, &name).await?); } Ok(FunctionRecord { + resource_id: self.resource_id, name, arn: self.arn, latest, @@ -985,7 +992,7 @@ impl FunctionStore { } let mut candidate = entry.value().clone(); let result = f(&mut candidate); - if candidate.name != name { + if candidate.name != name || candidate.resource_id != entry.resource_id { return Err(LambdaServiceError::InvalidParameter { message: "Function identity is immutable".into(), }); @@ -1054,7 +1061,13 @@ impl FunctionStore { ) -> Result<(), LambdaServiceError> { self.validate_root().await?; let mut staged = BTreeMap::new(); + let mut identities = HashSet::new(); for function in snapshot.functions { + if !identities.insert(function.resource_id) { + return Err(LambdaServiceError::InvalidParameter { + message: "Duplicate snapshot function resource identity".into(), + }); + } crate::resolver::FunctionName::parse(&function.name)?; let (name, qualifier) = crate::resolver::resolve_function_ref(&function.arn)?; if name != function.name || qualifier.is_some() { @@ -1631,6 +1644,7 @@ mod tests { fn sample_record(name: &str) -> FunctionRecord { FunctionRecord { + resource_id: FunctionResourceId::default(), name: name.to_owned(), arn: format!("arn:aws:lambda:us-east-1:000000000000:function:{name}"), latest: sample_version(), @@ -1647,6 +1661,95 @@ mod tests { } } + #[tokio::test] + async fn test_should_preserve_function_resource_identity_through_snapshot_and_updates() + -> Result<(), Box> { + let store = test_store(); + let record = sample_record("identity"); + let identity = record.resource_id; + store.insert(record)?; + store.update("identity", |record| { + record.latest.description = "changed".into(); + })?; + let saved = serde_json::to_vec(&store.export_snapshot())?; + let snapshot: FunctionStoreSnapshot = serde_json::from_slice(&saved)?; + assert_eq!( + snapshot.functions.first().map(|record| record.resource_id), + Some(identity) + ); + let restored = test_store(); + restored.import_snapshot(snapshot).await?; + assert_eq!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + let result = restored.update("identity", |record| { + record.resource_id = FunctionResourceId::default(); + }); + assert!(matches!( + result, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert_eq!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + restored.remove("identity")?; + restored.insert(sample_record("identity"))?; + assert_ne!( + restored.get("identity")?.map(|record| record.resource_id), + Some(identity) + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_duplicate_snapshot_identities_without_replacing_live_store() + -> Result<(), LambdaServiceError> { + let store = test_store(); + store.insert(sample_record("live"))?; + let mut first = sample_record("first"); + let second = sample_record("second"); + first.resource_id = second.resource_id; + let snapshot = FunctionStoreSnapshot { + functions: vec![ + FunctionRecordSnapshot::from_record(first), + FunctionRecordSnapshot::from_record(second), + ], + }; + assert!(matches!( + store.import_snapshot(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.contains("live")?); + assert_eq!(store.len(), 1); + Ok(()) + } + + #[test] + fn test_should_assign_identity_only_when_legacy_snapshot_omits_it() + -> Result<(), Box> { + let snapshot = FunctionRecordSnapshot::from_record(sample_record("legacy")); + let mut value = serde_json::to_value(snapshot)?; + let fields = value.as_object_mut().ok_or("snapshot must be an object")?; + fields.remove("resourceId"); + let restored: FunctionRecordSnapshot = serde_json::from_value(value.clone())?; + let saved = serde_json::to_value(&restored)?; + assert_eq!( + serde_json::from_value::(saved)?.resource_id, + restored.resource_id + ); + value + .as_object_mut() + .ok_or("snapshot must be an object")? + .insert( + "resourceId".into(), + "00000000-0000-0000-0000-000000000000".into(), + ); + assert!(serde_json::from_value::(value).is_err()); + Ok(()) + } + #[test] fn test_should_insert_and_get_function() { let store = test_store(); diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs index 24ebb4ed..fa7cad54 100644 --- a/crates/rustack-lambda-core/src/work.rs +++ b/crates/rustack-lambda-core/src/work.rs @@ -243,6 +243,10 @@ mod tests { } fn request(name: &str) -> InvokeRequest { InvokeRequest { + resource_id: crate::identity::FunctionResourceId::default(), + revision_id: "test-revision".into(), + image_config: None, + ephemeral_storage_mb: 512, function_name: name.into(), function_arn: "arn".into(), qualifier: "$LATEST".into(), diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index bbf622ab..18126d44 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -1,6 +1,6 @@ # Workflows 20: Docker Lambda Execution Environments -Status: implementation contract; relay composition requires P0 validation · Depends on: [PRD](./rustack-workflows-00-prd.md), [Lambda hardening](./rustack-lambda-hardening-design.md) +Status: implementation contract; relay feasibility validated with platform qualifications · Depends on: [PRD](./rustack-workflows-00-prd.md), [Lambda hardening](./rustack-lambda-hardening-design.md) ## 1. Ownership and execution contract @@ -62,6 +62,14 @@ Map x86_64 to linux/amd64 and arm64 to linux/arm64. Guarantee native architectur Inherit Dockerfile ENTRYPOINT, CMD and WORKDIR, then override only explicitly supplied ImageConfig fields. Missing and explicitly empty arrays differ. WorkingDirectory is an image-internal absolute path≤1,000 bytes, not a host path. Each argv list has≤1,500 elements, each≤4,096 bytes and total≤64 KiB locally. Pass JSON argv directly, never sh -c. Do not force CMD=bootstrap. P0 must prove official Lambda images enter their RIC path when AWS_LAMBDA_RUNTIME_API is set. +### 2.4 Invocation capture and resource identity + +A function receives a nonnil random FunctionResourceId on creation, independent of its name, version and configuration RevisionId. Updates and publication preserve that resource identity; deletion followed by recreation receives a new one. Store mutation cannot change it. Snapshots preserve it; importing an older snapshot without this field assigns an identity once for the restored resource, never once per invocation. An explicitly malformed or nil saved identity rejects restore. + +The provider captures the resource identity, resolved revision, complete ImageConfig (including omitted versus empty argv) and ephemeral storage before submitting work. The Runtime API invoked ARN is the function's canonical ARN with the caller's requested qualifier, if any. Alias invocation therefore reports `:prod` to the handler while ExecutedVersion reports the resolved numeric version. An unqualified request reports an unqualified ARN; explicit `:$LATEST` remains explicit. Pool reuse uses resource and resolved execution configuration, never the invoked alias. Requests' Debug representations redact environment values, payload, archive bytes and image argv; fingerprints may consume those values internally but must not be logged. + +P1 task1.1 lands this capture contract before the artifact-enum and cooperative-cancellation consumers. Its regression tests cover delete/recreate isolation, snapshot identity preservation and rejection, alias versus resolved version, configuration capture and redacted diagnostics. These foundation checks alone do not close M0. + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index fb04bd3f..f4667d29 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -1,6 +1,6 @@ # Workflows 90: Dependency-Ordered Implementation Plan -Status: P0 complete; P1 next; complete M0–M4 implementation objective remains active · Date: 2026-10-25 +Status: P0 complete; P1 in progress; complete M0–M4 implementation objective remains active · Date: 2026-10-25 ## 1. Readiness and dependencies @@ -123,7 +123,7 @@ Add `verify-workflows` and extend existing snapshot/smoke targets rather than in Cancellation/revision contracts precede both the Docker executor and workflow bridge; otherwise stopping a Task leaks resources or reuses mutable code. Compiler and history semantics precede Parallel/Map so branch machinery has one data/error contract. Child subscription/admission rules precede `.sync` so fast completion cannot be missed and parents cannot block child capacity. Snapshot integration follows fully settled lifecycle semantics, avoiding archives that appear resumable while silently replaying side effects. -For implementation, invoke the repository's impl workflow one phase at a time and its independent review requirement; this specification task itself performs no production implementation or runtime acceptance. +Implement phases with the repository's impl workflow and its independent review requirement. Specification publication and P0 feasibility evidence alone do not establish production runtime acceptance. ## 10. Cross-references @@ -138,8 +138,9 @@ The complete implementation objective includes P0–P5. This ledger records evid |---|---|---| | 0.1 | Evidence complete | Real official RIC image execution, warm reuse, handler failures, cgroup/nonroot/rootfs checks, busy-handler cancellation, attach EOF comparison and label reconciliation passed on arm64 VZ and Linux amd64/QEMU. Native x86/Desktop/rootless qualification remains M4. | | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | -| 0.3 | Evidence complete | Model commit/hash and37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task2.1. Rust updated to current stable1.99.1. | +| 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| P1–P5 | Pending | Remain required; no runtime implementation or acceptance is claimed yet. | +| 1.1 | In progress | Captured resource identity, requested qualifier, ImageConfig/ephemeral propagation and diagnostic redaction are implemented and independently reviewed. Standard workspace checks, 154 Lambda core tests, two doctests and six native/DryRun cases pass; artifact enum and cooperative cancellation remain required before task 1.1 closes. Baseline security/strict-lint gates B01/B03 remain open. | +| 1.2–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task1.1; P0 does not advertise Docker execution as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task 1.1; P0 does not advertise Docker execution as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index edfd20fc..99aa4cac 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -27,12 +27,17 @@ All six changes preserve the intended scope. No review finding has been dismisse The independent final recheck confirmed W07–W09 addressed, queried both daemon run labels and found zero containers, and reported no further in-P0 correctness issue. Production consumers remain subject to W01–W06. +## P1 task1.1 foundation slice review + +The independent read-only review against commit `955267f` found no new in-slice correctness, security or compatibility defect. It verified strict resource UUID deserialization, immutable lifetime identity, legacy and current snapshot behavior, duplicate identity rejection, complete configuration capture, alias metadata versus resolved version, warm-key isolation and request/result diagnostic redaction. Ten new regression tests pass within 154 Lambda core tests. This approval covers the capture foundation only; artifact enums, cooperative cancellation and Docker execution remain required in P1. Standard workspace build/test/Clippy/rustdoc and two identity doctests pass; the native target validates six native/DryRun cases (its separately gated Squib case is not execution evidence). Baseline advisory remediation B01 and strict-lint remediation B03 remain open. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | +| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:334`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | Strict production boundary Clippy on lambda-core reports 66 existing findings: 56 missing Errors sections, four documentation-format findings and six indexing/slicing sites. The unfiltered dependency run additionally fails existing rustack-auth/core/model findings. The new identity/request/pool code has no finding in the strict core run; standard workspace Clippy passes | P1 quality work: fix the affected production boundaries and documentation, then rerun strict checks without lint ignores; standard checks alone do not close this gate | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 3d49184e..1448620b 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -84,6 +84,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [runtime spike](../docs/research/spike-lambda-container-relay.md), [semantic evidence](../docs/research/spike-stepfunctions-semantic-oracle.md), [required review fixes](./rustack-workflows-93-review.md). - Date:2026-10-25. These amendments refine D2/D6/D9 without changing the complete M0–M4 scope. +## D12 — Separate resource, resolved revision and invoked reference + +- Alternatives: identify warm environments by function name/alias; use configuration RevisionId as the resource lifetime; reconstruct the invoked ARN from the resolved version. +- Decision: allocate a stable nonnil resource UUID, capture resolved execution configuration by value, and preserve the caller's qualifier separately for Runtime API metadata. +- Why: a recreated name must never acquire a deleted resource's environment, aliases can move without rewriting handler metadata, and snapshots must preserve the same resource identity. Importing legacy snapshots assigns an identity once; invalid or duplicate saved identities reject restore. +- Pinned by: [invocation capture](./rustack-workflows-20-lambda-runtime-design.md#24-invocation-capture-and-resource-identity). +- Date: 2026-10-25. + ## Glossary | Term | Meaning in this specification | From 0f78041be5e4624670e43459eba89ec809277bfa Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 16:45:25 -0700 Subject: [PATCH 003/106] phase 1: own validated deployment artifacts and roll back staging Advance P1 task1.1 toward M0 with closed ZIP/Image artifacts and architecture, shared archive leases, immutable image inspection metadata, bounded reference/settings validation and early source-IO preflight. Unpublished guards roll back failed/cancelled creates, revision conflicts and rejected restores. Native children retain code through exit; Squib wire compatibility is preserved. Workspace build/test/fmt/Clippy/rustdoc pass: 1752 tests, including 203 Lambda core cases; 12 doctests and six native/DryRun cases pass. Independent review findings W10-W12 are fixed. Existing audit/strict-lint debt B01/B03 remains open; actual Docker execution and cooperative invocation cancellation remain pending. The existing synchronous owned-directory Drop exception moves to crates/rustack-lambda-core/src/artifact.rs:133: last-lease destruction cannot await, operates only on the validated private directory and reports cleanup failures. No new gate ignore was added. --- Cargo.lock | 40 ++ Cargo.toml | 2 + crates/rustack-lambda-core/Cargo.toml | 5 + crates/rustack-lambda-core/src/artifact.rs | 152 +++++ .../rustack-lambda-core/src/artifact/image.rs | 567 ++++++++++++++++++ .../rustack-lambda-core/src/executor/auto.rs | 44 +- .../src/executor/instance.rs | 13 +- .../rustack-lambda-core/src/executor/mod.rs | 2 +- .../src/executor/native.rs | 44 +- .../rustack-lambda-core/src/executor/squib.rs | 89 +-- .../rustack-lambda-core/src/executor/types.rs | 66 +- .../src/function_config.rs | 353 +++++++++++ crates/rustack-lambda-core/src/lib.rs | 2 + crates/rustack-lambda-core/src/provider.rs | 430 ++++++++++--- crates/rustack-lambda-core/src/resolver.rs | 1 + crates/rustack-lambda-core/src/storage.rs | 518 ++++++++++++---- crates/rustack-lambda-core/src/work.rs | 30 +- ...tack-workflows-20-lambda-runtime-design.md | 27 +- specs/rustack-workflows-90-impl-plan.md | 2 +- specs/rustack-workflows-93-review.md | 14 +- specs/rustack-workflows-99-decisions.md | 8 + 21 files changed, 1991 insertions(+), 418 deletions(-) create mode 100644 crates/rustack-lambda-core/src/artifact.rs create mode 100644 crates/rustack-lambda-core/src/artifact/image.rs create mode 100644 crates/rustack-lambda-core/src/function_config.rs diff --git a/Cargo.lock b/Cargo.lock index a458b3e6..edd0bb5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1907,6 +1907,12 @@ version = "0.33.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e762b9b188634fc508d4ec54b62ea3d352c43fd431d18d05d46f559f217f4725" +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "group" version = "0.13.0" @@ -2934,6 +2940,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" @@ -3046,6 +3058,32 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b376e512f30a7686a8dbc6aeaace99b3ade1c26c1c63473473a817ade226dafc" +dependencies = [ + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88512dbfb5b73a0dfa38aef7bc73b5657035fce477b923faf12d332dd83051ea" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustack-apigatewayv2-core" version = "0.11.0" @@ -3746,6 +3784,7 @@ dependencies = [ "hyper", "hyper-util", "parking_lot", + "rstest", "rustack-auth", "rustack-core", "rustack-lambda-http", @@ -3760,6 +3799,7 @@ dependencies = [ "tracing", "typed-builder", "uuid", + "winnow", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 2ec944a2..591a2d15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,9 +171,11 @@ mime = "0.3" futures = "0.3" pin-project-lite = "0.2" typed-builder = "0.23" +winnow = { version = "1.0", default-features = false, features = ["std", "parser"] } form_urlencoded = "1.2" # Testing +rstest = { version = "0.28", default-features = false } tokio-test = "0.4" aws-config = { version = "1.12", default-features = false, features = [ "credentials-process", diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 8605e6db..8d88cb18 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -19,6 +19,7 @@ squib = { workspace = true } http = { workspace = true } bytes = { workspace = true } typed-builder = { workspace = true } +winnow = { workspace = true } dashmap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -37,3 +38,7 @@ hyper = { workspace = true } hyper-util = { workspace = true } http-body-util = { workspace = true } parking_lot = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs new file mode 100644 index 00000000..d1bf7a9d --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -0,0 +1,152 @@ +//! Owned, validated deployment artifacts shared by captured execution requests. +//! +//! ZIP leases keep successfully staged code alive through invocation cleanup. +//! Images carry immutable daemon identity rather than an unresolved mutable tag. + +mod image; + +use std::{ + fmt::{Debug, Formatter, Result as FmtResult}, + fs::{canonicalize, remove_dir_all, symlink_metadata}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use bytes::Bytes; +pub use image::{ + Architecture, ArtifactError, ImageMetadata, ImageReference, ResolvedImage, Sha256Digest, +}; +use tempfile::TempDir; + +use crate::storage::compute_sha256; +#[cfg(test)] +use crate::{storage::stage_zip, test_zip::minimal_zip}; + +/// One unambiguous deployment artifact for an invocation. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum DeploymentArtifact { + /// Successfully staged ZIP, with shared lifetime ownership. + Zip(ValidatedArtifact), + /// Resolved Linux image, pinned to immutable daemon content. + Image(ResolvedImage), +} + +/// Shared ownership of a ZIP archive and the extracted code it produced. +/// +/// Only the function store can issue a lease after successful staging. +#[derive(Clone)] +#[non_exhaustive] +pub struct ValidatedArtifact(Arc); + +struct ZipStorage { + directory: Option, + root: PathBuf, + extracted: PathBuf, + bytes: Bytes, + sha256: String, +} + +impl Debug for ZipStorage { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.debug_struct("ZipStorage") + .field("sha256", &self.sha256) + .field("archive_bytes", &self.bytes.len()) + .finish_non_exhaustive() + } +} + +impl Debug for ValidatedArtifact { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.debug_struct("ValidatedArtifact") + .field("sha256", &self.0.sha256) + .field("archive_bytes", &self.0.bytes.len()) + .finish_non_exhaustive() + } +} + +impl ValidatedArtifact { + #[cfg(test)] + pub(crate) fn fixture() -> Self { + stage_zip(&std::env::temp_dir(), Bytes::from(minimal_zip(b"fixture"))) + .expect("trusted ZIP unit fixture must stage successfully") + } + pub(crate) fn from_staged( + directory: TempDir, + root: PathBuf, + extracted: PathBuf, + bytes: Bytes, + ) -> Self { + let sha256 = compute_sha256(&bytes); + Self(Arc::new(ZipStorage { + directory: Some(directory), + root, + extracted, + bytes, + sha256, + })) + } + + /// Absolute root of the extracted code, not a user-controlled logical name. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// assert!(artifact.code_root().is_absolute()); + /// # } + /// ``` + #[must_use] + pub fn code_root(&self) -> &Path { + &self.0.extracted + } + + /// Original bytes from the same successfully staged ZIP. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// let shared_bytes = artifact.zip_bytes().clone(); + /// assert_eq!(shared_bytes, *artifact.zip_bytes()); + /// # } + /// ``` + #[must_use] + pub fn zip_bytes(&self) -> &Bytes { + &self.0.bytes + } + + /// Base64-encoded SHA-256 of the original archive. + /// + /// ``` + /// # fn inspect(artifact: &rustack_lambda_core::artifact::ValidatedArtifact) { + /// assert_eq!(artifact.sha256().len(), 44); + /// # } + /// ``` + #[must_use] + pub fn sha256(&self) -> &str { + &self.0.sha256 + } + + pub(crate) fn storage_root(&self) -> &Path { + &self.0.root + } +} + +impl Drop for ZipStorage { + #[allow(clippy::disallowed_methods)] // Last-lease Drop owns bounded directory cleanup; async IO is unavailable here. + fn drop(&mut self) { + let Some(directory) = self.directory.take() else { + return; + }; + let path = directory.keep(); + let safe = canonicalize(&self.root).is_ok_and(|root| root == self.root) + && symlink_metadata(&self.root) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()) + && symlink_metadata(&path) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()); + if !safe { + tracing::error!("Refusing cleanup of replaced Lambda artifact root"); + return; + } + if let Err(error) = remove_dir_all(path) { + tracing::error!(%error, "Lambda artifact cleanup failed"); + } + } +} diff --git a/crates/rustack-lambda-core/src/artifact/image.rs b/crates/rustack-lambda-core/src/artifact/image.rs new file mode 100644 index 00000000..ae52eaa5 --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/image.rs @@ -0,0 +1,567 @@ +//! Bounded Docker image reference grammar and immutable inspection metadata. + +use std::{fmt, net::Ipv6Addr, str::FromStr}; + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; +use winnow::{ + ModalResult, Parser, + combinator::{alt, repeat, separated}, + token::take_while, +}; + +/// Closed Lambda architecture, also used to select a Docker platform. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Architecture { + /// Linux amd64 image or `x86_64` function. + #[serde(rename = "x86_64")] + X86_64, + /// Linux aarch64 image or arm64 function. + #[serde(rename = "arm64")] + Arm64, +} + +impl Architecture { + /// Lambda wire spelling of this architecture. + /// + /// ``` + /// use rustack_lambda_core::artifact::Architecture; + /// assert_eq!(Architecture::Arm64.as_str(), "arm64"); + /// ``` + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::X86_64 => "x86_64", + Self::Arm64 => "arm64", + } + } + + /// Linux platform selector for Docker. + /// + /// ``` + /// use rustack_lambda_core::artifact::Architecture; + /// assert_eq!(Architecture::X86_64.docker_platform(), "linux/amd64"); + /// ``` + #[must_use] + pub const fn docker_platform(self) -> &'static str { + match self { + Self::X86_64 => "linux/amd64", + Self::Arm64 => "linux/arm64", + } + } +} + +impl FromStr for Architecture { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + match value { + "x86_64" => Ok(Self::X86_64), + "arm64" => Ok(Self::Arm64), + _ => Err(ArtifactError::Architecture), + } + } +} + +/// Invalid artifact metadata; errors never include the untrusted input. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ArtifactError { + /// Reference does not match the bounded Docker grammar. + #[error("invalid Docker image reference")] + Reference, + /// Digest must be SHA-256 with 64 lowercase hexadecimal digits. + #[error("invalid SHA-256 image digest")] + Digest, + /// Only the two Lambda architectures are supported. + #[error("unsupported Lambda architecture")] + Architecture, + /// Inspection did not describe a supported Linux image. + #[error("invalid Linux image inspection metadata")] + Inspection, +} + +/// A validated Docker repository, optional tag, and optional SHA-256 digest. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct ImageReference(String); + +impl ImageReference { + /// Original validated spelling, passed directly to the daemon without a shell. + /// + /// ``` + /// use rustack_lambda_core::artifact::ImageReference; + /// let image: ImageReference = "localhost:5000/team/worker:v1".parse()?; + /// assert_eq!(image.as_str(), "localhost:5000/team/worker:v1"); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for ImageReference { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + if value.is_empty() || value.len() > 1024 || !value.is_ascii() { + return Err(ArtifactError::Reference); + } + let (name_tag, digest) = match value.split_once('@') { + Some((name, digest)) => (name, Some(digest)), + None => (value, None), + }; + if let Some(digest) = digest { + digest.parse::()?; + } + let last_component = name_tag + .rsplit('/') + .next() + .ok_or(ArtifactError::Reference)?; + let (name, tag) = match last_component.split_once(':') { + Some((_, tag)) => { + let (name, _) = name_tag.rsplit_once(':').ok_or(ArtifactError::Reference)?; + (name, Some(tag)) + } + None => (name_tag, None), + }; + if let Some(tag) = tag { + tag_parser + .parse(tag) + .map_err(|_| ArtifactError::Reference)?; + } + if name.len() > 255 { + return Err(ArtifactError::Reference); + } + let repository = if let Some((first, remaining)) = name.split_once('/') { + if first.contains('.') + || first.contains(':') + || first == "localhost" + || first.starts_with('[') + || first.bytes().any(|byte| byte.is_ascii_uppercase()) + { + validate_registry(first)?; + remaining + } else { + name + } + } else { + name + }; + repository_parser + .parse(repository) + .map_err(|_| ArtifactError::Reference)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom for ImageReference { + type Error = ArtifactError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: ImageReference) -> Self { + value.0 + } +} +impl fmt::Display for ImageReference { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +fn component(input: &mut &str) -> ModalResult<()> { + ( + take_while(1.., |c: char| c.is_ascii_lowercase() || c.is_ascii_digit()), + repeat::<_, _, (), _, _>( + 0.., + ( + alt(("__", "_", ".", take_while(1.., '-'))), + take_while(1.., |c: char| c.is_ascii_lowercase() || c.is_ascii_digit()), + ), + ), + ) + .void() + .parse_next(input) +} +fn repository_parser(input: &mut &str) -> ModalResult<()> { + separated(1.., component, '/').parse_next(input) +} + +fn tag_parser(input: &mut &str) -> ModalResult<()> { + ( + take_while(1, |c: char| c.is_ascii_alphanumeric() || c == '_'), + take_while(0..=127, |c: char| { + c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') + }), + ) + .void() + .parse_next(input) +} + +fn digest_parser(input: &mut &str) -> ModalResult<()> { + ( + "sha256:", + take_while(64, |c: char| c.is_ascii_digit() || matches!(c, 'a'..='f')), + ) + .void() + .parse_next(input) +} + +fn validate_registry(registry: &str) -> Result<(), ArtifactError> { + let (host, port) = if let Some(bracketed) = registry.strip_prefix('[') { + let (host, tail) = bracketed.split_once(']').ok_or(ArtifactError::Reference)?; + if !host + .bytes() + .all(|byte| byte.is_ascii_hexdigit() || byte == b':') + { + return Err(ArtifactError::Reference); + } + host.parse::() + .map_err(|_| ArtifactError::Reference)?; + let port = if tail.is_empty() { + None + } else { + Some(tail.strip_prefix(':').ok_or(ArtifactError::Reference)?) + }; + (None, port) + } else { + let (host, port) = registry + .split_once(':') + .map_or((registry, None), |(host, port)| (host, Some(port))); + (Some(host), port) + }; + if let Some(host) = host { + if host.is_empty() + || host.len() > 253 + || host.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || !label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + || label.starts_with('-') + || label.ends_with('-') + }) + { + return Err(ArtifactError::Reference); + } + } + if let Some(port) = port { + if port.is_empty() + || port.len() > 5 + || !port.bytes().all(|b| b.is_ascii_digit()) + || port.parse::().map_or(true, |port| port == 0) + { + return Err(ArtifactError::Reference); + } + } + Ok(()) +} + +/// Immutable SHA-256 digest, including its algorithm prefix. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct Sha256Digest(String); + +impl Sha256Digest { + /// The validated `sha256:` value. + /// + /// ``` + /// use rustack_lambda_core::artifact::Sha256Digest; + /// let digest: Sha256Digest = format!("sha256:{}", "a".repeat(64)).parse()?; + /// assert_eq!(digest.as_str().len(), 71); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} +impl FromStr for Sha256Digest { + type Err = ArtifactError; + fn from_str(value: &str) -> Result { + if value.len() != 71 { + return Err(ArtifactError::Digest); + } + digest_parser + .parse(value) + .map_err(|_| ArtifactError::Digest)?; + Ok(Self(value.to_owned())) + } +} +impl TryFrom for Sha256Digest { + type Error = ArtifactError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: Sha256Digest) -> Self { + value.0 + } +} + +/// Validated primitive fields returned by image inspection. +#[derive(Debug, Clone, TypedBuilder, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ImageMetadata { + /// Caller-supplied deployment reference. + pub requested_reference: ImageReference, + /// Immutable daemon image content ID. + pub image_id: Sha256Digest, + /// Registry repository and selected manifest digest, when available. + pub registry_manifest: Option, + /// Inspected operating system, required to equal linux. + pub os: String, + /// Selected Lambda architecture. + pub architecture: Architecture, + /// SHA-256 of the normalized inspected execution metadata. + pub inspection_hash: Sha256Digest, + /// Inspected uncompressed size, bounded to 10 GiB locally. + pub size_bytes: u64, +} + +/// Immutable Linux image metadata that passed all inspection invariants. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(try_from = "ImageMetadata", into = "ImageMetadata")] +#[non_exhaustive] +pub struct ResolvedImage(ImageMetadata); + +impl TryFrom for ResolvedImage { + type Error = ArtifactError; + fn try_from(metadata: ImageMetadata) -> Result { + if metadata.os != "linux" + || metadata.size_bytes == 0 + || metadata.size_bytes > 10 * 1024 * 1024 * 1024 + || metadata + .registry_manifest + .as_ref() + .is_some_and(|image| !image.as_str().contains('@')) + { + return Err(ArtifactError::Inspection); + } + Ok(Self(metadata)) + } +} +impl From for ImageMetadata { + fn from(image: ResolvedImage) -> Self { + image.0 + } +} + +impl ResolvedImage { + #[cfg(test)] + pub(crate) fn fixture(architecture: Architecture) -> Self { + ImageMetadata::builder() + .requested_reference( + "rustack-fixture:unit" + .parse() + .expect("valid unit reference"), + ) + .image_id( + format!("sha256:{}", "a".repeat(64)) + .parse() + .expect("valid unit digest"), + ) + .registry_manifest(None) + .os("linux".into()) + .architecture(architecture) + .inspection_hash( + format!("sha256:{}", "b".repeat(64)) + .parse() + .expect("valid unit inspection digest"), + ) + .size_bytes(1024) + .build() + .try_into() + .expect("valid unit image metadata") + } + /// Immutable inspected metadata; Invoke must use `image_id` rather than a tag. + /// + /// ``` + /// # fn inspect(image: &rustack_lambda_core::artifact::ResolvedImage) { + /// assert_eq!(image.metadata().os, "linux"); + /// # } + /// ``` + #[must_use] + pub fn metadata(&self) -> &ImageMetadata { + &self.0 + } + + /// Registry manifest reference, or Rustack's explicit local-content extension. + /// + /// ``` + /// # fn inspect(image: &rustack_lambda_core::artifact::ResolvedImage) { + /// assert!(!image.resolved_uri().is_empty()); + /// # } + /// ``` + #[must_use] + pub fn resolved_uri(&self) -> String { + self.0.registry_manifest.as_ref().map_or_else( + || format!("local://{}", self.0.image_id.as_str()), + ToString::to_string, + ) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("worker")] + #[case("team/worker:v1")] + #[case("team/foo__bar:Tag_1.2-3")] + #[case("team/foo---bar:_")] + #[case("localhost:5000/worker:latest")] + #[case("registry.example.com/team/worker:V2")] + #[case("Registry/worker:v1")] + #[case("127.0.0.1:65535/worker")] + #[case("[::1]:5000/team/worker:v1")] + #[case("[2001:db8::1]/worker")] + fn test_should_parse_docker_references_and_round_trip_saved_values( + #[case] reference: &str, + ) -> Result<(), Box> { + let parsed: ImageReference = reference.parse()?; + assert_eq!(parsed.as_str(), reference); + let saved = serde_json::to_string(&parsed)?; + assert_eq!(serde_json::from_str::(&saved)?, parsed); + Ok(()) + } + + #[rstest] + #[case("")] + #[case("https://registry.example.com/worker")] + #[case("user@registry.example.com/worker")] + #[case("worker:$(id)")] + #[case("worker;id")] + #[case("worker\n")] + #[case("worker\0")] + #[case("team/UPPER")] + #[case("team//worker")] + #[case("team/../worker")] + #[case("team/foo___bar")] + #[case("team/foo..bar")] + #[case("worker:")] + #[case("worker:.tag")] + #[case("worker:tag:extra")] + #[case("registry.example.com:0/worker")] + #[case("registry.example.com:65536/worker")] + #[case("registry..example.com/worker")] + #[case("registry.example.com:/worker")] + #[case("[::1]extra/worker")] + #[case("[not-ipv6]:5000/worker")] + #[case("[::ffff:192.0.2.1]/worker")] + #[case("[fe80::1%eth0]/worker")] + #[case("worker@sha256:abc")] + fn test_should_reject_invalid_references_at_parse_and_deserialization(#[case] reference: &str) { + assert!(reference.parse::().is_err()); + assert!( + serde_json::from_value::(serde_json::Value::String(reference.into())) + .is_err() + ); + } + + #[test] + fn test_should_enforce_reference_tag_and_repository_byte_limits() -> Result<(), ArtifactError> { + let longest_tag = format!("worker:{}", "t".repeat(128)); + longest_tag.parse::()?; + assert!(format!("{longest_tag}t").parse::().is_err()); + "a".repeat(255).parse::()?; + assert!("a".repeat(256).parse::().is_err()); + assert!("a".repeat(1025).parse::().is_err()); + assert!("wörker".parse::().is_err()); + Ok(()) + } + + #[test] + fn test_should_parse_digest_references_without_confusing_registry_ports_and_tags() + -> Result<(), ArtifactError> { + let digest = format!("sha256:{}", "a".repeat(64)); + for reference in [ + format!("worker@{digest}"), + format!("localhost:5000/team/worker:v1@{digest}"), + format!("[::1]:5000/worker@{digest}"), + ] { + reference.parse::()?; + } + assert!( + format!("worker@{digest}@{digest}") + .parse::() + .is_err() + ); + assert!( + format!("sha256:{}", "A".repeat(64)) + .parse::() + .is_err() + ); + assert!( + format!("sha256:{}", "a".repeat(63)) + .parse::() + .is_err() + ); + assert!( + format!("sha512:{}", "a".repeat(64)) + .parse::() + .is_err() + ); + Ok(()) + } + + #[test] + fn test_should_preserve_local_and_registry_image_identity_through_snapshot_round_trips() + -> Result<(), Box> { + let local = ResolvedImage::fixture(Architecture::Arm64); + assert_eq!( + local.resolved_uri(), + format!("local://sha256:{}", "a".repeat(64)) + ); + let mut metadata = local.metadata().clone(); + let manifest = format!("registry.example.com/worker@sha256:{}", "c".repeat(64)); + metadata.registry_manifest = Some(manifest.parse()?); + let image = ResolvedImage::try_from(metadata)?; + assert_eq!(image.resolved_uri(), manifest); + let saved = serde_json::to_vec(&image)?; + let restored: ResolvedImage = serde_json::from_slice(&saved)?; + assert_eq!(restored.metadata().image_id, image.metadata().image_id); + assert_eq!(restored.metadata().architecture, Architecture::Arm64); + assert_eq!(restored.resolved_uri(), manifest); + Ok(()) + } + + #[test] + fn test_should_revalidate_image_inspection_on_deserialization() + -> Result<(), Box> { + let image = ResolvedImage::fixture(Architecture::X86_64); + let saved = serde_json::to_value(&image)?; + for (field, invalid) in [ + ("os", serde_json::json!("windows")), + ("architecture", serde_json::json!("riscv64")), + ("imageId", serde_json::json!("worker:latest")), + ("registryManifest", serde_json::json!("worker:latest")), + ("sizeBytes", serde_json::json!(0)), + ( + "sizeBytes", + serde_json::json!(10_u64 * 1024 * 1024 * 1024 + 1), + ), + ] { + let mut invalid_image = saved.clone(); + invalid_image + .as_object_mut() + .ok_or("image must be an object")? + .insert(field.into(), invalid); + assert!( + serde_json::from_value::(invalid_image).is_err(), + "{field}" + ); + } + Ok(()) + } +} diff --git a/crates/rustack-lambda-core/src/executor/auto.rs b/crates/rustack-lambda-core/src/executor/auto.rs index 4137940a..ae027524 100644 --- a/crates/rustack-lambda-core/src/executor/auto.rs +++ b/crates/rustack-lambda-core/src/executor/auto.rs @@ -10,9 +10,10 @@ use std::time::Duration; use async_trait::async_trait; use super::{ - Executor, ExecutorError, InvokeRequest, InvokeResponse, NativeExecutor, PackageType, - SquibExecutor, SquibExecutorConfig, + Executor, ExecutorError, InvokeRequest, InvokeResponse, NativeExecutor, SquibExecutor, + SquibExecutorConfig, }; +use crate::artifact::DeploymentArtifact; /// Executor that chooses the concrete backend for each invocation. #[derive(Debug)] @@ -38,7 +39,7 @@ impl AutoExecutor { } fn should_use_squib(req: &InvokeRequest) -> bool { - cfg!(target_os = "macos") && req.package_type == PackageType::Zip + cfg!(target_os = "macos") && matches!(req.artifact, DeploymentArtifact::Zip(_)) } } @@ -60,48 +61,21 @@ impl Executor for AutoExecutor { #[cfg(test)] mod tests { - use std::{collections::HashMap, path::PathBuf, time::Duration}; - - use bytes::Bytes; - use super::*; - - fn request(package_type: PackageType) -> InvokeRequest { - InvokeRequest { - resource_id: crate::identity::FunctionResourceId::default(), - revision_id: "test-revision".into(), - image_config: None, - ephemeral_storage_mb: 512, - function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: Some("provided.al2023".to_owned()), - handler: Some("bootstrap".to_owned()), - architectures: vec!["arm64".to_owned()], - package_type, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04")), - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_secs(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, - } - } + use crate::artifact::{Architecture, ResolvedImage}; #[test] fn test_should_default_macos_zip_to_squib() { assert_eq!( - AutoExecutor::should_use_squib(&request(PackageType::Zip)), + AutoExecutor::should_use_squib(&InvokeRequest::fixture()), cfg!(target_os = "macos") ); } #[test] fn test_should_not_use_squib_for_image_package() { - assert!(!AutoExecutor::should_use_squib(&request( - PackageType::Image - ))); + let mut request = InvokeRequest::fixture(); + request.artifact = DeploymentArtifact::Image(ResolvedImage::fixture(Architecture::Arm64)); + assert!(!AutoExecutor::should_use_squib(&request)); } } diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 8c713e6c..7a6d47e8 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -21,6 +21,7 @@ use super::{ runtime_api::{self, Job, RuntimeApiHandle, RuntimeResult}, types::{InvokeRequest, InvokeResponse}, }; +use crate::artifact::DeploymentArtifact; /// Identifier for the pool slot — a function/version pair. pub(crate) type PoolKey = (String, String); @@ -120,12 +121,16 @@ impl InstancePool { pub(crate) fn key(req: &InvokeRequest) -> PoolKey { let mut environment: Vec<_> = req.environment.iter().collect(); environment.sort(); + let code_root = match &req.artifact { + DeploymentArtifact::Zip(artifact) => Some(artifact.code_root()), + DeploymentArtifact::Image(_) => None, + }; let identity = format!( "{:?}", ( (req.resource_id, &req.revision_id, &req.qualifier), - (&req.code_root, &req.image_uri), - (&req.runtime, &req.handler, &req.architectures), + (&req.artifact, code_root), + (&req.runtime, &req.handler, req.architecture), environment, (req.timeout, req.memory_mb, req.ephemeral_storage_mb), &req.image_config, @@ -331,6 +336,10 @@ mod tests { let mut updated = request.clone(); updated.revision_id = "new-revision".into(); assert_ne!(InstancePool::key(&updated), original); + let mut rehydrated = request.clone(); + rehydrated.artifact = + DeploymentArtifact::Zip(crate::artifact::ValidatedArtifact::fixture()); + assert_ne!(InstancePool::key(&rehydrated), original); let mut storage = request.clone(); storage.ephemeral_storage_mb = 1024; assert_ne!(InstancePool::key(&storage), original); diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index aec3e863..81279bb2 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -31,7 +31,7 @@ pub use error::ExecutorError; pub use native::NativeExecutor; pub use noop::NoopExecutor; pub use squib::{SquibExecutor, SquibExecutorConfig}; -pub use types::{ExecutorBackend, InvokeRequest, InvokeResponse, PackageType}; +pub use types::{ExecutorBackend, InvokeRequest, InvokeResponse}; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. #[async_trait] diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 38310114..58d99726 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -18,9 +18,10 @@ use tokio::{ use tracing::{debug, warn}; use super::{ - Executor, ExecutorError, InvokeRequest, InvokeResponse, PackageType, + Executor, ExecutorError, InvokeRequest, InvokeResponse, instance::{BackendHandle, BackendHandleObj, InstanceBackend, InstancePool, spawn_reaper}, }; +use crate::artifact::{Architecture, DeploymentArtifact}; /// Native process executor. #[derive(Debug)] @@ -70,18 +71,15 @@ impl NativeExecutor { #[async_trait] impl Executor for NativeExecutor { async fn invoke(&self, req: InvokeRequest) -> Result { - if req.package_type != PackageType::Zip { + let DeploymentArtifact::Zip(artifact) = &req.artifact else { return Err(ExecutorError::Unsupported( - "native backend only supports Zip packages".to_owned(), + "native backend only supports Zip packages".into(), )); - } - let code_root = req - .code_root - .as_ref() - .ok_or_else(|| ExecutorError::InvalidCode("missing code root".to_owned()))?; + }; + let code_root = artifact.code_root(); let bootstrap = code_root.join("bootstrap"); ensure_executable(&bootstrap).await?; - if !bootstrap_runs_on_host(&bootstrap, &req.architectures).await? { + if !bootstrap_runs_on_host(&bootstrap, req.architecture).await? { return Err(ExecutorError::Unsupported(format!( "bootstrap {} cannot run on host {}/{}; use docker backend", bootstrap.display(), @@ -126,14 +124,16 @@ impl InstanceBackend for NativeBackend { req: &InvokeRequest, runtime_api_addr: SocketAddr, ) -> Result { - let code_root = req - .code_root - .clone() - .ok_or_else(|| ExecutorError::InvalidCode("missing code root".to_owned()))?; + let DeploymentArtifact::Zip(artifact) = &req.artifact else { + return Err(ExecutorError::Unsupported( + "native backend only supports Zip packages".into(), + )); + }; + let code_root = artifact.code_root(); let bootstrap = code_root.join("bootstrap"); let mut cmd = Command::new(&bootstrap); - cmd.current_dir(&code_root) + cmd.current_dir(code_root) .env_clear() .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -157,6 +157,7 @@ impl InstanceBackend for NativeBackend { logs.spawn(drain_to_buf(stderr, log_buf, "stderr")); } let (cancel, cancelled) = oneshot::channel(); + let artifact_lease = artifact.clone(); let mut processes = self.processes.lock(); while let Some(result) = processes.try_join_next() { if let Err(error) = result { @@ -164,6 +165,8 @@ impl InstanceBackend for NativeBackend { } } processes.spawn(async move { + // Keep code and resources alive until the child exit has been reaped. + let _artifact_lease = artifact_lease; tokio::select! { _ = cancelled => { if let Err(error) = child.kill().await { warn!(%error, "Lambda process cancellation failed"); } }, result = child.wait() => { if let Err(error) = result { warn!(%error, "Lambda process wait failed"); } }, @@ -201,7 +204,7 @@ impl Drop for NativeHandle { /// Check declared architecture and executable format with fallible asynchronous IO. async fn bootstrap_runs_on_host( path: &Path, - declared_archs: &[String], + architecture: Architecture, ) -> Result { // Architecture check first — declared `architectures` must include the // host arch. @@ -210,7 +213,7 @@ async fn bootstrap_runs_on_host( "aarch64" => "arm64", other => other, }; - if !declared_archs.iter().any(|a| a == host_arch) { + if architecture.as_str() != host_arch { return Ok(false); } let mut file = tokio::fs::File::open(path) @@ -252,11 +255,10 @@ async fn ensure_executable(_path: &Path) -> Result<(), ExecutorError> { fn lambda_env_vars(req: &InvokeRequest, runtime_api: SocketAddr) -> Vec<(String, String)> { let mut env = Vec::with_capacity(16 + req.environment.len()); - let task_root = req - .code_root - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_default(); + let task_root = match &req.artifact { + DeploymentArtifact::Zip(artifact) => artifact.code_root().display().to_string(), + DeploymentArtifact::Image(_) => String::new(), + }; env.push(("AWS_LAMBDA_RUNTIME_API".into(), runtime_api.to_string())); env.push(("AWS_LAMBDA_FUNCTION_NAME".into(), req.function_name.clone())); env.push(("AWS_LAMBDA_FUNCTION_VERSION".into(), req.qualifier.clone())); diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index d77b7973..f82523dd 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -19,7 +19,8 @@ use tokio::{sync::Mutex, time::Instant}; use tracing::warn; use uuid::Uuid; -use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse, PackageType}; +use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; +use crate::artifact::{Architecture, DeploymentArtifact}; const DEFAULT_INSTANCE_ID: &str = "rustack_lambda"; const DEFAULT_STAGE_PORT: u32 = 5003; @@ -351,10 +352,11 @@ struct SquibControlRequestWire { impl SquibInvokeRequestWire { fn from_request(req: &InvokeRequest) -> Result { - let code_zip = req - .code_zip - .as_ref() - .ok_or_else(|| ExecutorError::InvalidCode("missing code zip".to_owned()))?; + let DeploymentArtifact::Zip(artifact) = &req.artifact else { + return Err(ExecutorError::Unsupported( + "Squib backend only supports Zip packages".into(), + )); + }; Ok(Self { protocol_version: PROTOCOL_VERSION, request_id: Uuid::new_v4().to_string(), @@ -363,12 +365,9 @@ impl SquibInvokeRequestWire { qualifier: req.qualifier.clone(), runtime: req.runtime.clone(), handler: req.handler.clone(), - architectures: req.architectures.clone(), - code_root: req - .code_root - .as_ref() - .map(|path| path.display().to_string()), - code_zip_base64: STANDARD.encode(code_zip.as_ref()), + architectures: vec![req.architecture.as_str().into()], + code_root: Some(artifact.code_root().display().to_string()), + code_zip_base64: STANDARD.encode(artifact.zip_bytes()), environment: req.environment.clone(), timeout_ms: duration_millis(req.timeout), memory_mb: req.memory_mb, @@ -393,19 +392,16 @@ struct SquibInvokeResponseWire { } fn validate_request(req: &InvokeRequest) -> Result<(), ExecutorError> { - if req.package_type != PackageType::Zip { + if !matches!(req.artifact, DeploymentArtifact::Zip(_)) { return Err(ExecutorError::Unsupported( "Squib backend only supports Zip packages".to_owned(), )); } - if !req.architectures.iter().any(|arch| arch == "arm64") { + if req.architecture != Architecture::Arm64 { return Err(ExecutorError::Unsupported( "Squib backend requires an arm64 Lambda function".to_owned(), )); } - if req.code_zip.is_none() { - return Err(ExecutorError::InvalidCode("missing code zip".to_owned())); - } Ok(()) } @@ -700,33 +696,23 @@ mod tests { #[test] fn test_should_encode_request_payload_base64() { - let req = InvokeRequest { - resource_id: crate::identity::FunctionResourceId::default(), - revision_id: "test-revision".into(), - image_config: None, - ephemeral_storage_mb: 512, - function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo".to_owned(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: Some("provided.al2023".to_owned()), - handler: Some("bootstrap".to_owned()), - architectures: vec!["arm64".to_owned()], - package_type: PackageType::Zip, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04zip")), - image_uri: None, - environment: HashMap::from([("KEY".to_owned(), "VALUE".to_owned())]), - timeout: Duration::from_secs(3), - memory_mb: 128, - payload: Bytes::from_static(b"{}"), - capture_logs: true, - }; - + let mut req = InvokeRequest::fixture(); + req.payload = Bytes::from_static(b"{}"); + req.capture_logs = true; let wire = SquibInvokeRequestWire::from_request(&req).unwrap(); assert_eq!(wire.protocol_version, PROTOCOL_VERSION); - assert_eq!(wire.code_root.as_deref(), Some("/tmp/code")); - assert_eq!(wire.code_zip_base64, "UEsDBHppcA=="); + let DeploymentArtifact::Zip(artifact) = &req.artifact else { + panic!("unit fixture is ZIP"); + }; + assert_eq!( + wire.code_root, + Some(artifact.code_root().display().to_string()) + ); + assert_eq!( + STANDARD.decode(&wire.code_zip_base64).unwrap(), + *artifact.zip_bytes() + ); assert_eq!(wire.timeout_ms, 3000); assert_eq!(wire.payload_base64, "e30="); } @@ -789,27 +775,8 @@ mod tests { #[test] fn test_should_reject_non_arm64_request() { - let req = InvokeRequest { - resource_id: crate::identity::FunctionResourceId::default(), - revision_id: "test-revision".into(), - image_config: None, - ephemeral_storage_mb: 512, - function_arn: String::new(), - function_name: "echo".to_owned(), - qualifier: "$LATEST".to_owned(), - runtime: None, - handler: None, - architectures: vec!["x86_64".to_owned()], - package_type: PackageType::Zip, - code_root: Some(PathBuf::from("/tmp/code")), - code_zip: Some(Bytes::from_static(b"PK\x03\x04zip")), - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_secs(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, - }; + let mut req = InvokeRequest::fixture(); + req.architecture = Architecture::X86_64; assert!(matches!( validate_request(&req), diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index 8f61417e..061f400b 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -1,12 +1,17 @@ //! Shared executor request/response/config types. -use std::{collections::HashMap, fmt, path::PathBuf, str::FromStr, time::Duration}; +use std::{collections::HashMap, fmt, str::FromStr, time::Duration}; use bytes::Bytes; use rustack_lambda_model::types::ImageConfig; use typed_builder::TypedBuilder; -use crate::identity::FunctionResourceId; +#[cfg(test)] +use crate::artifact::ValidatedArtifact; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + identity::FunctionResourceId, +}; /// Selects which execution backend the provider should construct at startup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -40,27 +45,6 @@ impl FromStr for ExecutorBackend { } } -/// Deployment package kind, mirroring the Lambda API. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PackageType { - /// Unzipped function code on the local filesystem. - Zip, - /// Container image referenced by URI. - Image, -} - -impl PackageType { - /// Parse from the wire string used by the Lambda model. - #[must_use] - pub fn from_wire(s: &str) -> Self { - if s.eq_ignore_ascii_case("Image") { - Self::Image - } else { - Self::Zip - } - } -} - /// Everything an executor needs to run one invocation. #[derive(Clone, TypedBuilder)] #[non_exhaustive] @@ -83,16 +67,10 @@ pub struct InvokeRequest { pub runtime: Option, /// Handler string from the function config. pub handler: Option, - /// Architectures the function was created with, e.g. `["x86_64"]`. - pub architectures: Vec, - /// Package type — Zip uses `code_root`, Image uses `image_uri`. - pub package_type: PackageType, - /// Filesystem path to the unzipped code root (for Zip packages). - pub code_root: Option, - /// Raw uploaded Zip bytes (for microVM staging). - pub code_zip: Option, - /// Image URI (for Image packages). - pub image_uri: Option, + /// Validated single Lambda architecture. + pub architecture: Architecture, + /// Exactly one immutable deployment artifact, with lifetime ownership. + pub artifact: DeploymentArtifact, /// User-supplied environment variables. pub environment: HashMap, /// Function timeout. Excess => `Sandbox.Timedout`. @@ -112,11 +90,10 @@ impl fmt::Debug for InvokeRequest { .field("revision_id", &self.revision_id) .field("function_arn", &self.function_arn) .field("qualifier", &self.qualifier) - .field("package_type", &self.package_type) - .field("architectures", &self.architectures) + .field("artifact", &self.artifact) + .field("architecture", &self.architecture) .field("environment", &"[REDACTED]") .field("image_config", &"[REDACTED]") - .field("code_zip", &"[REDACTED]") .field("payload", &"[REDACTED]") .field("timeout", &self.timeout) .field("memory_mb", &self.memory_mb) @@ -139,11 +116,8 @@ impl InvokeRequest { .qualifier("$LATEST".into()) .runtime(Some("provided.al2023".into())) .handler(Some("bootstrap".into())) - .architectures(vec!["arm64".into()]) - .package_type(PackageType::Zip) - .code_root(Some(PathBuf::from("/fixture/code"))) - .code_zip(Some(Bytes::from_static(b"fixture-archive"))) - .image_uri(None) + .architecture(Architecture::Arm64) + .artifact(DeploymentArtifact::Zip(ValidatedArtifact::fixture())) .environment(HashMap::new()) .timeout(Duration::from_secs(3)) .memory_mb(128) @@ -205,7 +179,6 @@ mod tests { let mut request = InvokeRequest::fixture(); request.environment.insert("SECRET".into(), secret.into()); request.payload = Bytes::from(secret); - request.code_zip = Some(Bytes::from(secret)); request.image_config = Some(ImageConfig { entry_point: Some(vec![secret.into()]), command: Some(vec![secret.into()]), @@ -248,13 +221,4 @@ mod tests { ); assert!("nope".parse::().is_err()); } - - #[test] - fn test_should_parse_package_type_from_wire() { - assert_eq!(PackageType::from_wire("Image"), PackageType::Image); - assert_eq!(PackageType::from_wire("image"), PackageType::Image); - assert_eq!(PackageType::from_wire("Zip"), PackageType::Zip); - // Anything unrecognized falls back to Zip — matches the provider default. - assert_eq!(PackageType::from_wire(""), PackageType::Zip); - } } diff --git a/crates/rustack-lambda-core/src/function_config.rs b/crates/rustack-lambda-core/src/function_config.rs new file mode 100644 index 00000000..692c5fc9 --- /dev/null +++ b/crates/rustack-lambda-core/src/function_config.rs @@ -0,0 +1,353 @@ +//! Validate captured function settings before storage or execution admission. + +use std::{collections::HashMap, fmt}; + +use rustack_lambda_model::{input::CreateFunctionInput, types::ImageConfig}; +use typed_builder::TypedBuilder; + +use crate::{ + artifact::{Architecture, ImageReference, ResolvedImage}, + error::LambdaServiceError, + storage::{VersionRecord, VersionRecordSnapshot}, +}; + +fn invalid(message: &'static str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} + +#[derive(TypedBuilder)] +struct Settings<'a> { + package_type: &'a str, + runtime: Option<&'a str>, + handler: Option<&'a str>, + architectures: Option<&'a [String]>, + environment: Option<&'a HashMap>, + image_config: Option<&'a ImageConfig>, + image_uri: Option<&'a str>, + timeout: u32, + memory: u32, + ephemeral: u32, +} + +impl fmt::Debug for Settings<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Settings") + .field("package_type", &self.package_type) + .field("environment", &"[REDACTED]") + .field("image_config", &"[REDACTED]") + .finish_non_exhaustive() + } +} + +impl Settings<'_> { + fn validate(&self) -> Result { + if !(1..=900).contains(&self.timeout) + || !(128..=10240).contains(&self.memory) + || !(512..=10240).contains(&self.ephemeral) + { + return Err(invalid( + "Invalid function timeout, memory or ephemeral storage", + )); + } + let architecture = match self.architectures { + None => Architecture::X86_64, + Some([architecture]) => architecture + .parse() + .map_err(|_| invalid("Invalid Lambda architecture"))?, + Some(_) => return Err(invalid("Exactly one Lambda architecture is required")), + }; + match self.package_type { + "Zip" => { + if self.runtime.is_none_or(|runtime| { + runtime.is_empty() || runtime.len() > 128 || runtime.contains('\0') + }) || self.handler.is_none_or(|handler| { + handler.is_empty() || handler.len() > 256 || handler.contains('\0') + }) || self.image_uri.is_some() + || self.image_config.is_some() + { + return Err(invalid("Invalid ZIP runtime, handler or image fields")); + } + } + "Image" => { + let reference = self + .image_uri + .ok_or_else(|| invalid("ImageUri is required"))?; + reference + .parse::() + .map_err(|_| invalid("Invalid ImageUri"))?; + if self.runtime.is_some() || self.handler.is_some() { + return Err(invalid( + "Image functions cannot contain ZIP runtime or handler fields", + )); + } + if let Some(config) = self.image_config { + validate_image_config(config)?; + } + } + _ => return Err(invalid("Unsupported deployment package type")), + } + if let Some(environment) = self.environment { + validate_environment(environment)?; + } + Ok(architecture) + } +} + +pub(crate) fn validate_create_settings( + input: &CreateFunctionInput, +) -> Result { + Settings::builder() + .package_type(input.package_type.as_deref().unwrap_or("Zip")) + .runtime(input.runtime.as_deref()) + .handler(input.handler.as_deref()) + .architectures(input.architectures.as_deref()) + .environment( + input + .environment + .as_ref() + .and_then(|environment| environment.variables.as_ref()), + ) + .image_config(input.image_config.as_ref()) + .image_uri(input.code.image_uri.as_deref()) + .timeout(input.timeout.unwrap_or(3)) + .memory(input.memory_size.unwrap_or(128)) + .ephemeral( + input + .ephemeral_storage + .as_ref() + .map_or(512, |storage| storage.size), + ) + .build() + .validate() +} + +pub(crate) fn validate_execution_settings( + version: &VersionRecord, +) -> Result { + let architecture = Settings::builder() + .package_type(&version.package_type) + .runtime(version.runtime.as_deref()) + .handler(version.handler.as_deref()) + .architectures(Some(&version.architectures)) + .environment(Some(&version.environment)) + .image_config(version.image_config.as_ref()) + .image_uri(version.image_uri.as_deref()) + .timeout(version.timeout) + .memory(version.memory_size) + .ephemeral(version.ephemeral_storage_size) + .build() + .validate()?; + validate_layout( + &version.package_type, + version.zip_bytes.is_some(), + version.code_path.is_some(), + version.resolved_image.as_ref(), + architecture, + version.image_uri.as_deref(), + )?; + Ok(architecture) +} + +pub(crate) fn validate_snapshot_settings( + version: &VersionRecordSnapshot, +) -> Result<(), LambdaServiceError> { + let architecture = Settings::builder() + .package_type(&version.package_type) + .runtime(version.runtime.as_deref()) + .handler(version.handler.as_deref()) + .architectures(Some(&version.architectures)) + .environment(Some(&version.environment)) + .image_config(version.image_config.as_ref()) + .image_uri(version.image_uri.as_deref()) + .timeout(version.timeout) + .memory(version.memory_size) + .ephemeral(version.ephemeral_storage_size) + .build() + .validate()?; + validate_layout( + &version.package_type, + version.zip_bytes_base64.is_some(), + false, + version.resolved_image.as_ref(), + architecture, + version.image_uri.as_deref(), + ) +} + +fn validate_layout( + package_type: &str, + has_zip: bool, + has_extracted: bool, + image: Option<&ResolvedImage>, + architecture: Architecture, + reference: Option<&str>, +) -> Result<(), LambdaServiceError> { + if (package_type == "Zip" && image.is_some()) + || (package_type == "Image" && (has_extracted || has_zip)) + { + return Err(invalid("Deployment package fields are inconsistent")); + } + if let Some(image) = image { + if image.metadata().architecture != architecture + || Some(image.metadata().requested_reference.as_str()) != reference + { + return Err(invalid( + "Resolved image does not match function configuration", + )); + } + } + Ok(()) +} + +fn validate_image_config(config: &ImageConfig) -> Result<(), LambdaServiceError> { + let mut bytes = 0usize; + for args in [config.entry_point.as_deref(), config.command.as_deref()] + .into_iter() + .flatten() + { + if args.len() > 1500 { + return Err(invalid("Image argv count exceeds the local limit")); + } + for arg in args { + if arg.len() > 4096 || arg.contains('\0') { + return Err(invalid("Invalid image argument")); + } + bytes = bytes + .checked_add(arg.len()) + .ok_or_else(|| invalid("Image argv size overflow"))?; + if bytes > 64 * 1024 { + return Err(invalid("Image argv exceeds the local byte limit")); + } + } + } + if config + .working_directory + .as_ref() + .is_some_and(|path| !path.starts_with('/') || path.len() > 1000 || path.contains('\0')) + { + return Err(invalid( + "Image working directory must be a bounded absolute path", + )); + } + Ok(()) +} + +fn validate_environment(environment: &HashMap) -> Result<(), LambdaServiceError> { + if environment.len() > 128 { + return Err(invalid("Environment has too many variables")); + } + let mut bytes = 0usize; + for (key, value) in environment { + if !(2..=64).contains(&key.len()) + || !key.as_bytes().first().is_some_and(u8::is_ascii_alphabetic) + || !key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + || value.len() > 4096 + || value.contains('\0') + || is_reserved(key) + { + return Err(invalid("Invalid or reserved environment variable")); + } + bytes = bytes + .checked_add(key.len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .ok_or_else(|| invalid("Environment size overflow"))?; + if bytes > 4096 { + return Err(invalid("Environment exceeds 4 KiB")); + } + } + Ok(()) +} + +fn is_reserved(key: &str) -> bool { + matches!( + key, + "_HANDLER" + | "_X_AMZN_TRACE_ID" + | "AWS_DEFAULT_REGION" + | "AWS_REGION" + | "AWS_EXECUTION_ENV" + | "AWS_LAMBDA_FUNCTION_NAME" + | "AWS_LAMBDA_FUNCTION_MEMORY_SIZE" + | "AWS_LAMBDA_FUNCTION_VERSION" + | "AWS_LAMBDA_FUNCTION_TIMEOUT" + | "AWS_LAMBDA_INITIALIZATION_TYPE" + | "AWS_LAMBDA_LOG_GROUP_NAME" + | "AWS_LAMBDA_LOG_STREAM_NAME" + | "AWS_ACCESS_KEY" + | "AWS_ACCESS_KEY_ID" + | "AWS_SECRET_ACCESS_KEY" + | "AWS_SESSION_TOKEN" + | "AWS_LAMBDA_RUNTIME_API" + | "LAMBDA_TASK_ROOT" + | "LAMBDA_RUNTIME_DIR" + | "AWS_LAMBDA_MAX_CONCURRENCY" + | "AWS_LAMBDA_METADATA_API" + | "AWS_LAMBDA_METADATA_TOKEN" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_bound_environment_bytes_and_reject_reserved_or_invalid_keys() { + let at_limit = HashMap::from([("KEY".into(), "v".repeat(4093))]); + assert!(validate_environment(&at_limit).is_ok()); + let over_limit = HashMap::from([("KEY".into(), "v".repeat(4094))]); + assert!(validate_environment(&over_limit).is_err()); + for key in [ + "AWS_REGION", + "AWS_LAMBDA_RUNTIME_API", + "AWS_SECRET_ACCESS_KEY", + "LAMBDA_TASK_ROOT", + "A", + "A-B", + "ÄB", + ] { + let environment = HashMap::from([(key.into(), "value".into())]); + assert!(validate_environment(&environment).is_err(), "{key}"); + } + for key in ["AWS_ENDPOINT_URL", "TZ", "PATH"] { + assert!(validate_environment(&HashMap::from([(key.into(), "value".into())])).is_ok()); + } + } + + #[test] + fn test_should_validate_image_argv_without_collapsing_empty_and_missing() { + assert!(validate_image_config(&ImageConfig::default()).is_ok()); + assert!( + validate_image_config(&ImageConfig { + command: Some(vec![]), + ..ImageConfig::default() + }) + .is_ok() + ); + for config in [ + ImageConfig { + command: Some(vec!["x".repeat(4097)]), + ..ImageConfig::default() + }, + ImageConfig { + command: Some(vec![String::new(); 1501]), + ..ImageConfig::default() + }, + ImageConfig { + command: Some(vec!["x".repeat(4096); 17]), + ..ImageConfig::default() + }, + ImageConfig { + entry_point: Some(vec!["x\0".into()]), + ..ImageConfig::default() + }, + ImageConfig { + working_directory: Some("relative".into()), + ..ImageConfig::default() + }, + ] { + assert!(validate_image_config(&config).is_err()); + } + } +} diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 9ba7edc1..63ca483f 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -10,10 +10,12 @@ #![forbid(unsafe_code)] +pub mod artifact; pub mod code; pub mod config; pub mod error; pub mod executor; +mod function_config; pub mod handler; pub mod identity; pub mod provider; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 589a0b3d..e43d2009 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -5,7 +5,7 @@ //! implemented; later phases (versions, aliases, permissions, tags, URLs) //! return appropriate errors until implemented. -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use bytes::Bytes; use serde::{Deserialize, Serialize}; @@ -17,6 +17,9 @@ const MAX_ZIP_SIZE: u64 = 50 * 1024 * 1024; /// Maximum synchronous invoke payload size (6 MB, per Appendix C). const MAX_SYNC_PAYLOAD: usize = 6 * 1024 * 1024; +/// Hard upper bound for one code-source fetch before any artifact is staged. +const CODE_FETCH_TIMEOUT: Duration = Duration::from_secs(30); + /// Maximum S3 bucket name length (per S3 naming rules). const MAX_S3_BUCKET_LEN: usize = 63; @@ -51,13 +54,16 @@ use rustack_lambda_model::{ }, }; +#[cfg(test)] +use crate::artifact::Architecture; use crate::{ + artifact::{DeploymentArtifact, ImageReference}, code::{S3CodeFetchError, S3CodeFetcher, UnavailableS3CodeFetcher}, config::LambdaConfig, error::LambdaServiceError, executor::{ AutoExecutor, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, NoopExecutor, - PackageType, SquibExecutor, + SquibExecutor, }, identity::FunctionResourceId, resolver::{ @@ -68,7 +74,7 @@ use crate::{ AliasRecord, EventInvokeConfigRecord, EventSourceMappingRecord, EventSourceMappingStore, EventSourceMappingStoreSnapshot, FunctionRecord, FunctionStore, FunctionStoreSnapshot, FunctionUrlConfigRecord, LayerStore, LayerStoreSnapshot, LayerVersionRecord, - PolicyDocument, PolicyStatement, VersionRecord, compute_sha256, + PolicyDocument, PolicyStatement, StagedArtifact, VersionRecord, compute_sha256, }, }; @@ -166,6 +172,42 @@ struct CodeSource<'a> { image_uri: Option<&'a str>, } +#[derive(Debug)] +enum ProcessedCode<'a> { + Zip(StagedArtifact<'a>), + Image(ImageReference), +} + +impl ProcessedCode<'_> { + fn parts(&self) -> (String, u64, Option, Option, Option) { + match self { + Self::Zip(staged) => { + let artifact = staged.artifact(); + ( + artifact.sha256().to_owned(), + artifact.zip_bytes().len() as u64, + Some(artifact.zip_bytes().clone()), + Some(artifact.code_root().to_path_buf()), + None, + ) + } + Self::Image(reference) => ( + compute_sha256(reference.as_str().as_bytes()), + 0, + None, + None, + Some(reference.as_str().into()), + ), + } + } + + fn commit(self) { + if let Self::Zip(staged) = self { + staged.commit(); + } + } +} + impl RustackLambda { /// Create a new Lambda provider with the given store and config. /// @@ -459,9 +501,9 @@ impl RustackLambda { } } - // Process code. - let (code_sha256, code_size, zip_bytes, code_path, image_uri) = - self.process_code(name, "$LATEST", code_source).await?; + crate::function_config::validate_create_settings(&input)?; + let processed = self.process_code(name, "$LATEST", code_source).await?; + let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); let timeout = input.timeout.unwrap_or(3); let memory_size = input.memory_size.unwrap_or(128); @@ -488,6 +530,7 @@ impl RustackLambda { package_type: package_type.clone(), code_path, image_uri, + resolved_image: None, zip_bytes, state: "Active".to_owned(), last_modified: now.clone(), @@ -524,6 +567,7 @@ impl RustackLambda { let should_publish = input.publish.unwrap_or(false); self.store.insert(record)?; + processed.commit(); // If publish=true, publish version 1 immediately. let config = if should_publish { @@ -637,6 +681,11 @@ impl RustackLambda { let (name, _) = resolve_function_ref(function_ref)?; let should_publish = input.publish.unwrap_or(false); let before = self.get_record(&name)?; + if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) + || (before.latest.package_type == "Image" && code_source.image_uri.is_none()) + { + return Err(invalid_parameter("Deployment package type cannot change")); + } if input.architectures.as_ref().is_some_and(|architectures| { architectures.len() != 1 || architectures @@ -648,8 +697,8 @@ impl RustackLambda { )); } - let (code_sha256, code_size, zip_bytes, code_path, image_uri) = - self.process_code(&name, "$LATEST", code_source).await?; + let processed = self.process_code(&name, "$LATEST", code_source).await?; + let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); let config = self.store @@ -660,6 +709,7 @@ impl RustackLambda { record.latest.zip_bytes = zip_bytes; record.latest.code_path = code_path; record.latest.image_uri = image_uri; + record.latest.resolved_image = None; record.latest.last_modified = now; record.latest.revision_id = uuid::Uuid::new_v4().to_string(); @@ -679,6 +729,7 @@ impl RustackLambda { } })?; + processed.commit(); info!(function_name = %name, "updated Lambda function code"); Ok(config) } @@ -931,11 +982,9 @@ impl RustackLambda { message: "Docker execution is not implemented in this build".into(), }); } - self.store.validate_root().await?; - if let Some(path) = &version.code_path { - self.store.validate_artifact(path).await?; - } - let req = self.build_invoke_request(&record, version, qualifier, payload); + let req = self + .build_invoke_request(&record, version, qualifier, payload) + .await?; let (response, handle) = self .work @@ -961,13 +1010,55 @@ impl RustackLambda { } /// Build the executor request from a resolved function/version record. - fn build_invoke_request( + async fn build_invoke_request( &self, record: &FunctionRecord, version: &VersionRecord, invoked_qualifier: Option<&str>, payload: &[u8], - ) -> InvokeRequest { + ) -> Result { + let architecture = crate::function_config::validate_execution_settings(version)?; + if version.state != "Active" { + return Err(LambdaServiceError::ResourceNotReady { + message: "Function revision is not Active".into(), + }); + } + let artifact = match version.package_type.as_str() { + "Zip" => { + let path = version + .code_path + .as_ref() + .ok_or_else(|| invalid_parameter("ZIP code has not been staged"))?; + let artifact = self.store.capture_artifact(path).await?; + if artifact.sha256() != version.code_sha256 + || artifact.zip_bytes().len() as u64 != version.code_size + { + return Err(invalid_parameter( + "Captured ZIP metadata does not match the revision", + )); + } + DeploymentArtifact::Zip(artifact) + } + "Image" => { + let image = version.resolved_image.as_ref().ok_or_else(|| { + LambdaServiceError::ResourceNotReady { + message: "Image deployment has not been resolved".into(), + } + })?; + if image.metadata().architecture != architecture + || version.image_uri.as_deref() + != Some(image.metadata().requested_reference.as_str()) + || version.runtime.is_some() + || version.handler.is_some() + { + return Err(invalid_parameter( + "Image metadata does not match the captured revision", + )); + } + DeploymentArtifact::Image(image.clone()) + } + _ => return Err(invalid_parameter("Unsupported deployment package type")), + }; let function_arn = invoked_qualifier.map_or_else( || record.arn.clone(), |qualifier| format!("{}:{qualifier}", record.arn), @@ -979,7 +1070,7 @@ impl RustackLambda { .or_insert_with(|| self.config.default_region.clone()); env.entry("AWS_DEFAULT_REGION".to_owned()) .or_insert_with(|| self.config.default_region.clone()); - InvokeRequest::builder() + Ok(InvokeRequest::builder() .resource_id(record.resource_id) .revision_id(version.revision_id.clone()) .image_config(version.image_config.clone()) @@ -989,17 +1080,14 @@ impl RustackLambda { .qualifier(version.version.clone()) .runtime(version.runtime.clone()) .handler(version.handler.clone()) - .architectures(version.architectures.clone()) - .package_type(PackageType::from_wire(&version.package_type)) - .code_root(version.code_path.clone()) - .code_zip(version.zip_bytes.clone()) - .image_uri(version.image_uri.clone()) + .architecture(architecture) + .artifact(artifact) .environment(env) .timeout(Duration::from_secs(u64::from(version.timeout))) .memory_mb(version.memory_size) .payload(Bytes::copy_from_slice(payload)) .capture_logs(false) - .build() + .build()) } // --------------------------------------------------------------- @@ -2685,73 +2773,48 @@ impl RustackLambda { /// S3 packages are downloaded through the configured [`S3CodeFetcher`] and /// stored via the same path as inline `ZipFile`, so downstream validation, /// checksum computation, and executor behavior are identical. - async fn process_code( - &self, + async fn process_code<'a>( + &'a self, function_name: &str, version: &str, source: CodeSource<'_>, - ) -> Result< - ( - String, - u64, - Option, - Option, - Option, - ), - LambdaServiceError, - > { - if let Some(b64) = source.zip_file_b64 { + ) -> Result, LambdaServiceError> { + let zip_bytes = if let Some(b64) = source.zip_file_b64 { use base64::Engine; - let zip_bytes = base64::engine::general_purpose::STANDARD - .decode(b64) - .map_err(|e| LambdaServiceError::InvalidZipFile { - message: format!("Invalid base64 encoding: {e}"), - })?; - // Fail fast on oversized packages before any storage or - // extraction happens. - check_code_size(zip_bytes.len())?; - - let (code_path, sha256, size) = self - .store - .store_zip_code(function_name, version, &zip_bytes) - .await?; - - Ok(( - sha256, - size, - Some(Bytes::from(zip_bytes)), - Some(code_path), - None, - )) + Bytes::from( + base64::engine::general_purpose::STANDARD + .decode(b64) + .map_err(|error| LambdaServiceError::InvalidZipFile { + message: format!("Invalid base64 encoding: {error}"), + })?, + ) } else if let Some(bucket) = source.s3_bucket { - let key = source - .s3_key - .ok_or_else(|| LambdaServiceError::InvalidParameter { - message: "Code.S3Key is required when Code.S3Bucket is provided".to_owned(), - })?; - let zip_bytes = self - .code_fetcher - .fetch_code(bucket, key, source.s3_object_version) - .await - .map_err(map_s3_code_fetch_error)?; - // Fail fast on oversized packages before any storage or - // extraction happens. - check_code_size(zip_bytes.len())?; - - let (code_path, sha256, size) = self - .store - .store_zip_code(function_name, version, &zip_bytes) - .await?; - - Ok((sha256, size, Some(zip_bytes), Some(code_path), None)) - } else if let Some(uri) = source.image_uri { - let sha256 = compute_sha256(uri.as_bytes()); - Ok((sha256, 0, None, None, Some(uri.to_owned()))) + let key = source.s3_key.ok_or_else(|| { + invalid_parameter("Code.S3Key is required when Code.S3Bucket is provided") + })?; + tokio::time::timeout( + CODE_FETCH_TIMEOUT, + self.code_fetcher + .fetch_code(bucket, key, source.s3_object_version), + ) + .await + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "Code source fetch deadline exceeded".into(), + })? + .map_err(map_s3_code_fetch_error)? + } else if let Some(reference) = source.image_uri { + return reference + .parse() + .map(ProcessedCode::Image) + .map_err(|_| invalid_parameter("Invalid ImageUri")); } else { - // No code provided - use empty hash. - let sha256 = compute_sha256(b""); - Ok((sha256, 0, None, None, None)) - } + return Err(invalid_parameter("Deployment code is required")); + }; + check_code_size(zip_bytes.len())?; + self.store + .stage_zip_code(function_name, version, &zip_bytes) + .await + .map(ProcessedCode::Zip) } /// Build a `FunctionConfiguration` from internal records. @@ -3025,6 +3088,112 @@ fn map_s3_code_fetch_error(err: S3CodeFetchError) -> LambdaServiceError { mod tests { use super::*; + #[derive(Debug)] + struct DelayedFetcher { + started: tokio::sync::Notify, + release: tokio::sync::Notify, + } + + #[async_trait::async_trait] + impl S3CodeFetcher for DelayedFetcher { + async fn fetch_code( + &self, + _bucket: &str, + _key: &str, + _version: Option<&str>, + ) -> Result { + self.started.notify_one(); + self.release.notified().await; + Ok(Bytes::from(crate::test_zip::minimal_zip( + b"conflicting-update", + ))) + } + } + + #[tokio::test(start_paused = true)] + async fn test_should_bound_a_stalled_code_fetch_without_creating_an_artifact() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let fetcher = Arc::new(DelayedFetcher { + started: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher); + let started = tokio::time::Instant::now(); + assert!(matches!( + provider + .create_function(s3_code_input("fetch-timeout")) + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + assert!(started.elapsed() >= CODE_FETCH_TIMEOUT); + assert!(provider.store.is_empty()); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_staged_code_after_a_real_revision_compare_and_swap_conflict() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let fetcher = Arc::new(DelayedFetcher { + started: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let provider = Arc::new( + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher.clone()), + ); + provider + .create_function(sample_create_input("conflict")) + .await?; + let original = provider.get_record("conflict")?.latest; + let updating = Arc::clone(&provider); + let task = tokio::spawn(async move { + updating + .update_function_code( + "conflict", + UpdateFunctionCodeInput { + s3_bucket: Some("code-bucket".into()), + s3_key: Some("new.zip".into()), + ..UpdateFunctionCodeInput::default() + }, + ) + .await + }); + let _cancel = crate::work::CancelOnDrop(task.abort_handle()); + tokio::time::timeout(Duration::from_secs(5), fetcher.started.notified()).await?; + provider.update_function_configuration( + "conflict", + &UpdateFunctionConfigurationInput { + description: Some("newer configuration".into()), + ..UpdateFunctionConfigurationInput::default() + }, + )?; + fetcher.release.notify_one(); + let result = tokio::time::timeout(Duration::from_secs(5), task).await??; + assert!(matches!( + result, + Err(LambdaServiceError::ResourceConflict { .. }) + )); + let current = provider.get_record("conflict")?.latest; + assert_eq!(current.code_sha256, original.code_sha256); + assert_eq!(current.code_path, original.code_path); + assert_eq!(current.description, "newer configuration"); + let mut entries = tokio::fs::read_dir(root.path()).await?; + assert!(entries.next_entry().await?.is_some()); + assert!(entries.next_entry().await?.is_none()); + Ok(()) + } + fn test_provider() -> RustackLambda { let tmp = tempfile::tempdir().unwrap(); let store = FunctionStore::new(tmp.path()); @@ -3049,6 +3218,44 @@ mod tests { } } + #[tokio::test] + async fn test_should_reject_invalid_settings_before_fetching_or_staging_code() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()); + let mut input = sample_create_input("invalid-env"); + input.environment = Some(rustack_lambda_model::types::Environment { + variables: Some(HashMap::from([("AWS_REGION".into(), "us-east-1".into())])), + }); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut input = sample_create_input("invalid-image-config"); + input.image_config = Some(rustack_lambda_model::types::ImageConfig::default()); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut input = s3_code_input("invalid-s3-settings"); + input.timeout = Some(901); + // The default fetcher is unavailable; InvalidParameter proves it was not called. + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(provider.store.is_empty()); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + #[tokio::test] async fn test_should_preserve_invoked_alias_separately_from_resolved_version() -> Result<(), LambdaServiceError> { @@ -3073,7 +3280,9 @@ mod tests { (Some("1"), "1"), ] { let version = resolve_version(&record, qualifier)?; - let request = provider.build_invoke_request(&record, version, qualifier, b"{}"); + let request = provider + .build_invoke_request(&record, version, qualifier, b"{}") + .await?; let expected_arn = qualifier.map_or_else(|| record.arn.clone(), |q| format!("{}:{q}", record.arn)); assert_eq!(request.function_arn, expected_arn); @@ -3094,12 +3303,22 @@ mod tests { let record = provider.get_record("captured")?; let mut version = record.latest.clone(); version.ephemeral_storage_size = 2048; + version.package_type = "Image".into(); + version.runtime = None; + version.handler = None; + version.code_path = None; + version.zip_bytes = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + version.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + version.resolved_image = Some(resolved); version.image_config = Some(rustack_lambda_model::types::ImageConfig { entry_point: Some(vec!["/custom/entrypoint".into()]), command: Some(vec![]), working_directory: Some("/function".into()), }); - let request = provider.build_invoke_request(&record, &version, None, b"{\"value\":1}"); + let request = provider + .build_invoke_request(&record, &version, None, b"{\"value\":1}") + .await?; version.ephemeral_storage_size = 4096; version.image_config = None; assert_eq!(request.ephemeral_storage_mb, 2048); @@ -3119,6 +3338,47 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_should_reject_unresolved_or_inconsistent_artifacts_before_work_admission() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("artifact-capture")) + .await?; + let record = provider.get_record("artifact-capture")?; + let mut invalid_zip = record.latest.clone(); + invalid_zip.code_sha256 = "different".into(); + assert!(matches!( + provider + .build_invoke_request(&record, &invalid_zip, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let mut image = record.latest.clone(); + image.package_type = "Image".into(); + image.runtime = None; + image.handler = None; + image.zip_bytes = None; + image.code_path = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + image.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + assert!(matches!( + provider + .build_invoke_request(&record, &image, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + image.resolved_image = Some(resolved); + image.architectures = vec!["arm64".into()]; + assert!(matches!( + provider + .build_invoke_request(&record, &image, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + Ok(()) + } + #[tokio::test] async fn test_should_create_and_get_function() { let provider = test_provider(); diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index d21dc523..3c7c928d 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -304,6 +304,7 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, + resolved_image: None, zip_bytes: None, state: "Active".to_owned(), last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index d0b1b220..79609a26 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -6,6 +6,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, + fmt, io::ErrorKind, path::{Path, PathBuf}, }; @@ -20,7 +21,11 @@ use rustack_lambda_model::types::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::{error::LambdaServiceError, identity::FunctionResourceId}; +use crate::{ + artifact::{ResolvedImage, ValidatedArtifact}, + error::LambdaServiceError, + identity::FunctionResourceId, +}; /// In-memory store for Lambda functions. #[derive(Debug)] @@ -30,36 +35,47 @@ pub struct FunctionStore { /// Root directory for storing extracted code. code_dir: PathBuf, /// Own completed artifacts until the store is dropped; in-flight revisions remain readable. - artifacts: DashMap, + artifacts: DashMap, staging: std::sync::Arc, } -/// Owns a completed artifact and its resolved filesystem boundary. -#[derive(Debug)] -struct Artifact { - directory: Option, - root: PathBuf, - extracted: PathBuf, +/// Rolls back a newly registered artifact unless its record mutation commits. +pub(crate) struct StagedArtifact<'a> { + store: &'a FunctionStore, + artifact: ValidatedArtifact, + committed: bool, } -impl Drop for Artifact { - #[allow(clippy::disallowed_methods)] // Drop has no async context; cleanup is bounded to a store-owned directory and failures are reported. - fn drop(&mut self) { - let Some(directory) = self.directory.take() else { - return; - }; - let path = directory.keep(); - let safe = std::fs::canonicalize(&self.root).is_ok_and(|root| root == self.root) - && std::fs::symlink_metadata(&self.root) - .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()) - && std::fs::symlink_metadata(&path) - .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()); - if !safe { - tracing::error!("Refusing cleanup of replaced Lambda artifact root"); - return; +impl fmt::Debug for StagedArtifact<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StagedArtifact") + .field("artifact", &self.artifact) + .field("committed", &self.committed) + .finish() + } +} + +impl<'a> StagedArtifact<'a> { + fn new(store: &'a FunctionStore, artifact: ValidatedArtifact) -> Self { + Self { + store, + artifact, + committed: false, } - if let Err(error) = std::fs::remove_dir_all(path) { - tracing::error!(%error, "Lambda artifact cleanup failed"); + } + pub(crate) fn artifact(&self) -> &ValidatedArtifact { + &self.artifact + } + pub(crate) fn commit(mut self) { + self.committed = true; + } +} + +impl Drop for StagedArtifact<'_> { + fn drop(&mut self) { + if !self.committed { + self.store + .discard_staged_artifact(self.artifact.code_root()); } } } @@ -140,6 +156,8 @@ pub struct VersionRecord { pub code_path: Option, /// Container image URI. pub image_uri: Option, + /// Immutable inspected image metadata, absent until deployment resolution. + pub resolved_image: Option, /// Raw zip bytes. pub zip_bytes: Option, /// Function state (`Active`, `Pending`, etc.). @@ -658,6 +676,9 @@ pub struct VersionRecordSnapshot { pub package_type: String, /// Container image URI. pub image_uri: Option, + /// Immutable image identity; older metadata-only snapshots have none. + #[serde(default)] + pub resolved_image: Option, /// Raw zip bytes encoded as standard base64. pub zip_bytes_base64: Option, /// Function state. @@ -717,29 +738,35 @@ impl FunctionRecordSnapshot { async fn into_record( self, store: &FunctionStore, - ) -> Result { + ) -> Result<(FunctionRecord, Vec>), LambdaServiceError> { let name = self.name; - let latest = self.latest.into_record(store, &name).await?; + let (latest, pending) = self.latest.into_record(store, &name).await?; + let mut artifacts: Vec<_> = pending.into_iter().collect(); let mut versions = BTreeMap::new(); for (version, record) in self.versions { - versions.insert(version, record.into_record(store, &name).await?); + let (record, artifact) = record.into_record(store, &name).await?; + artifacts.extend(artifact); + versions.insert(version, record); } - Ok(FunctionRecord { - resource_id: self.resource_id, - name, - arn: self.arn, - latest, - versions, - next_version: self.next_version, - aliases: self.aliases, - policy: self.policy, - tags: self.tags, - code_signing_config_arn: self.code_signing_config_arn, - url_config: self.url_config, - reserved_concurrent_executions: self.reserved_concurrent_executions, - event_invoke_configs: self.event_invoke_configs, - created_at: self.created_at, - }) + Ok(( + FunctionRecord { + resource_id: self.resource_id, + name, + arn: self.arn, + latest, + versions, + next_version: self.next_version, + aliases: self.aliases, + policy: self.policy, + tags: self.tags, + code_signing_config_arn: self.code_signing_config_arn, + url_config: self.url_config, + reserved_concurrent_executions: self.reserved_concurrent_executions, + event_invoke_configs: self.event_invoke_configs, + created_at: self.created_at, + }, + artifacts, + )) } } @@ -756,6 +783,7 @@ impl VersionRecordSnapshot { environment: record.environment, package_type: record.package_type, image_uri: record.image_uri, + resolved_image: record.resolved_image, zip_bytes_base64: record .zip_bytes .as_ref() @@ -777,11 +805,12 @@ impl VersionRecordSnapshot { } } - async fn into_record( + async fn into_record<'a>( self, - store: &FunctionStore, + store: &'a FunctionStore, function_name: &str, - ) -> Result { + ) -> Result<(VersionRecord, Option>), LambdaServiceError> { + crate::function_config::validate_snapshot_settings(&self)?; let zip_bytes = match self.zip_bytes_base64 { Some(encoded) => { let decoded = STANDARD.decode(encoded.as_bytes()).map_err(|source| { @@ -793,43 +822,51 @@ impl VersionRecordSnapshot { } None => None, }; - let code_path = if let Some(bytes) = zip_bytes.as_ref() { - let (path, _, _) = store - .store_zip_code(function_name, &self.version, bytes.as_ref()) - .await?; - Some(path) + let artifact = if let Some(bytes) = zip_bytes.as_ref() { + Some( + store + .stage_zip_code(function_name, &self.version, bytes) + .await?, + ) } else { None }; - - Ok(VersionRecord { - version: self.version, - runtime: self.runtime, - handler: self.handler, - role: self.role, - description: self.description, - timeout: self.timeout, - memory_size: self.memory_size, - environment: self.environment, - package_type: self.package_type, - code_path, - image_uri: self.image_uri, - zip_bytes, - state: self.state, - last_modified: self.last_modified, - architectures: self.architectures, - ephemeral_storage_size: self.ephemeral_storage_size, - code_sha256: self.code_sha256, - code_size: self.code_size, - revision_id: self.revision_id, - layers: self.layers, - vpc_config: self.vpc_config, - dead_letter_config: self.dead_letter_config, - tracing_config: self.tracing_config, - image_config: self.image_config, - logging_config: self.logging_config, - snap_start: self.snap_start, - }) + let code_path = artifact + .as_ref() + .map(|artifact| artifact.artifact().code_root().to_path_buf()); + + Ok(( + VersionRecord { + version: self.version, + runtime: self.runtime, + handler: self.handler, + role: self.role, + description: self.description, + timeout: self.timeout, + memory_size: self.memory_size, + environment: self.environment, + package_type: self.package_type, + code_path, + image_uri: self.image_uri, + resolved_image: self.resolved_image, + zip_bytes, + state: self.state, + last_modified: self.last_modified, + architectures: self.architectures, + ephemeral_storage_size: self.ephemeral_storage_size, + code_sha256: self.code_sha256, + code_size: self.code_size, + revision_id: self.revision_id, + layers: self.layers, + vpc_config: self.vpc_config, + dead_letter_config: self.dead_letter_config, + tracing_config: self.tracing_config, + image_config: self.image_config, + logging_config: self.logging_config, + snap_start: self.snap_start, + }, + artifact, + )) } } @@ -877,32 +914,25 @@ impl FunctionStore { } for version in std::iter::once(&record.latest).chain(record.versions.values()) { crate::resolver::Qualifier::parse(&version.version)?; - if !(1..=900).contains(&version.timeout) - || !(128..=10240).contains(&version.memory_size) - || version.architectures.len() != 1 - || version - .architectures - .iter() - .any(|architecture| architecture != "arm64" && architecture != "x86_64") - || version.environment.len() > 128 - || version - .environment - .iter() - .any(|(key, value)| key.len() > 256 || value.len() > 4096) - { - return Err(invalid("Invalid execution configuration")); - } + crate::function_config::validate_execution_settings(version)?; if let Some(path) = &version.code_path { - if !self.artifacts.contains_key(path) { - return Err(invalid("Code path is not an owned immutable artifact")); - } - } - if let Some(bytes) = &version.zip_bytes { - if version.code_sha256 != compute_sha256(bytes) - || version.code_size != bytes.len() as u64 + let artifact = self + .artifacts + .get(path) + .ok_or_else(|| invalid("Code path is not an owned immutable artifact"))?; + let zip_matches = version.zip_bytes.as_ref().is_some_and(|bytes| { + bytes.len() == artifact.zip_bytes().len() + && (bytes.as_ptr() == artifact.zip_bytes().as_ptr() + || bytes == artifact.zip_bytes()) + }); + if !zip_matches + || version.code_sha256 != artifact.sha256() + || version.code_size != artifact.zip_bytes().len() as u64 { return Err(invalid("Deployment package metadata mismatch")); } + } else if version.zip_bytes.is_some() { + return Err(invalid("ZIP bytes have no owned extracted artifact")); } } Ok(()) @@ -1062,6 +1092,7 @@ impl FunctionStore { self.validate_root().await?; let mut staged = BTreeMap::new(); let mut identities = HashSet::new(); + let mut artifacts = Vec::new(); for function in snapshot.functions { if !identities.insert(function.resource_id) { return Err(LambdaServiceError::InvalidParameter { @@ -1079,7 +1110,8 @@ impl FunctionStore { crate::resolver::Qualifier::parse(alias)?; crate::resolver::Qualifier::parse(&record.function_version)?; } - let record = function.into_record(self).await?; + let (record, allocations) = function.into_record(self).await?; + artifacts.extend(allocations); self.validate_record(&record)?; if staged.insert(record.name.clone(), record).is_some() { return Err(LambdaServiceError::InvalidParameter { @@ -1091,6 +1123,9 @@ impl FunctionStore { for (name, record) in staged { self.functions.insert(name, record); } + for artifact in artifacts { + artifact.commit(); + } Ok(()) } @@ -1115,7 +1150,7 @@ impl FunctionStore { &self, function_name: &str, version: &str, - zip_bytes: &[u8], + zip_bytes: &Bytes, ) -> Result<(PathBuf, String, u64), LambdaServiceError> { crate::resolver::FunctionName::parse(function_name)?; crate::resolver::Qualifier::parse(version)?; @@ -1131,10 +1166,10 @@ impl FunctionStore { message: "ZIP staging capacity exhausted".into(), })?; let root = self.code_dir.clone(); - let bytes_owned = zip_bytes.to_vec(); + let bytes_owned = zip_bytes.clone(); let task = tokio::task::spawn_blocking(move || { let _permit = permit; - stage_zip(&root, &bytes_owned) + stage_zip(&root, bytes_owned) }); let artifact = tokio::time::timeout(std::time::Duration::from_secs(30), task) .await @@ -1144,15 +1179,35 @@ impl FunctionStore { .map_err(|e| LambdaServiceError::Internal { message: format!("ZIP extraction task failed: {e}"), })??; - let extracted = artifact.extracted.clone(); + let extracted = artifact.code_root().to_path_buf(); + let sha256 = artifact.sha256().to_owned(); + let code_size = artifact.zip_bytes().len() as u64; self.artifacts.insert(extracted.clone(), artifact); - let sha256 = compute_sha256(zip_bytes); - let code_size = zip_bytes.len() as u64; - Ok((extracted, sha256, code_size)) } + pub(crate) async fn stage_zip_code( + &self, + function_name: &str, + version: &str, + bytes: &Bytes, + ) -> Result, LambdaServiceError> { + let (path, _, _) = self.store_zip_code(function_name, version, bytes).await?; + let artifact = self + .artifacts + .get(&path) + .map(|artifact| artifact.value().clone()) + .ok_or_else(|| LambdaServiceError::Internal { + message: "Newly staged artifact is missing".into(), + })?; + Ok(StagedArtifact::new(self, artifact)) + } + + fn discard_staged_artifact(&self, path: &Path) { + self.artifacts.remove(path); + } + /// Validate logical retirement of a function's code. /// /// Immutable artifacts are retained until store drop so published/in-flight @@ -1168,7 +1223,7 @@ impl FunctionStore { let owned_root = self .artifacts .get(path) - .map(|artifact| artifact.root.clone()) + .map(|artifact| artifact.storage_root().to_path_buf()) .ok_or_else(|| LambdaServiceError::InvalidParameter { message: "Unowned Lambda artifact".into(), })?; @@ -1226,6 +1281,38 @@ impl FunctionStore { } } + /// Issue a shared artifact lease after bounded ownership/filesystem validation. + /// + /// # Errors + /// Rejects unowned or replaced paths, filesystem failures and validation timeout. + /// + /// ```no_run + /// # async fn capture(store: &rustack_lambda_core::storage::FunctionStore, path: &std::path::Path) + /// # -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let artifact = store.capture_artifact(path).await?; + /// assert_eq!(artifact.code_root(), path); + /// # Ok(()) } + /// ``` + pub async fn capture_artifact( + &self, + path: &Path, + ) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.validate_artifact(path), + ) + .await + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "Artifact validation deadline exceeded".into(), + })??; + self.artifacts + .get(path) + .map(|artifact| artifact.value().clone()) + .ok_or_else(|| LambdaServiceError::InvalidParameter { + message: "Unowned Lambda artifact".into(), + }) + } + /// Reject a symlink at the operator-selected artifact root. pub async fn validate_root(&self) -> Result<(), LambdaServiceError> { match tokio::fs::symlink_metadata(&self.code_dir).await { @@ -1245,7 +1332,10 @@ impl FunctionStore { /// Stage a package in a freshly created private directory on a blocking worker. #[allow(clippy::disallowed_methods, clippy::disallowed_types)] // Synchronous filesystem work is isolated to spawn_blocking. -fn stage_zip(root: &Path, bytes: &[u8]) -> Result { +pub(crate) fn stage_zip( + root: &Path, + bytes: Bytes, +) -> Result { let io_error = |source| LambdaServiceError::ArtifactIo { source }; std::fs::create_dir_all(root).map_err(io_error)?; if std::fs::symlink_metadata(root) @@ -1265,18 +1355,16 @@ fn stage_zip(root: &Path, bytes: &[u8]) -> Result let extracted = artifact.path().join("extracted"); let result = (|| { std::fs::create_dir(&extracted).map_err(io_error)?; - std::fs::write(artifact.path().join("code.zip"), bytes).map_err(io_error)?; - extract_zip(bytes, &extracted, MAX_EXTRACTED_SIZE) + std::fs::write(artifact.path().join("code.zip"), &bytes).map_err(io_error)?; + extract_zip(&bytes, &extracted, MAX_EXTRACTED_SIZE) })(); if let Err(error) = result { artifact.close().map_err(io_error)?; return Err(error); } - Ok(Artifact { - directory: Some(artifact), - root, - extracted, - }) + Ok(ValidatedArtifact::from_staged( + artifact, root, extracted, bytes, + )) } /// Maximum extracted deployment package size (250 MB, mirroring the AWS @@ -1417,6 +1505,138 @@ pub fn compute_sha256(data: &[u8]) -> String { mod tests { use super::*; + #[tokio::test] + async fn test_should_release_unpublished_artifacts_and_preserve_committed_code() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"code")); + let (committed, _, _) = store.store_zip_code("committed", "$LATEST", &bytes).await?; + let pending = store.stage_zip_code("pending", "$LATEST", &bytes).await?; + let pending_path = pending.artifact().code_root().to_path_buf(); + assert_eq!(store.artifacts.len(), 2); + drop(pending); + assert_eq!(store.artifacts.len(), 1); + assert!(tokio::fs::try_exists(committed).await?); + assert!(!tokio::fs::try_exists(pending_path).await?); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_unpublished_code_when_its_owner_future_is_aborted() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = std::sync::Arc::new(FunctionStore::new(root.path())); + let owner = std::sync::Arc::clone(&store); + let (started, receiver) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"pending")); + let staged = owner.stage_zip_code("cancelled", "$LATEST", &bytes).await?; + let _ = started.send(staged.artifact().code_root().to_path_buf()); + std::future::pending::<()>().await; + staged.commit(); + Ok::<(), LambdaServiceError>(()) + }); + let _cancel = crate::work::CancelOnDrop(task.abort_handle()); + let path = tokio::time::timeout(std::time::Duration::from_secs(5), receiver).await??; + task.abort(); + assert!(task.await.is_err_and(|error| error.is_cancelled())); + assert!(store.artifacts.is_empty()); + assert!(!tokio::fs::try_exists(path).await?); + Ok(()) + } + + #[tokio::test] + async fn test_should_rollback_snapshot_artifacts_when_later_validation_rejects_restore() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + store.insert(sample_record("live"))?; + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"unpublished")); + let mut candidate = sample_record("candidate"); + candidate.latest.code_sha256 = "mismatched-snapshot-hash".into(); + candidate.latest.code_size = bytes.len() as u64; + candidate.latest.zip_bytes = Some(bytes); + let snapshot = FunctionStoreSnapshot { + functions: vec![FunctionRecordSnapshot::from_record(candidate)], + }; + assert!(matches!( + store.import_snapshot(snapshot).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.contains("live")?); + assert_eq!(store.artifacts.len(), 0); + assert!( + tokio::fs::read_dir(root.path()) + .await? + .next_entry() + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_validate_snapshot_settings_before_decoding_or_staging_its_archive() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let mut snapshot = FunctionRecordSnapshot::from_record(sample_record("invalid-settings")); + snapshot.latest.timeout = 0; + snapshot.latest.zip_bytes_base64 = Some("invalid-base64".into()); + let result = store + .import_snapshot(FunctionStoreSnapshot { + functions: vec![snapshot], + }) + .await; + assert!(matches!( + result, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(store.is_empty()); + assert!(store.artifacts.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn test_should_keep_code_alive_until_the_last_invocation_lease_releases_it() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + let bytes = Bytes::from(crate::test_zip::minimal_zip(b"owned-code")); + let (path, sha256, size) = store.store_zip_code("leased", "$LATEST", &bytes).await?; + let lease = store.capture_artifact(&path).await?; + assert_eq!(lease.zip_bytes().as_ptr(), bytes.as_ptr()); + assert_eq!(lease.sha256(), sha256); + assert_eq!(lease.zip_bytes().len() as u64, size); + drop(store); + assert!(tokio::fs::metadata(&path).await?.is_dir()); + assert_eq!( + tokio::fs::read(path.join("fixture.txt")).await?, + b"owned-code" + ); + let retained = lease.clone(); + drop(lease); + assert!(tokio::fs::metadata(&path).await?.is_dir()); + drop(retained); + assert!( + matches!(tokio::fs::metadata(path).await, Err(error) if error.kind() == ErrorKind::NotFound) + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_reject_unowned_code_when_issuing_an_artifact_lease() + -> Result<(), Box> { + let root = tempfile::tempdir()?; + let store = FunctionStore::new(root.path()); + assert!(matches!( + store.capture_artifact(root.path()).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + Ok(()) + } + #[tokio::test] async fn test_should_reject_hostile_names_at_every_store_boundary() { let root = tempfile::tempdir().unwrap(); @@ -1437,7 +1657,12 @@ mod tests { assert!(store.contains(name).is_err()); assert!(store.remove(name).is_err()); assert!(store.update(name, |_| ()).is_err()); - assert!(store.store_zip_code(name, "$LATEST", &zip).await.is_err()); + assert!( + store + .store_zip_code(name, "$LATEST", &Bytes::copy_from_slice(&zip)) + .await + .is_err() + ); assert!(store.cleanup_code(name).await.is_err()); } assert!(store.is_empty()); @@ -1457,15 +1682,25 @@ mod tests { let root = tempfile::tempdir().unwrap(); let store = FunctionStore::new(root.path()); let (a, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); assert!(matches!( - store.store_zip_code("safe", "$LATEST", b"bad ZIP").await, + store + .store_zip_code("safe", "$LATEST", &Bytes::from_static(b"bad ZIP")) + .await, Err(LambdaServiceError::InvalidZipFile { .. }) )); let (b, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"B")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"B")), + ) .await .unwrap(); assert_ne!(a, b); @@ -1483,7 +1718,11 @@ mod tests { let root = tempfile::tempdir().unwrap(); let store = FunctionStore::new(root.path()); let (old, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"old")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"old")), + ) .await .unwrap(); let mut corrupt = crate::test_zip::minimal_zip(b"crc-marker"); @@ -1494,7 +1733,7 @@ mod tests { corrupt[position] ^= 1; assert!( store - .store_zip_code("safe", "$LATEST", &corrupt) + .store_zip_code("safe", "$LATEST", &Bytes::copy_from_slice(&corrupt)) .await .is_err() ); @@ -1510,7 +1749,7 @@ mod tests { let conflict = writer.finish().unwrap().into_inner(); assert!( store - .store_zip_code("safe", "$LATEST", &conflict) + .store_zip_code("safe", "$LATEST", &Bytes::copy_from_slice(&conflict)) .await .is_err() ); @@ -1560,7 +1799,11 @@ mod tests { let store = FunctionStore::new(&link); assert!( store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")) + ) .await .is_err() ); @@ -1573,7 +1816,11 @@ mod tests { ); let store = FunctionStore::new(root.path().join("owned")); let (artifact, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); tokio::fs::remove_dir_all(&artifact).await.unwrap(); @@ -1584,7 +1831,11 @@ mod tests { let managed = root.path().join("replaceable"); let store = FunctionStore::new(&managed); let (artifact, _, _) = store - .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"A")), + ) .await .unwrap(); let artifact_id = artifact.parent().unwrap().file_name().unwrap(); @@ -1624,6 +1875,7 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, + resolved_image: None, zip_bytes: None, state: "Active".to_owned(), last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), @@ -1945,7 +2197,7 @@ mod tests { let zip_data = crate::test_zip::minimal_zip(b"fixture"); let (dir, sha256, size) = store - .store_zip_code("test-func", "$LATEST", &zip_data) + .store_zip_code("test-func", "$LATEST", &Bytes::copy_from_slice(&zip_data)) .await .unwrap(); @@ -1994,7 +2246,7 @@ mod tests { } let (dir, _sha, _size) = store - .store_zip_code("real-func", "$LATEST", &buf) + .store_zip_code("real-func", "$LATEST", &Bytes::copy_from_slice(&buf)) .await .unwrap(); @@ -2035,7 +2287,7 @@ mod tests { } let err = store - .store_zip_code("evil", "$LATEST", &buf) + .store_zip_code("evil", "$LATEST", &Bytes::copy_from_slice(&buf)) .await .unwrap_err(); assert!(matches!(err, LambdaServiceError::InvalidZipFile { .. })); diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs index fa7cad54..cda05c8b 100644 --- a/crates/rustack-lambda-core/src/work.rs +++ b/crates/rustack-lambda-core/src/work.rs @@ -224,13 +224,10 @@ async fn execute( #[cfg(test)] mod tests { - use std::collections::HashMap; - use async_trait::async_trait; - use bytes::Bytes; use super::*; - use crate::executor::{ExecutorError, PackageType}; + use crate::executor::ExecutorError; #[derive(Debug)] struct WaitingExecutor; @@ -242,27 +239,10 @@ mod tests { async fn shutdown(&self) {} } fn request(name: &str) -> InvokeRequest { - InvokeRequest { - resource_id: crate::identity::FunctionResourceId::default(), - revision_id: "test-revision".into(), - image_config: None, - ephemeral_storage_mb: 512, - function_name: name.into(), - function_arn: "arn".into(), - qualifier: "$LATEST".into(), - runtime: None, - handler: None, - architectures: vec![], - package_type: PackageType::Zip, - code_root: None, - code_zip: None, - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_mins(1), - memory_mb: 128, - payload: Bytes::new(), - capture_logs: false, - } + let mut request = InvokeRequest::fixture(); + request.function_name = name.into(); + request.timeout = Duration::from_mins(1); + request } #[tokio::test] async fn test_should_bound_event_set_before_accepting_and_release_on_cancel() { diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 18126d44..8187d059 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -60,7 +60,7 @@ Tag movement never changes deployed code. Published versions are immutable; alia Map x86_64 to linux/amd64 and arm64 to linux/arm64. Guarantee native architecture only. Reject cross-architecture calls unless the operator explicitly enables emulation; do not promise emulation performance/parity. Registry manifest lists select exactly one matching platform and pin its child manifest digest. Inspect local image architecture rather than trusting the declaration. -Inherit Dockerfile ENTRYPOINT, CMD and WORKDIR, then override only explicitly supplied ImageConfig fields. Missing and explicitly empty arrays differ. WorkingDirectory is an image-internal absolute path≤1,000 bytes, not a host path. Each argv list has≤1,500 elements, each≤4,096 bytes and total≤64 KiB locally. Pass JSON argv directly, never sh -c. Do not force CMD=bootstrap. P0 must prove official Lambda images enter their RIC path when AWS_LAMBDA_RUNTIME_API is set. +Inherit Dockerfile ENTRYPOINT, CMD and WORKDIR, then override only explicitly supplied ImageConfig fields. Missing and explicitly empty arrays differ. WorkingDirectory is an image-internal absolute path≤1,000 bytes, not a host path. Each argv list has≤1,500 elements, each≤4,096 bytes and the combined argv bytes≤64 KiB locally. Pass JSON argv directly, never sh -c. Do not force CMD=bootstrap. P0 must prove official Lambda images enter their RIC path when AWS_LAMBDA_RUNTIME_API is set. ### 2.4 Invocation capture and resource identity @@ -70,11 +70,34 @@ The provider captures the resource identity, resolved revision, complete ImageCo P1 task1.1 lands this capture contract before the artifact-enum and cooperative-cancellation consumers. Its regression tests cover delete/recreate isolation, snapshot identity preservation and rejection, alias versus resolved version, configuration capture and redacted diagnostics. These foundation checks alone do not close M0. +### 2.5 Captured artifact representation + +InvokeRequest carries one DeploymentArtifact instead of independent package-kind/path/archive/image options. A Zip variant owns a cloneable ValidatedArtifact lease: its extracted root, raw Bytes and content hash refer to the same successfully staged archive. Only FunctionStore issues leases after ownership and filesystem validation. The last store/invocation lease owns directory cleanup, so dropping the store cannot remove code still used by an invocation. Staging shares Bytes instead of retaining a second archive allocation. + +Validate each revision's normalized settings before downloading or staging its code. A code-source fetch has a hard 30-second deadline before staging; a paused-clock regression verifies stalled fetches leave no artifact. A newly registered ZIP belongs to an unpublished-artifact guard until the resource insertion or revision compare-and-swap succeeds. Failure, conflict or cancellation drops the guard, unregisters that new artifact and releases its directory; committed artifacts are unaffected. Snapshot restoration holds all new guards until the entire candidate store validates and commits. A rejected restore releases its new code without changing live resources. Hashing runs on the staging worker, not under a store mutation lock. Native's child supervisor retains an artifact lease until the child exit is reaped, including after the requesting store is dropped. + +```text +Provider / restore Store + staging worker Artifact ownership + │ 1. validate settings │ │ + │ 2. stage shared Bytes ─────────▶│ private directory + digest │ + │◀── unpublished guard ──────────│─────────────────────────────────▶│ + │ 3. validate record + insert/CAS │ │ + │── failure/conflict/cancel ──────┼── drop guard → unregister → remove │ + │── success → commit guard ──────▶│ store retains artifact lease │ + │ 4. capture invocation ─────────▶│ clone validated lease ────────────▶│ + │ │ native child also retains lease │ + │ 5. store/request release │ child exit → last lease → remove │ +``` + +An Image variant carries ResolvedImage, never a mutable tag alone. Its validated metadata contains the requested ImageReference, daemon SHA-256 content ID, optional repository/manifest reference, Linux architecture, inspection SHA-256 and inspected size. Digests use exactly 64 lowercase hexadecimal digits; inspected image size is bounded to 10 GiB locally. ImageReference accepts bounded Docker repository components, tags, SHA-256 digest suffixes and explicit DNS/IPv4/bracketed-IPv6 registry ports; URLs, userinfo, controls, invalid separators and empty components reject. Repository names are≤255 bytes and the whole reference≤1,024 bytes. Registry DNS labels are≤63 bytes; explicit ports use at most five decimal digits in1..65,535, an advertised local restriction. Bracketed IPv6 uses hexadecimal/colon syntax, excluding zone identifiers and dotted mapped forms. Follow the [distribution reference grammar](https://github.com/distribution/reference/blob/main/regexp.go), implemented with the pinned winnow parser rather than shell parsing. References are parsed again on deserialization. Missing resolution metadata cannot become an executable image; an existing metadata-only image record fails invocation explicitly until its deployment is resolved. + +Artifact replacement does not change public AWS wire shapes. Native and Squib accept only Zip and consume its lease; Docker accepts Image in M0 and adds Zip in M4. The resolved architecture is a closed x86_64/arm64 enum, not an unchecked list. Capture validates numeric settings and full image overrides before work admission. No per-invoke image inspection or pulling is introduced by this contract. + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. -Inject `AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001` and revision-derived Lambda name/version/memory/region/log/task-root metadata. Reject reserved-variable overrides. User environment totals≤4 KiB with Lambda key grammar. Inject only explicitly configured local test credentials; do not inherit host AWS secret variables. +Inject `AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001` and revision-derived Lambda name/version/memory/region/log/task-root metadata. Reject reserved-variable overrides. User environment totals≤4 KiB with Lambda key grammar. Reject NUL values and the [reserved Lambda environment keys](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime), plus the local AWS_LAMBDA_FUNCTION_TIMEOUT injection. PATH, TZ and AWS_ENDPOINT_URL remain configurable. Inject only explicitly configured local test credentials; do not inherit host AWS secret variables. Control frames are 4-byte big-endian length plus CBOR envelope, protocol version 1. Maximum frame is 6 MiB+64 KiB, with metadata≤64 KiB and payload encoded as CBOR bytes rather than base64. Message variants are Hello, Ready, BootstrapReady, Invoke, InvocationResult, InitError, Cancel, Close and Ack. Include environment UUID and invocation-attempt UUID. Hello advertises protocol version and architecture; mismatch retires the environment. EOF, malformed frames or attach failure invalidate it. Never reconnect and silently replay an invocation. Relay stdout carries frames only; stderr carries bounded redacted diagnostics. Collect function stdout/stderr separately from its own container. Enforce an independent subscriber filter that rejects all Bollard events more verbose than INFO regardless of RUST_LOG; the selected dependency logs HTTP request/response bodies and trailing attach bytes at trace/debug. Test secret-bearing create requests and truncated attach frames under RUST_LOG=trace. Application-level redacted Debug alone is insufficient. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index f4667d29..33da02f9 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -140,7 +140,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| 1.1 | In progress | Captured resource identity, requested qualifier, ImageConfig/ephemeral propagation and diagnostic redaction are implemented and independently reviewed. Standard workspace checks, 154 Lambda core tests, two doctests and six native/DryRun cases pass; artifact enum and cooperative cancellation remain required before task 1.1 closes. Baseline security/strict-lint gates B01/B03 remain open. | +| 1.1 | In progress | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented and independently reviewed. Standard workspace suite: 1,752 passing tests; Lambda: 203 unit tests, twelve doctests, six native/DryRun checks. ExecutionRevision ownership and cooperative cancellation remain required. B01/B03 remain open. | | 1.2–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task 1.1; P0 does not advertise Docker execution as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 99aa4cac..e0c474cf 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -31,13 +31,25 @@ The independent final recheck confirmed W07–W09 addressed, queried both daemon The independent read-only review against commit `955267f` found no new in-slice correctness, security or compatibility defect. It verified strict resource UUID deserialization, immutable lifetime identity, legacy and current snapshot behavior, duplicate identity rejection, complete configuration capture, alias metadata versus resolved version, warm-key isolation and request/result diagnostic redaction. Ten new regression tests pass within 154 Lambda core tests. This approval covers the capture foundation only; artifact enums, cooperative cancellation and Docker execution remain required in P1. Standard workspace build/test/Clippy/rustdoc and two identity doctests pass; the native target validates six native/DryRun cases (its separately gated Squib case is not execution evidence). Baseline advisory remediation B01 and strict-lint remediation B03 remain open. +## P1 task1.1 artifact slice review + +The independent review against commit `7915281` identified two implementation findings and one preflight-order documentation mismatch. All were fixed and the final independent recheck found no remaining in-slice issue. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W10 | P2; `crates/rustack-lambda-core/src/provider.rs:504`, `src/storage.rs:74` | Rejected settings could retain a staged ZIP. Shared settings preflight now runs before source IO. Unpublished-artifact guards roll back on failure/cancellation/CAS conflict, and restore commits guards only after the complete candidate validates | Invalid settings leave no artifacts; real revision-conflict, aborted owner and rejected restore regressions preserve committed resources | +| W11 | P3; `crates/rustack-lambda-core/src/artifact/image.rs:141` | Docker classifies uppercase first components as registry domains. Parser and cases now accept Registry/worker while rejecting uppercase repository components | Parameterized reference and serde cases cover single-label registries, DNS/IPv6 ports, tags/digests and invalid syntax | +| W12 | P3; `crates/rustack-lambda-core/src/storage.rs:813` | Snapshot settings validation followed staging despite the documented ordering. Shared settings preflight now runs before archive decoding or staging; full integrity checks still retain rollback ownership | Early-invalid-settings/invalid-base64 regression plus a separate post-staging hash-mismatch rollback regression | + +The slice also preserves extracted ZIP roots in Native warm keys and retains code in Native child supervision through exit. The standard workspace suite passes 1,752 tests (392 preexisting ignored), including 203 Lambda core tests; twelve Lambda doctests and six native/DryRun cases pass. The separately gated Squib case selected by the native Make target is not Squib execution evidence. New winnow/rstest dependency use introduces no audit finding; the same 16 baseline advisories remain B01. Strict production boundary lint debt remains B03; no ignore was added. ExecutionRevision ownership, cooperative invocation cancellation, deployment resolution and Docker remain required before P1/M0 completion. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | -| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:334`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | Strict production boundary Clippy on lambda-core reports 66 existing findings: 56 missing Errors sections, four documentation-format findings and six indexing/slicing sites. The unfiltered dependency run additionally fails existing rustack-auth/core/model findings. The new identity/request/pool code has no finding in the strict core run; standard workspace Clippy passes | P1 quality work: fix the affected production boundaries and documentation, then rerun strict checks without lint ignores; standard checks alone do not close this gate | +| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | Strict production boundary Clippy on lambda-core reports 66 existing findings: 56 missing Errors sections, four documentation-format findings and six indexing/slicing sites. The unfiltered dependency run additionally fails existing rustack-auth/core/model findings. The new identity/request/pool code has no finding in the strict core run; standard workspace Clippy passes | P1 quality work: fix the affected production boundaries and documentation, then rerun strict checks without lint ignores; standard checks alone do not close this gate | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 1448620b..1b1afbb3 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -92,6 +92,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [invocation capture](./rustack-workflows-20-lambda-runtime-design.md#24-invocation-capture-and-resource-identity). - Date: 2026-10-25. +## D13 — Own code through staging, publication and invocation cleanup + +- Alternatives: pass independent ZIP paths/bytes/image options; retain every rejected staging result until shutdown; drop extracted code with its function store. +- Decision: use a closed artifact enum, shared ZIP leases, validated immutable image metadata and a rollback guard for unpublished ZIPs. Validate normalized settings before source IO; commit new ownership only after insertion/CAS or full restore validation. +- Why: request cancellation must not delete live code or retain rejected uploads. The archive, extracted root and hash must describe the same staged data, and restoring a candidate must not leak its new artifacts on failure. Native process supervision retains its lease through child exit. +- Pinned by: [artifact representation and ownership sequence](./rustack-workflows-20-lambda-runtime-design.md#25-captured-artifact-representation). +- Date: 2026-10-25. + ## Glossary | Term | Meaning in this specification | From 93bd1ecd8dab2b78384f368ba60c68baf23763e4 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 17:18:17 -0700 Subject: [PATCH 004/106] phase 1: supervise cancellable invocation and backend cleanup Share captured ExecutionRevision values and attempt IDs, expose cancellable invocation handles, and retain bounded actor-owned execution through cleanup. Native retirement reaps before settlement; Squib owners retain startup and shutdown futures, including idle Drop. Independent review W13-W15 fixed and rechecked. Workspace build, 1767 tests, Clippy, rustdoc, 26 Lambda doctests, five real Native fixture tests and native SDK checks pass. P1/M0 remains open: Docker runtime, B01 advisory remediation and B03 strict-boundary debt are still required. --- .../rustack-lambda-core/src/executor/auto.rs | 34 +- .../rustack-lambda-core/src/executor/error.rs | 17 +- .../src/executor/instance.rs | 211 +++-- .../rustack-lambda-core/src/executor/mod.rs | 51 +- .../src/executor/native.rs | 244 ++++-- .../rustack-lambda-core/src/executor/noop.rs | 13 +- .../rustack-lambda-core/src/executor/squib.rs | 333 +++++--- .../rustack-lambda-core/src/executor/types.rs | 135 +++- crates/rustack-lambda-core/src/invocation.rs | 471 ++++++++++++ crates/rustack-lambda-core/src/lib.rs | 1 + crates/rustack-lambda-core/src/provider.rs | 271 +++++-- crates/rustack-lambda-core/src/work.rs | 724 ++++++++++++++---- ...tack-workflows-20-lambda-runtime-design.md | 30 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 12 + specs/rustack-workflows-99-decisions.md | 8 + .../echo-bootstrap/tests/hardening.rs | 159 ++++ 17 files changed, 2194 insertions(+), 524 deletions(-) create mode 100644 crates/rustack-lambda-core/src/invocation.rs diff --git a/crates/rustack-lambda-core/src/executor/auto.rs b/crates/rustack-lambda-core/src/executor/auto.rs index ae027524..9c01cfcb 100644 --- a/crates/rustack-lambda-core/src/executor/auto.rs +++ b/crates/rustack-lambda-core/src/executor/auto.rs @@ -13,7 +13,7 @@ use super::{ Executor, ExecutorError, InvokeRequest, InvokeResponse, NativeExecutor, SquibExecutor, SquibExecutorConfig, }; -use crate::artifact::DeploymentArtifact; +use crate::{artifact::DeploymentArtifact, invocation::InvocationContext}; /// Executor that chooses the concrete backend for each invocation. #[derive(Debug)] @@ -39,28 +39,43 @@ impl AutoExecutor { } fn should_use_squib(req: &InvokeRequest) -> bool { - cfg!(target_os = "macos") && matches!(req.artifact, DeploymentArtifact::Zip(_)) + cfg!(target_os = "macos") && matches!(req.revision.artifact, DeploymentArtifact::Zip(_)) } } #[async_trait] impl Executor for AutoExecutor { - async fn invoke(&self, req: InvokeRequest) -> Result { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + if Self::should_use_squib(request) { + self.squib.available(request) + } else { + self.native.available(request) + } + } + + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { if Self::should_use_squib(&req) { - self.squib.invoke(req).await + self.squib.invoke(req, context).await } else { - self.native.invoke(req).await + self.native.invoke(req, context).await } } - async fn shutdown(&self) { - self.squib.shutdown().await; - self.native.shutdown().await; + async fn shutdown(&self) -> Result<(), ExecutorError> { + let squib = self.squib.shutdown().await; + let native = self.native.shutdown().await; + squib.and(native) } } #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; use crate::artifact::{Architecture, ResolvedImage}; @@ -75,7 +90,8 @@ mod tests { #[test] fn test_should_not_use_squib_for_image_package() { let mut request = InvokeRequest::fixture(); - request.artifact = DeploymentArtifact::Image(ResolvedImage::fixture(Architecture::Arm64)); + Arc::make_mut(&mut request.revision).artifact = + DeploymentArtifact::Image(ResolvedImage::fixture(Architecture::Arm64)); assert!(!AutoExecutor::should_use_squib(&request)); } } diff --git a/crates/rustack-lambda-core/src/executor/error.rs b/crates/rustack-lambda-core/src/executor/error.rs index 78719d73..0e593dff 100644 --- a/crates/rustack-lambda-core/src/executor/error.rs +++ b/crates/rustack-lambda-core/src/executor/error.rs @@ -3,8 +3,17 @@ use crate::error::LambdaServiceError; /// Failure modes for an execution backend. -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] pub enum ExecutorError { + /// Caller or owner requested cancellation; settlement is tracked separately. + #[error("invocation cancelled")] + Cancelled, + /// Absolute invocation budget expired. + #[error("invocation deadline exceeded")] + DeadlineExceeded, + /// Cleanup could not be confirmed; ownership remains supervised. + #[error("cleanup not confirmed: {0}")] + CleanupFailed(String), /// No backend is wired up — set `LAMBDA_EXECUTOR=native` (or `docker`). #[error("execution is disabled. Set LAMBDA_EXECUTOR=native or LAMBDA_EXECUTOR=docker")] Disabled, @@ -39,6 +48,9 @@ pub enum ExecutorError { impl From for LambdaServiceError { fn from(e: ExecutorError) -> Self { match e { + ExecutorError::Cancelled | ExecutorError::DeadlineExceeded => Self::ResourceNotReady { + message: e.to_string(), + }, ExecutorError::Disabled => Self::ResourceNotReady { message: "Lambda execution is disabled; explicitly select a trusted execution \ backend" @@ -48,7 +60,8 @@ impl From for LambdaServiceError { | ExecutorError::InvalidCode(m) | ExecutorError::InitFailed(m) | ExecutorError::RuntimeExited(m) - | ExecutorError::Io(m) => Self::Internal { message: m }, + | ExecutorError::Io(m) + | ExecutorError::CleanupFailed(m) => Self::Internal { message: m }, ExecutorError::Timeout(d) => Self::Internal { message: format!("function timed out after {d:?}"), }, diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 7a6d47e8..305f15df 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -21,7 +21,7 @@ use super::{ runtime_api::{self, Job, RuntimeApiHandle, RuntimeResult}, types::{InvokeRequest, InvokeResponse}, }; -use crate::artifact::DeploymentArtifact; +use crate::{artifact::DeploymentArtifact, invocation::InvocationContext}; /// Identifier for the pool slot — a function/version pair. pub(crate) type PoolKey = (String, String); @@ -47,8 +47,11 @@ pub(crate) trait InstanceBackend: Send + Sync + std::fmt::Debug { /// Opaque handle to a backend-specific running thing (process or container). /// Drop must clean it up; the pool also calls `kill` on graceful shutdown. +#[async_trait] pub(crate) trait BackendHandleObj: Send + Sync + std::fmt::Debug { fn kill(&mut self); + fn running(&self) -> bool; + async fn wait(&mut self) -> Result<(), ExecutorError>; } /// Wrapper for object-safety + Drop. @@ -59,8 +62,15 @@ impl BackendHandle { Self(Box::new(handle)) } - pub(crate) fn kill(mut self) { + fn running(&self) -> bool { + self.0.running() + } + async fn wait(&mut self) -> Result<(), ExecutorError> { + self.0.wait().await + } + async fn terminate(mut self) -> Result<(), ExecutorError> { self.0.kill(); + self.0.wait().await } } @@ -119,101 +129,136 @@ impl InstancePool { } pub(crate) fn key(req: &InvokeRequest) -> PoolKey { - let mut environment: Vec<_> = req.environment.iter().collect(); + let mut environment: Vec<_> = req.revision.environment.iter().collect(); environment.sort(); - let code_root = match &req.artifact { + let code_root = match &req.revision.artifact { DeploymentArtifact::Zip(artifact) => Some(artifact.code_root()), DeploymentArtifact::Image(_) => None, }; let identity = format!( "{:?}", ( - (req.resource_id, &req.revision_id, &req.qualifier), - (&req.artifact, code_root), - (&req.runtime, &req.handler, req.architecture), + ( + req.revision.resource_id, + &req.revision.revision_id, + &req.revision.qualifier, + req.revision.backend, + ), + (&req.revision.artifact, code_root), + ( + &req.revision.runtime, + &req.revision.handler, + req.revision.architecture + ), environment, - (req.timeout, req.memory_mb, req.ephemeral_storage_mb), - &req.image_config, + ( + req.revision.timeout, + req.revision.memory_mb, + req.revision.ephemeral_storage_mb + ), + &req.revision.image_config, ) ); ( - req.function_name.clone(), + req.revision.function_name.clone(), crate::storage::compute_sha256(identity.as_bytes()), ) } /// Run a single invocation against an acquired (or freshly spawned) instance. - pub(crate) async fn invoke(&self, req: InvokeRequest) -> Result { + pub(crate) async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + context.check()?; let key = Self::key(&req); - let mut instance = match self.try_acquire(&key) { - Some(inst) => inst, - None => self.spawn_new(&req).await?, + let mut instance = if let Some(instance) = self.try_acquire(&key) { + instance + } else { + tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = self.spawn_new(&req) => result?, + } }; - - let request_id = uuid::Uuid::new_v4().to_string(); - let deadline = Instant::now() + req.timeout; + let Some(handler_deadline) = Instant::now().checked_add(req.revision.timeout) else { + instance.api.shutdown().await; + instance.backend.terminate().await?; + return Err(ExecutorError::InvalidCode("Invalid handler timeout".into())); + }; + let deadline = handler_deadline.min(context.deadline().into_std()); let (resp_tx, resp_rx) = oneshot::channel(); let job = Job { - request_id: request_id.clone(), + request_id: context.id().to_string(), function_arn: req.function_arn.clone(), deadline, payload: req.payload.clone(), response_tx: resp_tx, }; - instance - .api - .submit(job) - .await - .map_err(|e| ExecutorError::Io(e.to_string()))?; - - let response = async { - match instance.init_error.take() { - Some(mut init) => tokio::select! { - response = resp_rx => response, - error = &mut init => error.map(RuntimeResult::InitError), - }, - None => resp_rx.await, + let result = { + let response = async { + instance + .api + .submit(job) + .await + .map_err(|error| ExecutorError::Io(error.into()))?; + let response = async { + match instance.init_error.take() { + Some(mut init) => { + tokio::select! { response = resp_rx => response, error = &mut init => error.map(RuntimeResult::InitError) } + } + None => resp_rx.await, + } + }; + match tokio::time::timeout(req.revision.timeout, response).await { + Ok(Ok(response)) => Ok(response), + Ok(Err(_)) => Err(ExecutorError::RuntimeExited( + "Runtime response channel closed".into(), + )), + Err(_) => Err(ExecutorError::Timeout(req.revision.timeout)), + } + }; + tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = response => result, + exit = instance.backend.wait() => { + exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before completing invocation".into()))) + } } }; - let result = match tokio::time::timeout(req.timeout, response).await { - Ok(Ok(r)) => r, - Ok(Err(_)) => { - // Bootstrap died before responding. - instance.backend.kill(); - return Err(ExecutorError::RuntimeExited( - "bootstrap exited mid-invocation".to_owned(), - )); - } - Err(_) => { - instance.backend.kill(); - return Err(ExecutorError::Timeout(req.timeout)); - } + let result = match context.check() { + Ok(()) => result, + Err(error) => Err(error), }; - - if !matches!(result, RuntimeResult::InitError(_)) { + let reusable = + matches!(&result, Ok(RuntimeResult::Success(_))) && instance.backend.running(); + if reusable { instance.last_used = Instant::now(); - self.release(key, instance); + if let Some(mut instance) = self.release(key, instance) { + instance.api.shutdown().await; + instance.backend.terminate().await?; + } + } else { + instance.api.shutdown().await; + instance.backend.terminate().await?; } - - match result { - RuntimeResult::Success(payload) => Ok(InvokeResponse { - status: 200, + match result? { + RuntimeResult::Success(payload) => Ok(InvokeResponse::success( payload, - function_error: None, - log_tail: None, - executed_version: req.qualifier, - }), + req.revision.qualifier.clone(), + )), RuntimeResult::Error(payload) => Ok(InvokeResponse { status: 200, payload, - function_error: Some("Unhandled".to_owned()), + function_error: Some("Unhandled".into()), log_tail: None, - executed_version: req.qualifier, + executed_version: req.revision.qualifier.clone(), }), - RuntimeResult::InitError(payload) => { - let msg = String::from_utf8_lossy(&payload).into_owned(); - Err(ExecutorError::InitFailed(msg)) - } + RuntimeResult::InitError(payload) => Err(ExecutorError::InitFailed( + String::from_utf8_lossy(&payload).into_owned(), + )), } } @@ -228,14 +273,17 @@ impl InstancePool { Some(instance) } - fn release(&self, key: PoolKey, mut instance: Instance) { + fn release(&self, key: PoolKey, mut instance: Instance) -> Option { let Ok(permit) = Arc::clone(&self.idle_capacity).try_acquire_owned() else { - return; + return Some(instance); }; let mut bucket = self.pools.entry(key.0).or_default(); if bucket.len() < self.max_warm.min(1) { instance.idle_permit = Some(permit); bucket.push((key.1, instance)); + None + } else { + Some(instance) } } @@ -269,7 +317,7 @@ impl InstancePool { idle_permit: None, init_error, }; - debug!(function = %req.function_name, addr = %addr, "spawned new lambda instance"); + debug!(function = %req.revision.function_name, addr = %addr, "spawned new lambda instance"); Ok(inst) } @@ -331,40 +379,51 @@ mod tests { assert_eq!(InstancePool::key(&alias), original); let mut recreated = request.clone(); - recreated.resource_id = crate::identity::FunctionResourceId::default(); + Arc::make_mut(&mut recreated.revision).resource_id = + crate::identity::FunctionResourceId::default(); assert_ne!(InstancePool::key(&recreated), original); let mut updated = request.clone(); - updated.revision_id = "new-revision".into(); + Arc::make_mut(&mut updated.revision).revision_id = "new-revision".into(); assert_ne!(InstancePool::key(&updated), original); let mut rehydrated = request.clone(); - rehydrated.artifact = + Arc::make_mut(&mut rehydrated.revision).artifact = DeploymentArtifact::Zip(crate::artifact::ValidatedArtifact::fixture()); assert_ne!(InstancePool::key(&rehydrated), original); let mut storage = request.clone(); - storage.ephemeral_storage_mb = 1024; + Arc::make_mut(&mut storage.revision).ephemeral_storage_mb = 1024; assert_ne!(InstancePool::key(&storage), original); let mut image = request.clone(); - image.image_config = Some(ImageConfig { + Arc::make_mut(&mut image.revision).image_config = Some(ImageConfig { command: Some(vec![]), ..ImageConfig::default() }); assert_ne!(InstancePool::key(&image), original); let explicitly_empty = InstancePool::key(&image); - image.image_config = Some(ImageConfig::default()); + Arc::make_mut(&mut image.revision).image_config = Some(ImageConfig::default()); assert_ne!(InstancePool::key(&image), explicitly_empty); } #[test] fn test_should_fingerprint_environment_independently_of_hashmap_iteration_order() { let mut first = InvokeRequest::fixture(); - first.environment.insert("FIRST".into(), "one".into()); - first.environment.insert("SECOND".into(), "two".into()); + Arc::make_mut(&mut first.revision) + .environment + .insert("FIRST".into(), "one".into()); + Arc::make_mut(&mut first.revision) + .environment + .insert("SECOND".into(), "two".into()); let mut second = first.clone(); - second.environment.clear(); - second.environment.insert("SECOND".into(), "two".into()); - second.environment.insert("FIRST".into(), "one".into()); + Arc::make_mut(&mut second.revision).environment.clear(); + Arc::make_mut(&mut second.revision) + .environment + .insert("SECOND".into(), "two".into()); + Arc::make_mut(&mut second.revision) + .environment + .insert("FIRST".into(), "one".into()); assert_eq!(InstancePool::key(&first), InstancePool::key(&second)); - second.environment.insert("SECOND".into(), "changed".into()); + Arc::make_mut(&mut second.revision) + .environment + .insert("SECOND".into(), "changed".into()); assert_ne!(InstancePool::key(&first), InstancePool::key(&second)); } } diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index 81279bb2..c401fc8e 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -9,8 +9,8 @@ //! //! Docker execution is not supported; it never falls back to native or successful echo. //! -//! All backends share a single in-process Lambda Runtime API server (Phase 2) -//! so the bootstrap-side protocol is identical to AWS. +//! Native instances expose a host Runtime API socket; Squib delegates the +//! bootstrap-side protocol to its guest agent. //! //! `async-trait` is required because `RustackLambda` stores the executor as //! `Arc` for backend swapping at startup; the trait must be @@ -31,19 +31,58 @@ pub use error::ExecutorError; pub use native::NativeExecutor; pub use noop::NoopExecutor; pub use squib::{SquibExecutor, SquibExecutorConfig}; -pub use types::{ExecutorBackend, InvokeRequest, InvokeResponse}; +pub use types::{ExecutionRevision, ExecutorBackend, InvokeRequest, InvokeResponse}; + +use crate::invocation::InvocationContext; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. #[async_trait] pub trait Executor: std::fmt::Debug + Send + Sync + 'static { /// Check whether execution is enabled before accepting asynchronous work. - fn available(&self) -> Result<(), ExecutorError> { + /// + /// # Errors + /// Returns an unavailable-backend or incompatible-artifact/configuration error. + /// + /// ``` + /// # use rustack_lambda_core::executor::{Executor, InvokeRequest}; + /// # fn probe(executor: &dyn Executor, request: &InvokeRequest) { + /// let available = executor.available(request).is_ok(); + /// # } + /// ``` + fn available(&self, _request: &InvokeRequest) -> Result<(), ExecutorError> { Ok(()) } /// Run the function and return its response. - async fn invoke(&self, req: InvokeRequest) -> Result; + /// + /// The work supervisor retains this future through cancellation and cleanup. + /// + /// # Errors + /// Returns invocation, initialization, transport or cleanup errors. A clean + /// outcome requires backend resources to be reusable or confirmed retired. + /// + /// ```no_run + /// # use rustack_lambda_core::{executor::{Executor, InvokeRequest}, invocation::InvocationContext}; + /// # async fn invoke(executor: &dyn Executor, request: InvokeRequest, context: InvocationContext) { + /// let result = executor.invoke(request, context).await; + /// # } + /// ``` + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result; /// Stop all warm instances and release resources. - async fn shutdown(&self); + /// + /// # Errors + /// Returns CleanupFailed if owned resource cleanup cannot be confirmed. + /// Dropping a shutdown waiter must retain the underlying cleanup operation. + /// + /// ```no_run + /// # async fn stop(executor: &dyn rustack_lambda_core::executor::Executor) { + /// let clean = executor.shutdown().await.is_ok(); + /// # } + /// ``` + async fn shutdown(&self) -> Result<(), ExecutorError>; } diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 58d99726..06463e92 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -5,13 +5,24 @@ //! match the host. The auto executor keeps macOS Zip functions on Squib and //! leaves unsupported image packages for the future Docker backend. -use std::{net::SocketAddr, path::Path, process::Stdio, sync::Arc, time::Duration}; +use std::{ + future::poll_fn, + net::SocketAddr, + path::Path, + process::Stdio, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; use async_trait::async_trait; use parking_lot::Mutex as PMutex; use tokio::{ io::AsyncReadExt, process::Command, + runtime::Handle, sync::{Semaphore, oneshot, watch}, task::JoinSet, }; @@ -21,7 +32,10 @@ use super::{ Executor, ExecutorError, InvokeRequest, InvokeResponse, instance::{BackendHandle, BackendHandleObj, InstanceBackend, InstancePool, spawn_reaper}, }; -use crate::artifact::{Architecture, DeploymentArtifact}; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + invocation::{CleanupTask, InvocationContext, Settlement}, +}; /// Native process executor. #[derive(Debug)] @@ -29,16 +43,52 @@ pub struct NativeExecutor { pool: Arc, cancel_tx: watch::Sender, reaper: PMutex>>, - processes: Arc>>, + processes: Arc, +} + +#[derive(Debug)] +struct NativeProcesses { + tasks: PMutex>, + cleanup: CleanupTask, + accepting: AtomicBool, + capacity: Arc, + runtime: Handle, +} + +impl NativeProcesses { + fn start_cleanup(self: &Arc) { + let processes = Arc::clone(self); + self.cleanup.start_on(&self.runtime, move || async move { + let mut failed = false; + while let Some(result) = + poll_fn(|context| processes.tasks.lock().poll_join_next(context)).await + { + if result.is_err() { + failed = true; + } + } + if failed { + Err(ExecutorError::CleanupFailed( + "Native process supervisor failed".into(), + )) + } else { + Ok(()) + } + }); + } } impl Drop for NativeExecutor { fn drop(&mut self) { + self.processes.accepting.store(false, Ordering::Release); + self.cancel_tx.send_replace(true); self.pool.shutdown(); if let Some(reaper) = self.reaper.lock().take() { reaper.abort(); } - self.processes.lock().abort_all(); + // Process supervisors retain children, artifact leases and reservations + // until exit. Dropping the executor must not abort these owners. + self.processes.start_cleanup(); } } @@ -46,10 +96,17 @@ impl NativeExecutor { /// Build a new native executor. #[must_use] pub fn new(max_warm: usize, idle_timeout: Duration, init_timeout: Duration) -> Self { - let processes = Arc::new(PMutex::new(JoinSet::new())); + let processes = Arc::new(NativeProcesses { + tasks: PMutex::new(JoinSet::new()), + cleanup: CleanupTask::default(), + accepting: AtomicBool::new(true), + capacity: Arc::new(Semaphore::new(32)), + runtime: Handle::current(), + }); + let (cancel_tx, cancel_rx) = watch::channel(false); let backend = Arc::new(NativeBackend { processes: Arc::clone(&processes), - capacity: Arc::new(Semaphore::new(32)), + shutdown: cancel_rx.clone(), }); let pool = Arc::new(InstancePool::new( backend, @@ -57,7 +114,6 @@ impl NativeExecutor { idle_timeout, init_timeout, )); - let (cancel_tx, cancel_rx) = watch::channel(false); let reaper = spawn_reaper(Arc::clone(&pool), Duration::from_secs(30), cancel_rx); Self { pool, @@ -70,16 +126,43 @@ impl NativeExecutor { #[async_trait] impl Executor for NativeExecutor { - async fn invoke(&self, req: InvokeRequest) -> Result { - let DeploymentArtifact::Zip(artifact) = &req.artifact else { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + if !self.processes.accepting.load(Ordering::Acquire) { + return Err(ExecutorError::Cancelled); + } + if !matches!(request.revision.artifact, DeploymentArtifact::Zip(_)) { + return Err(ExecutorError::Unsupported( + "native backend only supports Zip packages".into(), + )); + } + if request.revision.architecture.as_str() != host_architecture() { + return Err(ExecutorError::Unsupported( + "native backend requires the host architecture".into(), + )); + } + Ok(()) + } + + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.available(&req)?; + context.check()?; + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { return Err(ExecutorError::Unsupported( "native backend only supports Zip packages".into(), )); }; let code_root = artifact.code_root(); let bootstrap = code_root.join("bootstrap"); - ensure_executable(&bootstrap).await?; - if !bootstrap_runs_on_host(&bootstrap, req.architecture).await? { + let supported = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = async { ensure_executable(&bootstrap).await?; bootstrap_runs_on_host(&bootstrap, req.revision.architecture).await } => result?, + }; + if !supported { return Err(ExecutorError::Unsupported(format!( "bootstrap {} cannot run on host {}/{}; use docker backend", bootstrap.display(), @@ -87,44 +170,41 @@ impl Executor for NativeExecutor { std::env::consts::ARCH, ))); } - self.pool.invoke(req).await + self.pool.invoke(req, context).await } - async fn shutdown(&self) { - let _ = self.cancel_tx.send(true); - let reaper = self.reaper.lock().take(); - if let Some(r) = reaper { - if let Err(error) = r.await { - warn!(%error, "Lambda idle reaper failed"); - } - } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.processes.accepting.store(false, Ordering::Release); + self.cancel_tx.send_replace(true); self.pool.shutdown(); - let mut processes = std::mem::take(&mut *self.processes.lock()); - while let Some(result) = processes.join_next().await { - if let Err(error) = result { - warn!(%error, "Lambda process supervisor failed"); - } + self.processes.start_cleanup(); + let reaper = self.reaper.lock().take(); + if let Some(reaper) = reaper { + reaper + .await + .map_err(|_| ExecutorError::CleanupFailed("Native reaper failed".into()))?; } + self.processes.cleanup.wait().await } } #[derive(Debug)] struct NativeBackend { - processes: Arc>>, - capacity: Arc, + processes: Arc, + shutdown: watch::Receiver, } #[async_trait] impl InstanceBackend for NativeBackend { fn has_capacity(&self) -> bool { - self.capacity.available_permits() != 0 + self.processes.capacity.available_permits() != 0 } async fn spawn( &self, req: &InvokeRequest, runtime_api_addr: SocketAddr, ) -> Result { - let DeploymentArtifact::Zip(artifact) = &req.artifact else { + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { return Err(ExecutorError::Unsupported( "native backend only supports Zip packages".into(), )); @@ -143,6 +223,14 @@ impl InstanceBackend for NativeBackend { cmd.env(k, v); } + let permit = Arc::clone(&self.processes.capacity) + .acquire_owned() + .await + .map_err(|_| ExecutorError::Cancelled)?; + let mut processes = self.processes.tasks.lock(); + if !self.processes.accepting.load(Ordering::Acquire) { + return Err(ExecutorError::Cancelled); + } let mut child = cmd .spawn() .map_err(|e| ExecutorError::Io(format!("spawn bootstrap: {e}")))?; @@ -158,24 +246,46 @@ impl InstanceBackend for NativeBackend { } let (cancel, cancelled) = oneshot::channel(); let artifact_lease = artifact.clone(); - let mut processes = self.processes.lock(); while let Some(result) = processes.try_join_next() { if let Err(error) = result { warn!(%error, "Lambda process supervisor failed"); } } + let (completion, completed) = watch::channel(Settlement::Pending); + let mut shutdown = self.shutdown.clone(); processes.spawn(async move { - // Keep code and resources alive until the child exit has been reaped. - let _artifact_lease = artifact_lease; - tokio::select! { - _ = cancelled => { if let Err(error) = child.kill().await { warn!(%error, "Lambda process cancellation failed"); } }, - result = child.wait() => { if let Err(error) = result { warn!(%error, "Lambda process wait failed"); } }, + { + let _artifact_lease = artifact_lease; + let _permit = permit; + let result = tokio::select! { + biased; + () = async { let _ = shutdown.wait_for(|stopped| *stopped).await; } => child.kill().await, + _ = cancelled => child.kill().await, + result = child.wait() => result.map(|_| ()), + }; + if result.is_err() { + completion.send_replace(Settlement::Uncertain); + // Preserve child/lease/allocation ownership until reaping is confirmed. + loop { + let _ = child.start_kill(); + match child.wait().await { + Ok(_) => break, + Err(_) => tokio::time::sleep(Duration::from_secs(1)).await, + } + } + } + logs.abort_all(); + while let Some(result) = logs.join_next().await { + if result.is_err_and(|error| !error.is_cancelled()) { + warn!("Native log task failed"); + } + } } - logs.abort_all(); - while let Some(result) = logs.join_next().await { if let Err(error) = result { if !error.is_cancelled() { warn!(%error, "Lambda log task failed"); } } } + completion.send_replace(Settlement::Clean); }); Ok(BackendHandle::new(NativeHandle { cancel: Some(cancel), + completion: completed, })) } } @@ -183,9 +293,28 @@ impl InstanceBackend for NativeBackend { #[derive(Debug)] struct NativeHandle { cancel: Option>, + completion: watch::Receiver, } +#[async_trait] impl BackendHandleObj for NativeHandle { + fn running(&self) -> bool { + *self.completion.borrow() == Settlement::Pending + } + + async fn wait(&mut self) -> Result<(), ExecutorError> { + match self + .completion + .wait_for(|state| *state != Settlement::Pending) + .await + { + Ok(state) if *state == Settlement::Clean => Ok(()), + _ => Err(ExecutorError::CleanupFailed( + "Native child exit is not confirmed".into(), + )), + } + } + fn kill(&mut self) { if let Some(cancel) = self.cancel.take() { if cancel.send(()).is_err() { @@ -208,12 +337,7 @@ async fn bootstrap_runs_on_host( ) -> Result { // Architecture check first — declared `architectures` must include the // host arch. - let host_arch = match std::env::consts::ARCH { - "x86_64" => "x86_64", - "aarch64" => "arm64", - other => other, - }; - if architecture.as_str() != host_arch { + if architecture.as_str() != host_architecture() { return Ok(false); } let mut file = tokio::fs::File::open(path) @@ -233,6 +357,13 @@ async fn bootstrap_runs_on_host( Ok((elf && host_is_linux) || (macho && host_is_macos)) } +fn host_architecture() -> &'static str { + match std::env::consts::ARCH { + "aarch64" => "arm64", + other => other, + } +} + /// Require a regular executable bootstrap without modifying immutable artifacts. #[cfg(unix)] async fn ensure_executable(path: &Path) -> Result<(), ExecutorError> { @@ -254,31 +385,40 @@ async fn ensure_executable(_path: &Path) -> Result<(), ExecutorError> { } fn lambda_env_vars(req: &InvokeRequest, runtime_api: SocketAddr) -> Vec<(String, String)> { - let mut env = Vec::with_capacity(16 + req.environment.len()); - let task_root = match &req.artifact { + let mut env = Vec::with_capacity(16 + req.revision.environment.len()); + let task_root = match &req.revision.artifact { DeploymentArtifact::Zip(artifact) => artifact.code_root().display().to_string(), DeploymentArtifact::Image(_) => String::new(), }; env.push(("AWS_LAMBDA_RUNTIME_API".into(), runtime_api.to_string())); - env.push(("AWS_LAMBDA_FUNCTION_NAME".into(), req.function_name.clone())); - env.push(("AWS_LAMBDA_FUNCTION_VERSION".into(), req.qualifier.clone())); + env.push(( + "AWS_LAMBDA_FUNCTION_NAME".into(), + req.revision.function_name.clone(), + )); + env.push(( + "AWS_LAMBDA_FUNCTION_VERSION".into(), + req.revision.qualifier.clone(), + )); env.push(( "AWS_LAMBDA_FUNCTION_MEMORY_SIZE".into(), - req.memory_mb.to_string(), + req.revision.memory_mb.to_string(), )); env.push(( "AWS_LAMBDA_FUNCTION_TIMEOUT".into(), - req.timeout.as_secs().to_string(), + req.revision.timeout.as_secs().to_string(), )); env.push(( "AWS_LAMBDA_LOG_GROUP_NAME".into(), - format!("/aws/lambda/{}", req.function_name), + format!("/aws/lambda/{}", req.revision.function_name), )); env.push(( "AWS_LAMBDA_LOG_STREAM_NAME".into(), uuid::Uuid::new_v4().to_string(), )); - env.push(("_HANDLER".into(), req.handler.clone().unwrap_or_default())); + env.push(( + "_HANDLER".into(), + req.revision.handler.clone().unwrap_or_default(), + )); env.push(("LAMBDA_TASK_ROOT".into(), task_root.clone())); env.push(("LAMBDA_RUNTIME_DIR".into(), task_root)); env.push(("TZ".into(), ":UTC".into())); @@ -288,7 +428,7 @@ fn lambda_env_vars(req: &InvokeRequest, runtime_api: SocketAddr) -> Vec<(String, env.push(("AWS_SECRET_ACCESS_KEY".into(), "test".into())); env.push(("AWS_SESSION_TOKEN".into(), "test".into())); // User env overrides last, but do not let it shadow the runtime API var. - for (k, v) in &req.environment { + for (k, v) in &req.revision.environment { if k == "AWS_LAMBDA_RUNTIME_API" || k == "_HANDLER" || k == "LAMBDA_TASK_ROOT" { warn!(key = %k, "ignoring user override for reserved Lambda env var"); continue; diff --git a/crates/rustack-lambda-core/src/executor/noop.rs b/crates/rustack-lambda-core/src/executor/noop.rs index 7d052c74..823aee27 100644 --- a/crates/rustack-lambda-core/src/executor/noop.rs +++ b/crates/rustack-lambda-core/src/executor/noop.rs @@ -2,6 +2,7 @@ use async_trait::async_trait; use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; +use crate::invocation::InvocationContext; /// Explicit unavailable backend for metadata-only operation. #[derive(Debug, Default, Clone)] @@ -29,11 +30,17 @@ impl NoopExecutor { } #[async_trait] impl Executor for NoopExecutor { - fn available(&self) -> Result<(), ExecutorError> { + fn available(&self, _request: &InvokeRequest) -> Result<(), ExecutorError> { Err(self.error()) } - async fn invoke(&self, _req: InvokeRequest) -> Result { + async fn invoke( + &self, + _req: InvokeRequest, + _context: InvocationContext, + ) -> Result { Err(self.error()) } - async fn shutdown(&self) {} + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } } diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index f82523dd..5c03a388 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -6,8 +6,10 @@ use std::{ collections::HashMap, - env, + env, fmt, + future::Future, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; @@ -15,12 +17,18 @@ use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use tokio::{sync::Mutex, time::Instant}; -use tracing::warn; +use tokio::{ + runtime::Handle, + sync::{Mutex, watch}, + time::Instant, +}; use uuid::Uuid; use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; -use crate::artifact::{Architecture, DeploymentArtifact}; +use crate::{ + artifact::{Architecture, DeploymentArtifact}, + invocation::{CleanupTask, InvocationContext}, +}; const DEFAULT_INSTANCE_ID: &str = "rustack_lambda"; const DEFAULT_STAGE_PORT: u32 = 5003; @@ -34,9 +42,7 @@ const DEFAULT_RESPONSE_LIMIT_BYTES: usize = const DEFAULT_RUN_BUDGET: Duration = Duration::from_hours(24); const DEFAULT_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); const CONNECT_RESPONSE_LIMIT_BYTES: usize = 128; -const CONTROL_RESPONSE_LIMIT_BYTES: usize = 1024; const PROTOCOL_VERSION: &str = "rustack.squib.lambda.v1"; -const SHUTDOWN_REQUEST_KIND: &str = "shutdown"; /// Configuration for [`SquibExecutor`]. #[derive(Debug, Clone)] @@ -166,12 +172,61 @@ impl Default for SquibExecutorConfig { #[derive(Debug)] pub struct SquibExecutor { config: SquibExecutorConfig, - runtime: Mutex>, + runtime: Mutex>>, poisoned: std::sync::atomic::AtomicBool, + stop: watch::Sender, + cleanup: Arc, +} + +/// A concrete VM owner; even an idle owner's Drop starts retained shutdown. +struct OwnedVm { + runtime: Option, + cleanup: Arc, + spawner: Handle, +} + +trait VmShutdown: Send + 'static { + fn shutdown(self) -> impl Future> + Send; +} + +impl VmShutdown for squib::Squib { + async fn shutdown(mut self) -> Result<(), ExecutorError> { + squib::Squib::shutdown(&mut self) + .await + .map_err(|_| ExecutorError::CleanupFailed("Squib VM shutdown failed".into())) + } +} + +impl OwnedVm { + fn new(runtime: R, cleanup: Arc) -> Self { + Self { + runtime: Some(runtime), + cleanup, + spawner: Handle::current(), + } + } +} + +impl fmt::Debug for OwnedVm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedVm").finish_non_exhaustive() + } +} + +impl Drop for OwnedVm { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + // This owner creates the only VM for this cleanup task. Use the + // captured runtime even if Drop runs outside an entered Tokio context. + self.cleanup + .start_on(&self.spawner, move || runtime.shutdown()); + } + } } +#[derive(Debug)] struct RuntimeLease<'a> { - runtime: Option, + runtime: Option>, poisoned: &'a std::sync::atomic::AtomicBool, } impl Drop for RuntimeLease<'_> { @@ -179,24 +234,36 @@ impl Drop for RuntimeLease<'_> { if self.runtime.is_some() { self.poisoned .store(true, std::sync::atomic::Ordering::Release); - // Squib's RAII handle requests VM shutdown. Never admit another guest after - // cancellation. + // OwnedVm transfers shutdown to the retained cleanup task on Drop. } } } +impl Drop for SquibExecutor { + fn drop(&mut self) { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + self.stop.send_replace(true); + // An idle VM must follow the same owned shutdown path as a busy one. + drop(self.runtime.get_mut().take()); + } +} + impl SquibExecutor { /// Build a Squib executor. #[must_use] pub fn new(config: SquibExecutorConfig) -> Self { + let (stop, _) = watch::channel(false); Self { config, runtime: Mutex::new(None), poisoned: std::sync::atomic::AtomicBool::new(false), + stop, + cleanup: Arc::new(CleanupTask::default()), } } - async fn start_runtime(&self) -> Result { + async fn start_runtime(&self) -> Result, ExecutorError> { let config_file = self.config.required_config_file()?; ensure_config_file_exists(config_file).await?; @@ -207,21 +274,26 @@ impl SquibExecutor { .without_api_socket() .run_budget(self.config.run_budget); - let squib = builder - .spawn() - .await - .map_err(|err| ExecutorError::InitFailed(format!("start Squib runtime: {err}")))?; - Ok(squib) + let squib = builder.spawn().await.map_err(|_| { + ExecutorError::CleanupFailed( + "Squib startup did not establish confirmed runtime ownership".into(), + ) + })?; + Ok(OwnedVm::new(squib, Arc::clone(&self.cleanup))) } - async fn invoke_guest(&self, req: InvokeRequest) -> Result { + async fn invoke_guest( + &self, + req: &InvokeRequest, + context: &InvocationContext, + ) -> Result { #[cfg(unix)] { let mut stream = connect_stage_socket_with_retry(&self.config).await?; - let request = SquibInvokeRequestWire::from_request(&req)?; + let request = SquibInvokeRequestWire::from_request(req, context.id())?; let request_bytes = serde_json::to_vec(&request) .map_err(|err| ExecutorError::Io(format!("encode Squib request: {err}")))?; - tokio::time::timeout(req.timeout, async { + tokio::time::timeout(req.revision.timeout, async { use tokio::io::AsyncWriteExt as _; stream @@ -237,10 +309,10 @@ impl SquibExecutor { "Squib response", ) .await?; - decode_response(&line, req.qualifier.as_str()) + decode_response(&line, req.revision.qualifier.as_str()) }) .await - .map_err(|_| ExecutorError::Timeout(req.timeout))? + .map_err(|_| ExecutorError::Timeout(req.revision.timeout))? } #[cfg(not(unix))] @@ -251,11 +323,29 @@ impl SquibExecutor { )) } } + async fn retire_vm(&self, vm: OwnedVm) -> Result<(), ExecutorError> { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + drop(vm); + self.wait_for_cleanup().await + } + + async fn wait_for_cleanup(&self) -> Result<(), ExecutorError> { + // Only the waiter has a deadline. CleanupTask retains the VM shutdown + // future even when this budget expires or the caller drops the waiter. + tokio::time::timeout(self.config.shutdown_timeout, self.cleanup.wait()) + .await + .map_err(|_| { + ExecutorError::CleanupFailed("Squib VM shutdown is not confirmed".into()) + })? + } } #[async_trait] impl Executor for SquibExecutor { - fn available(&self) -> Result<(), ExecutorError> { + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + validate_request(request)?; + self.config.required_vsock_path()?; if self.poisoned.load(std::sync::atomic::Ordering::Acquire) { return Err(ExecutorError::Unsupported( "Squib guest execution was cancelled; restart the runtime before invoking again" @@ -265,60 +355,70 @@ impl Executor for SquibExecutor { Ok(()) } - async fn invoke(&self, req: InvokeRequest) -> Result { - validate_request(&req)?; - self.config.required_vsock_path()?; - let mut runtime = self.runtime.lock().await; - self.available()?; + async fn invoke( + &self, + req: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.available(&req)?; + context.check()?; + let mut runtime = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + runtime = self.runtime.lock() => runtime, + }; + self.available(&req)?; + // Squib's startup future owns unjoined VMM handles; never drop it on cancellation. let vm = match runtime.take() { Some(vm) => vm, - None => self.start_runtime().await?, + None => match self.start_runtime().await { + Ok(vm) => vm, + Err(error) => { + if matches!(&error, ExecutorError::CleanupFailed(_)) { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + } + return Err(error); + } + }, }; let mut lease = RuntimeLease { runtime: Some(vm), poisoned: &self.poisoned, }; - let result = self.invoke_guest(req).await; + let mut stop = self.stop.subscribe(); + let mut result = tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + _ = stop.wait_for(|stopped| *stopped) => Err(ExecutorError::Cancelled), + result = self.invoke_guest(&req, &context) => result, + }; + if self.poisoned.load(std::sync::atomic::Ordering::Acquire) || *self.stop.borrow() { + result = Err(ExecutorError::Cancelled); + } + if result.is_ok() { + if let Err(error) = context.check() { + result = Err(error); + } + } if result.is_ok() { *runtime = lease.runtime.take(); + } else if let Some(vm) = lease.runtime.take() { + self.retire_vm(vm).await?; } result } - async fn shutdown(&self) { - let mut runtime = self.runtime.lock().await.take(); - if let Some(squib) = runtime.as_mut() { - #[cfg(unix)] - match tokio::time::timeout( - self.config.shutdown_timeout, - request_guest_shutdown(&self.config), - ) - .await - { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!(error = %err, "failed to request Squib Lambda guest shutdown"); - } - Err(_) => { - warn!( - timeout_ms = self.config.shutdown_timeout.as_millis(), - "timed out requesting Squib Lambda guest shutdown" - ); - } - } - match tokio::time::timeout(self.config.shutdown_timeout, squib.shutdown()).await { - Ok(Ok(())) => {} - Ok(Err(err)) => { - warn!(error = %err, "failed to shut down Squib Lambda runtime"); - } - Err(_) => { - warn!( - timeout_ms = self.config.shutdown_timeout.as_millis(), - "timed out shutting down Squib Lambda runtime" - ); - } - } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + self.stop.send_replace(true); + let mut runtime = self.runtime.lock().await; + if let Some(vm) = runtime.take() { + self.retire_vm(vm).await?; } + drop(runtime); + self.wait_for_cleanup().await } } @@ -343,34 +443,27 @@ struct SquibInvokeRequestWire { payload_base64: String, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct SquibControlRequestWire { - protocol_version: &'static str, - kind: &'static str, -} - impl SquibInvokeRequestWire { - fn from_request(req: &InvokeRequest) -> Result { - let DeploymentArtifact::Zip(artifact) = &req.artifact else { + fn from_request(req: &InvokeRequest, request_id: Uuid) -> Result { + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { return Err(ExecutorError::Unsupported( "Squib backend only supports Zip packages".into(), )); }; Ok(Self { protocol_version: PROTOCOL_VERSION, - request_id: Uuid::new_v4().to_string(), + request_id: request_id.to_string(), function_arn: req.function_arn.clone(), - function_name: req.function_name.clone(), - qualifier: req.qualifier.clone(), - runtime: req.runtime.clone(), - handler: req.handler.clone(), - architectures: vec![req.architecture.as_str().into()], + function_name: req.revision.function_name.clone(), + qualifier: req.revision.qualifier.clone(), + runtime: req.revision.runtime.clone(), + handler: req.revision.handler.clone(), + architectures: vec![req.revision.architecture.as_str().into()], code_root: Some(artifact.code_root().display().to_string()), code_zip_base64: STANDARD.encode(artifact.zip_bytes()), - environment: req.environment.clone(), - timeout_ms: duration_millis(req.timeout), - memory_mb: req.memory_mb, + environment: req.revision.environment.clone(), + timeout_ms: duration_millis(req.revision.timeout), + memory_mb: req.revision.memory_mb, capture_logs: req.capture_logs, payload_base64: STANDARD.encode(req.payload.as_ref()), }) @@ -392,12 +485,12 @@ struct SquibInvokeResponseWire { } fn validate_request(req: &InvokeRequest) -> Result<(), ExecutorError> { - if !matches!(req.artifact, DeploymentArtifact::Zip(_)) { + if !matches!(req.revision.artifact, DeploymentArtifact::Zip(_)) { return Err(ExecutorError::Unsupported( "Squib backend only supports Zip packages".to_owned(), )); } - if req.architecture != Architecture::Arm64 { + if req.revision.architecture != Architecture::Arm64 { return Err(ExecutorError::Unsupported( "Squib backend requires an arm64 Lambda function".to_owned(), )); @@ -427,30 +520,6 @@ async fn ensure_config_file_exists(path: &Path) -> Result<(), ExecutorError> { Ok(()) } -#[cfg(unix)] -async fn request_guest_shutdown(config: &SquibExecutorConfig) -> Result<(), ExecutorError> { - use tokio::io::AsyncWriteExt as _; - - let mut stream = connect_stage_socket_with_retry(config).await?; - let request = SquibControlRequestWire { - protocol_version: PROTOCOL_VERSION, - kind: SHUTDOWN_REQUEST_KIND, - }; - let request_bytes = serde_json::to_vec(&request) - .map_err(|err| ExecutorError::Io(format!("encode Squib shutdown request: {err}")))?; - stream - .write_all(&request_bytes) - .await - .map_err(|err| ExecutorError::Io(format!("write Squib shutdown request: {err}")))?; - stream - .write_all(b"\n") - .await - .map_err(|err| ExecutorError::Io(format!("write Squib shutdown newline: {err}")))?; - let _line = - read_bounded_line(&mut stream, CONTROL_RESPONSE_LIMIT_BYTES, "Squib shutdown").await?; - Ok(()) -} - #[cfg(unix)] async fn connect_stage_socket_with_retry( config: &SquibExecutorConfig, @@ -632,10 +701,54 @@ fn default_artifact_dir() -> PathBuf { #[cfg(test)] mod tests { - use std::time::Duration; + use std::{sync::Arc, time::Duration}; use super::*; + struct ProbeVm { + started: Arc, + gate: Arc, + reservation: tokio::sync::OwnedSemaphorePermit, + } + + impl VmShutdown for ProbeVm { + async fn shutdown(self) -> Result<(), ExecutorError> { + self.started.add_permits(1); + self.gate.acquire().await.unwrap().forget(); + drop(self.reservation); + Ok(()) + } + } + + #[tokio::test] + async fn test_should_transfer_idle_vm_drop_to_owned_shutdown_before_releasing_resources() { + let cleanup = Arc::new(CleanupTask::default()); + let capacity = Arc::new(tokio::sync::Semaphore::new(1)); + let started = Arc::new(tokio::sync::Semaphore::new(0)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let owner = OwnedVm::new( + ProbeVm { + started: Arc::clone(&started), + gate: Arc::clone(&gate), + reservation: capacity.clone().acquire_owned().await.unwrap(), + }, + Arc::clone(&cleanup), + ); + // Drop on an ordinary thread with no entered Tokio context. + std::thread::spawn(move || drop(owner)).join().unwrap(); + started.acquire().await.unwrap().forget(); + assert_eq!(capacity.available_permits(), 0); + assert!( + tokio::time::timeout(Duration::ZERO, cleanup.wait()) + .await + .is_err() + ); + assert_eq!(capacity.available_permits(), 0); + gate.add_permits(1); + cleanup.wait().await.unwrap(); + assert_eq!(capacity.available_permits(), 1); + } + #[test] fn test_should_parse_squib_config_from_env_reader() { let config = SquibExecutorConfig::from_env_reader(|key| match key { @@ -699,10 +812,10 @@ mod tests { let mut req = InvokeRequest::fixture(); req.payload = Bytes::from_static(b"{}"); req.capture_logs = true; - let wire = SquibInvokeRequestWire::from_request(&req).unwrap(); + let wire = SquibInvokeRequestWire::from_request(&req, Uuid::new_v4()).unwrap(); assert_eq!(wire.protocol_version, PROTOCOL_VERSION); - let DeploymentArtifact::Zip(artifact) = &req.artifact else { + let DeploymentArtifact::Zip(artifact) = &req.revision.artifact else { panic!("unit fixture is ZIP"); }; assert_eq!( @@ -776,7 +889,7 @@ mod tests { #[test] fn test_should_reject_non_arm64_request() { let mut req = InvokeRequest::fixture(); - req.architecture = Architecture::X86_64; + Arc::make_mut(&mut req.revision).architecture = Architecture::X86_64; assert!(matches!( validate_request(&req), diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index 061f400b..dcf1e299 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -1,6 +1,6 @@ //! Shared executor request/response/config types. -use std::{collections::HashMap, fmt, str::FromStr, time::Duration}; +use std::{collections::HashMap, fmt, str::FromStr, sync::Arc, time::Duration}; use bytes::Bytes; use rustack_lambda_model::types::ImageConfig; @@ -45,85 +45,133 @@ impl FromStr for ExecutorBackend { } } -/// Everything an executor needs to run one invocation. +/// Validated function configuration captured before invocation admission. +/// +/// Requests share this value through Arc; updates create another captured revision. #[derive(Clone, TypedBuilder)] #[non_exhaustive] -pub struct InvokeRequest { - /// Function lifetime identity; deletion/recreation never reuses it. +pub struct ExecutionRevision { + /// Stable function lifetime identity. pub resource_id: FunctionResourceId, - /// Captured configuration revision, independent of the invoked alias. + /// Captured configuration revision identifier. pub revision_id: String, - /// Complete Docker entry configuration; omitted and empty argv differ. - pub image_config: Option, - /// Ephemeral storage size in MiB, captured with this invocation's revision. - pub ephemeral_storage_mb: u32, - /// Qualified ARN — used for `Lambda-Runtime-Invoked-Function-Arn`. - pub function_arn: String, + /// Backend selection policy captured at admission. + pub backend: ExecutorBackend, /// Bare function name. pub function_name: String, - /// Resolved version string, e.g. `"$LATEST"` or `"3"`. + /// Resolved numeric version or `$LATEST`, never an invoked alias. pub qualifier: String, - /// Lambda runtime identifier, e.g. `"provided.al2023"`. + /// Runtime identifier for ZIP code. pub runtime: Option, - /// Handler string from the function config. + /// Configured ZIP handler. pub handler: Option, - /// Validated single Lambda architecture. + /// One validated architecture. pub architecture: Architecture, - /// Exactly one immutable deployment artifact, with lifetime ownership. + /// One immutable artifact, with shared lifetime ownership. pub artifact: DeploymentArtifact, - /// User-supplied environment variables. + /// Complete image entry configuration; missing and empty argv differ. + pub image_config: Option, + /// Captured user and provider-injected environment. pub environment: HashMap, - /// Function timeout. Excess => `Sandbox.Timedout`. + /// Function handler time limit. pub timeout: Duration, - /// Memory size in MB (used as the container memory cap). + /// Function memory in MiB. pub memory_mb: u32, - /// Raw invocation payload (typically JSON, but not required). - pub payload: Bytes, - /// When set, the executor captures the last 4KB of stderr as `log_tail`. - pub capture_logs: bool, + /// Ephemeral storage in MiB. + pub ephemeral_storage_mb: u32, } -impl fmt::Debug for InvokeRequest { +impl fmt::Debug for ExecutionRevision { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("InvokeRequest") + f.debug_struct("ExecutionRevision") .field("resource_id", &self.resource_id) .field("revision_id", &self.revision_id) - .field("function_arn", &self.function_arn) + .field("backend", &self.backend) + .field("function_name", &self.function_name) .field("qualifier", &self.qualifier) .field("artifact", &self.artifact) .field("architecture", &self.architecture) .field("environment", &"[REDACTED]") .field("image_config", &"[REDACTED]") - .field("payload", &"[REDACTED]") .field("timeout", &self.timeout) .field("memory_mb", &self.memory_mb) .field("ephemeral_storage_mb", &self.ephemeral_storage_mb) - .field("capture_logs", &self.capture_logs) .finish_non_exhaustive() } } +/// One invocation of an immutable captured revision. +#[derive(Clone)] +#[non_exhaustive] +pub struct InvokeRequest { + /// Shared captured configuration and artifact ownership. + pub revision: Arc, + /// Canonical requested ARN, preserving the invoked qualifier. + pub function_arn: String, + /// Original event bytes. + pub payload: Bytes, + /// Whether to return the final runtime log bytes. + pub capture_logs: bool, +} + +impl InvokeRequest { + /// Create an invocation without copying its captured configuration. + /// + /// ``` + /// # use std::sync::Arc; + /// # use bytes::Bytes; + /// # use rustack_lambda_core::executor::{ExecutionRevision, InvokeRequest}; + /// # fn capture(revision: Arc, arn: String) { + /// let request = InvokeRequest::new(Arc::clone(&revision), arn, Bytes::new()); + /// assert!(Arc::ptr_eq(&request.revision, &revision)); + /// # } + /// ``` + #[must_use] + pub fn new(revision: Arc, function_arn: String, payload: Bytes) -> Self { + Self { + revision, + function_arn, + payload, + capture_logs: false, + } + } +} + +impl fmt::Debug for InvokeRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvokeRequest") + .field("revision", &self.revision) + .field("function_arn", &self.function_arn) + .field("payload", &"[REDACTED]") + .field("capture_logs", &self.capture_logs) + .finish() + } +} + #[cfg(test)] impl InvokeRequest { pub(crate) fn fixture() -> Self { - Self::builder() + let revision = ExecutionRevision::builder() .resource_id(FunctionResourceId::default()) .revision_id("fixture-revision".into()) - .image_config(None) - .ephemeral_storage_mb(512) - .function_arn("arn:aws:lambda:us-east-1:000000000000:function:fixture".into()) + .backend(ExecutorBackend::Native) .function_name("fixture".into()) .qualifier("$LATEST".into()) .runtime(Some("provided.al2023".into())) .handler(Some("bootstrap".into())) .architecture(Architecture::Arm64) .artifact(DeploymentArtifact::Zip(ValidatedArtifact::fixture())) + .image_config(None) .environment(HashMap::new()) .timeout(Duration::from_secs(3)) .memory_mb(128) - .payload(Bytes::new()) - .capture_logs(false) - .build() + .ephemeral_storage_mb(512) + .build(); + Self::new( + Arc::new(revision), + "arn:aws:lambda:us-east-1:000000000000:function:fixture".into(), + Bytes::new(), + ) } } @@ -173,13 +221,26 @@ impl InvokeResponse { mod tests { use super::*; + #[test] + fn test_should_share_captured_revisions_and_isolate_separately_updated_configuration() { + let original = InvokeRequest::fixture(); + let mut updated = original.clone(); + assert!(Arc::ptr_eq(&original.revision, &updated.revision)); + Arc::make_mut(&mut updated.revision).memory_mb = 512; + assert_eq!(original.revision.memory_mb, 128); + assert_eq!(updated.revision.memory_mb, 512); + assert!(!Arc::ptr_eq(&original.revision, &updated.revision)); + } + #[test] fn test_should_redact_invocation_request_and_response_debug() { let secret = "fixture-secret-should-never-be-rendered"; let mut request = InvokeRequest::fixture(); - request.environment.insert("SECRET".into(), secret.into()); + Arc::make_mut(&mut request.revision) + .environment + .insert("SECRET".into(), secret.into()); request.payload = Bytes::from(secret); - request.image_config = Some(ImageConfig { + Arc::make_mut(&mut request.revision).image_config = Some(ImageConfig { entry_point: Some(vec![secret.into()]), command: Some(vec![secret.into()]), working_directory: Some(format!("/{secret}")), diff --git a/crates/rustack-lambda-core/src/invocation.rs b/crates/rustack-lambda-core/src/invocation.rs new file mode 100644 index 00000000..a4e6e099 --- /dev/null +++ b/crates/rustack-lambda-core/src/invocation.rs @@ -0,0 +1,471 @@ +//! Attempt identity, cooperative cancellation and supervised invocation handles. +//! +//! Cancellation signals do not drop backend futures. The work supervisor owns +//! those futures until their resources are settled. + +use std::{fmt, future::Future, sync::Arc}; + +use parking_lot::Mutex as OwnerMutex; +use tokio::{ + runtime::Handle, + sync::{Mutex, oneshot, watch}, + task::JoinHandle, + time::{Instant, sleep_until, timeout_at}, +}; +use uuid::Uuid; + +use crate::executor::{ExecutorError, InvokeResponse}; + +/// Why an invocation must stop accepting a result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CancellationReason { + /// Explicit cancellation or loss of the attached caller. + Requested, + /// The absolute invocation budget expired. + Deadline, +} + +impl CancellationReason { + pub(crate) fn error(self) -> ExecutorError { + match self { + Self::Requested => ExecutorError::Cancelled, + Self::Deadline => ExecutorError::DeadlineExceeded, + } + } +} + +/// Immutable attempt metadata and a cooperative cancellation receiver. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct InvocationContext { + id: Uuid, + deadline: Instant, + cancellation: watch::Receiver, +} + +impl InvocationContext { + pub(crate) fn channel(deadline: Instant) -> (watch::Sender, Self) { + let (sender, cancellation) = watch::channel(false); + ( + sender, + Self { + id: Uuid::new_v4(), + deadline, + cancellation, + }, + ) + } + + /// Attempt identity shared with the Runtime API and invocation handle. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// assert!(!context.id().is_nil()); + /// # } + /// ``` + #[must_use] + pub fn id(&self) -> Uuid { + self.id + } + + /// Absolute monotonic budget, including queue and cold start. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let deadline = context.deadline(); + /// # } + /// ``` + #[must_use] + pub fn deadline(&self) -> Instant { + self.deadline + } + + /// Check cancellation before allocating a runtime resource. + /// + /// # Errors + /// Returns cancellation or deadline expiry, without cancelling cleanup. + /// + /// ``` + /// # fn inspect(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let still_active = context.check().is_ok(); + /// # } + /// ``` + pub fn check(&self) -> Result<(), ExecutorError> { + if Instant::now() >= self.deadline { + return Err(ExecutorError::DeadlineExceeded); + } + if *self.cancellation.borrow() || self.cancellation.has_changed().is_err() { + return Err(ExecutorError::Cancelled); + } + Ok(()) + } + + /// Wait for an explicit signal or the absolute deadline. + /// + /// ```no_run + /// # async fn stop(context: &rustack_lambda_core::invocation::InvocationContext) { + /// let reason = context.cancelled().await; + /// # } + /// ``` + pub async fn cancelled(&self) -> CancellationReason { + if Instant::now() >= self.deadline { + return CancellationReason::Deadline; + } + let mut cancellation = self.cancellation.clone(); + if *cancellation.borrow() { + return CancellationReason::Requested; + } + tokio::select! { + biased; + () = sleep_until(self.deadline) => CancellationReason::Deadline, + _ = cancellation.wait_for(|cancelled| *cancelled) => { + if Instant::now() >= self.deadline { CancellationReason::Deadline } else { CancellationReason::Requested } + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Settlement { + Pending, + Clean, + Uncertain, +} + +/// Attached ownership of an admitted invocation; Drop requests cancellation. +#[non_exhaustive] +pub struct InvocationHandle { + context: InvocationContext, + cancellation: watch::Sender, + response: oneshot::Receiver>, + settlement: watch::Receiver, + cancel_on_drop: bool, +} + +impl fmt::Debug for InvocationHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvocationHandle") + .field("id", &self.context.id) + .field("deadline", &self.context.deadline) + .field("settlement", &*self.settlement.borrow()) + .finish_non_exhaustive() + } +} + +impl InvocationHandle { + pub(crate) fn new( + context: InvocationContext, + cancellation: watch::Sender, + response: oneshot::Receiver>, + settlement: watch::Receiver, + ) -> Self { + Self { + context, + cancellation, + response, + settlement, + cancel_on_drop: true, + } + } + + /// UUID used for this admitted attempt. + /// + /// ``` + /// # fn inspect(handle: &rustack_lambda_core::invocation::InvocationHandle) { + /// assert!(!handle.id().is_nil()); + /// # } + /// ``` + #[must_use] + pub fn id(&self) -> Uuid { + self.context.id + } + + /// Signal cancellation while retaining the ability to await cleanup. + /// + /// ``` + /// # fn stop(handle: &rustack_lambda_core::invocation::InvocationHandle) { + /// handle.cancel(); + /// # } + /// ``` + pub fn cancel(&self) { + self.cancellation.send_replace(true); + } + + /// Await the backend outcome within the admitted invocation deadline. + /// + /// # Errors + /// Returns backend errors, deadline expiry, or an uncertain worker failure. + /// Deadline failure signals cancellation; the supervisor still owns cleanup. + /// + /// ```no_run + /// # async fn finish(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// let response = handle.wait().await; + /// # } + /// ``` + pub async fn wait(mut self) -> Result { + self.context.check()?; + let response = tokio::select! { + biased; + reason = self.context.cancelled() => return Err(reason.error()), + result = &mut self.response => result, + }; + self.context.check()?; + match response { + Ok(result) => { + self.cancel_on_drop = false; + result + } + Err(_) => Err(ExecutorError::CleanupFailed( + "Invocation worker terminated without settlement".into(), + )), + } + } + + /// Request cancellation and await confirmed cleanup, independently of the call deadline. + /// + /// # Errors + /// Returns CleanupFailed when the cleanup waiter expires or settlement is uncertain. + /// Cleanup remains owned after this method returns an error or is dropped. + /// + /// ```no_run + /// # async fn stop(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + /// let settled = handle.cancel_and_wait(deadline).await; + /// # } + /// ``` + pub async fn cancel_and_wait(mut self, deadline: Instant) -> Result<(), ExecutorError> { + self.cancel(); + let state = timeout_at( + deadline, + self.settlement + .wait_for(|state| *state != Settlement::Pending), + ) + .await; + match state { + Ok(Ok(state)) if *state == Settlement::Clean => { + self.cancel_on_drop = false; + Ok(()) + } + _ => Err(ExecutorError::CleanupFailed( + "Invocation cleanup is not confirmed".into(), + )), + } + } + + /// Transfer Event invocation ownership to the supervisor after successful admission. + /// + /// ``` + /// # fn accept(handle: rustack_lambda_core::invocation::InvocationHandle) { + /// handle.detach(); + /// # } + /// ``` + pub fn detach(mut self) { + self.cancel_on_drop = false; + } +} + +impl Drop for InvocationHandle { + fn drop(&mut self) { + if self.cancel_on_drop { + self.cancel(); + } + } +} + +/// Single cleanup operation whose future survives cancellation of any waiter. +#[derive(Default)] +pub(crate) struct CleanupTask { + state: OwnerMutex>>>, +} + +enum CleanupState { + Running(JoinHandle>), + Complete(Result<(), ExecutorError>), +} + +impl fmt::Debug for CleanupTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CleanupTask") + .field("started", &self.state.lock().is_some()) + .finish() + } +} +impl fmt::Debug for CleanupState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Running(_) => "Running", + Self::Complete(Ok(())) => "Clean", + Self::Complete(Err(_)) => "Failed", + }) + } +} + +impl CleanupTask { + #[cfg(test)] + pub(crate) fn start(&self, factory: F) -> bool + where + F: FnOnce() -> Fut, + Fut: Future> + Send + 'static, + { + self.start_on(&Handle::current(), factory) + } + + pub(crate) fn start_on(&self, runtime: &Handle, factory: F) -> bool + where + F: FnOnce() -> Fut, + Fut: Future> + Send + 'static, + { + let mut state = self.state.lock(); + if state.is_some() { + return false; + } + // Retain the JoinHandle before returning; dropping a waiter never drops this future. + *state = Some(Arc::new(Mutex::new(CleanupState::Running( + runtime.spawn(factory()), + )))); + true + } + + pub(crate) async fn wait(&self) -> Result<(), ExecutorError> { + let state = self.state.lock().clone(); + let Some(state) = state else { + return Ok(()); + }; + let mut state = state.lock().await; + if let CleanupState::Running(task) = &mut *state { + let result = task.await.unwrap_or_else(|_| { + Err(ExecutorError::CleanupFailed( + "Cleanup supervisor task failed".into(), + )) + }); + *state = CleanupState::Complete(result); + } + match &*state { + CleanupState::Complete(result) => result.clone(), + CleanupState::Running(_) => Err(ExecutorError::CleanupFailed( + "Cleanup supervisor is unsettled".into(), + )), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use bytes::Bytes; + use tokio::sync::Semaphore; + + use super::*; + + #[tokio::test(start_paused = true)] + async fn test_should_prioritize_absolute_expiry_and_notice_closed_cancellation_owner() { + let deadline = Instant::now() + Duration::from_secs(2); + let (cancel, context) = InvocationContext::channel(deadline); + assert!(context.check().is_ok()); + cancel.send_replace(true); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(context.cancelled().await, CancellationReason::Deadline); + assert!(matches!( + context.check(), + Err(ExecutorError::DeadlineExceeded) + )); + + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(2)); + drop(cancel); + assert_eq!(context.cancelled().await, CancellationReason::Requested); + assert!(matches!(context.check(), Err(ExecutorError::Cancelled))); + } + + #[tokio::test(start_paused = true)] + async fn test_should_discard_a_result_received_after_the_attempt_deadline() { + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(1)); + let (response, received) = oneshot::channel(); + let (_, settled) = watch::channel(Settlement::Clean); + let handle = InvocationHandle::new(context, cancel.clone(), received, settled); + response + .send(Ok(InvokeResponse::success(Bytes::new(), "$LATEST".into()))) + .unwrap(); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!( + handle.wait().await, + Err(ExecutorError::DeadlineExceeded) + )); + assert!( + *cancel.borrow(), + "expiry must request owned backend cleanup" + ); + } + + #[tokio::test(start_paused = true)] + async fn test_should_wait_for_cleanup_after_expiry_and_detach_only_explicitly() { + let (cancel, context) = InvocationContext::channel(Instant::now()); + let (_, received) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let handle = InvocationHandle::new(context, cancel.clone(), received, settled); + settlement.send_replace(Settlement::Clean); + handle + .cancel_and_wait(Instant::now() + Duration::from_secs(2)) + .await + .unwrap(); + assert!(*cancel.borrow()); + + let (cancel, context) = InvocationContext::channel(Instant::now() + Duration::from_secs(2)); + let (_, received) = oneshot::channel(); + let (_, settled) = watch::channel(Settlement::Pending); + InvocationHandle::new(context, cancel.clone(), received, settled).detach(); + assert!(!*cancel.borrow(), "Event detach preserves admitted work"); + } + + #[tokio::test] + async fn test_should_retain_cleanup_resources_when_a_waiter_is_aborted() { + let cleanup = Arc::new(CleanupTask::default()); + let resources = Arc::new(Semaphore::new(1)); + let reservation = resources.clone().acquire_owned().await.unwrap(); + let release = Arc::new(Semaphore::new(0)); + let ready = Arc::new(Semaphore::new(0)); + let gate = Arc::clone(&release); + let started = Arc::clone(&ready); + assert!(cleanup.start(move || async move { + let _reservation = reservation; + started.add_permits(1); + gate.acquire().await.unwrap().forget(); + Ok(()) + })); + ready.acquire().await.unwrap().forget(); + let owner = Arc::clone(&cleanup); + let waiter = tokio::spawn(async move { owner.wait().await }); + tokio::task::yield_now().await; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!(resources.available_permits(), 0); + assert!(!cleanup.start(|| async { Ok(()) })); + + release.add_permits(1); + let (first, second) = tokio::join!(cleanup.wait(), cleanup.wait()); + first.unwrap(); + second.unwrap(); + assert_eq!(resources.available_permits(), 1); + cleanup.wait().await.unwrap(); + } + + #[tokio::test] + async fn test_should_preserve_failed_cleanup_and_supervised_panics_for_all_waiters() { + let cleanup = CleanupTask::default(); + assert!(cleanup.start(|| async { Err(ExecutorError::CleanupFailed("failed".into())) })); + for _ in 0..2 { + assert!( + matches!(cleanup.wait().await, Err(ExecutorError::CleanupFailed(message)) if message == "failed") + ); + } + let cleanup = CleanupTask::default(); + assert!(cleanup.start(|| async { panic!("intentional cleanup task panic") })); + for _ in 0..2 { + assert!(matches!( + cleanup.wait().await, + Err(ExecutorError::CleanupFailed(_)) + )); + } + } +} diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 63ca483f..84169357 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod executor; mod function_config; pub mod handler; pub mod identity; +pub mod invocation; pub mod provider; pub mod resolver; pub mod storage; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index e43d2009..5dd3bdaa 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -62,10 +62,11 @@ use crate::{ config::LambdaConfig, error::LambdaServiceError, executor::{ - AutoExecutor, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, NoopExecutor, - SquibExecutor, + AutoExecutor, ExecutionRevision, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, + NoopExecutor, SquibExecutor, }, identity::FunctionResourceId, + invocation::InvocationHandle, resolver::{ alias_arn, function_arn, function_version_arn, layer_arn, layer_version_arn, parse_layer_version_arn, resolve_function_ref, resolve_version, @@ -281,10 +282,13 @@ impl RustackLambda { /// Stop all warm executor instances. Wired into the rustack server's /// graceful shutdown path. pub async fn shutdown(&self) { - if let Err(error) = self.work.quiesce(Duration::ZERO).await { + self.work.cancel_all(); + if let Err(error) = self.work.quiesce(Duration::from_secs(10)).await { tracing::error!(%error, "Lambda shutdown cancelled unfinished work"); } - self.executor.shutdown().await; + if self.executor.shutdown().await.is_err() { + tracing::error!("Lambda executor shutdown did not confirm cleanup"); + } } /// Stop accepting invocation work and drain it before snapshot export. @@ -292,16 +296,19 @@ impl RustackLambda { /// # Errors /// Returns an error after cancelling remaining tasks when the deadline expires. pub async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { - let deadline = tokio::time::Instant::now() + timeout; + let deadline = tokio::time::Instant::now() + .checked_add(timeout) + .ok_or_else(|| invalid_parameter("Invalid Lambda quiesce budget"))?; let cleanup_budget = (timeout / 10).min(Duration::from_secs(1)); let result = self .work .quiesce(timeout.saturating_sub(cleanup_budget)) .await; if result.is_err() - && tokio::time::timeout_at(deadline, self.executor.shutdown()) - .await - .is_err() + && !matches!( + tokio::time::timeout_at(deadline, self.executor.shutdown()).await, + Ok(Ok(())) + ) { tracing::error!("Lambda executor cancellation deadline exceeded"); } @@ -934,12 +941,11 @@ impl RustackLambda { // Phase 0: Invoke // --------------------------------------------------------------- - /// Invoke a function. + /// Invoke a function through shared admission and supervised execution. /// - /// Resolves the target version, validates the payload size, then routes - /// to the configured [`Executor`]. `DryRun` short-circuits before the - /// executor is touched. `Event` returns immediately with a synthetic - /// request id only after a slot in the bounded, supervised work set is acquired. + /// # Errors + /// Rejects invalid payload/reference, missing resources, unavailable execution, + /// exhausted capacity and mapped backend errors. DryRun starts no runtime. pub async fn invoke( &self, function_ref: &str, @@ -947,66 +953,128 @@ impl RustackLambda { payload: &[u8], invocation_type: InvokeKind, ) -> Result { - // Validate synchronous payload size (Appendix C: 6 MB). - let payload_limit = if invocation_type == InvokeKind::Event { - 1024 * 1024 - } else { - MAX_SYNC_PAYLOAD - }; - if payload.len() > payload_limit { - let payload_len = payload.len(); - return Err(LambdaServiceError::RequestTooLarge { - message: format!( - "Request payload size {payload_len} exceeds the invoke limit of \ - {payload_limit} bytes", - ), - }); + validate_invoke_payload( + payload, + if invocation_type == InvokeKind::Event { + 1024 * 1024 + } else { + MAX_SYNC_PAYLOAD + }, + )?; + let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; + resolve_version(&record, qualifier.as_deref())?; + if invocation_type == InvokeKind::DryRun { + return Ok(InvokeOutcome::DryRun); } - - let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let qualifier = qualifier.or(ref_qualifier.as_deref()); - if let Some(q) = qualifier { - crate::resolver::Qualifier::parse(q)?; + let handle = self + .admit_invocation( + &record, + qualifier.as_deref(), + payload, + invocation_type == InvokeKind::Event, + None, + ) + .await?; + if invocation_type == InvokeKind::Event { + let request_id = handle.id().to_string(); + handle.detach(); + return Ok(InvokeOutcome::Async { request_id }); } + handle + .wait() + .await + .map(InvokeOutcome::Sync) + .map_err(LambdaServiceError::from) + } - // Validate function exists and qualifier resolves. - let record = self.get_record(&name)?; - let version = resolve_version(&record, qualifier)?; + /// Admit a cancellable synchronous invocation for an internal service adapter. + /// + /// Shares version resolution, validation and quotas with ordinary HTTP Invoke. + /// The optional absolute deadline bounds capture, queueing and cold start. + /// + /// # Errors + /// Rejects invalid input, missing resources, capture timeout, unavailable backend + /// or exhausted admission/execution capacity. Runtime outcomes use the handle. + /// + /// ```no_run + /// # async fn invoke(provider: &rustack_lambda_core::provider::RustackLambda) + /// # -> Result<(), Box> { + /// let handle = provider.start_invocation("worker", None, b"{}", None).await?; + /// let result = handle.wait().await?; + /// # Ok(()) } + /// ``` + pub async fn start_invocation( + &self, + function_ref: &str, + qualifier: Option<&str>, + payload: &[u8], + deadline: Option, + ) -> Result { + validate_invoke_payload(payload, MAX_SYNC_PAYLOAD)?; + let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; + self.admit_invocation(&record, qualifier.as_deref(), payload, false, deadline) + .await + } - if invocation_type == InvokeKind::DryRun { - return Ok(InvokeOutcome::DryRun); + fn resolve_invocation( + &self, + reference: &str, + qualifier: Option<&str>, + ) -> Result<(FunctionRecord, Option), LambdaServiceError> { + let (name, embedded) = resolve_function_ref(reference)?; + if qualifier + .zip(embedded.as_deref()) + .is_some_and(|(explicit, embedded)| explicit != embedded) + { + return Err(invalid_parameter("Conflicting invocation qualifiers")); } - - if self.config.executor == ExecutorBackend::Docker { - return Err(LambdaServiceError::Internal { - message: "Docker execution is not implemented in this build".into(), - }); + let qualifier = qualifier.map(str::to_owned).or(embedded.clone()); + if let Some(qualifier) = &qualifier { + crate::resolver::Qualifier::parse(qualifier)?; } - let req = self - .build_invoke_request(&record, version, qualifier, payload) - .await?; + let record = self.get_record(&name)?; + if reference.starts_with("arn:") { + let expected = embedded.as_ref().map_or_else( + || record.arn.clone(), + |qualifier| format!("{}:{qualifier}", record.arn), + ); + if reference != expected { + return Err(invalid_parameter( + "Invocation ARN does not match this Lambda scope", + )); + } + } + Ok((record, qualifier)) + } - let (response, handle) = self - .work + async fn admit_invocation( + &self, + record: &FunctionRecord, + qualifier: Option<&str>, + payload: &[u8], + asynchronous: bool, + deadline: Option, + ) -> Result { + let version = resolve_version(record, qualifier)?; + let capture = self.build_invoke_request(record, version, qualifier, payload); + let request = if let Some(deadline) = deadline { + tokio::time::timeout_at(deadline, capture) + .await + .map_err(|_| { + LambdaServiceError::from(crate::executor::ExecutorError::DeadlineExceeded) + })?? + } else { + capture.await? + }; + self.work .submit( Arc::clone(&self.executor), - req, + request, record.reserved_concurrent_executions, - invocation_type == InvokeKind::Event, + asynchronous, + deadline, ) - .await?; - if invocation_type == InvokeKind::Event { - return Ok(InvokeOutcome::Async { - request_id: uuid::Uuid::new_v4().to_string(), - }); - } - let _cancel = crate::work::CancelOnDrop(handle); - let response = response .await - .map_err(|error| LambdaServiceError::Internal { - message: format!("Lambda worker cancelled: {error}"), - })??; - Ok(InvokeOutcome::Sync(response)) } /// Build the executor request from a resolved function/version record. @@ -1070,12 +1138,11 @@ impl RustackLambda { .or_insert_with(|| self.config.default_region.clone()); env.entry("AWS_DEFAULT_REGION".to_owned()) .or_insert_with(|| self.config.default_region.clone()); - Ok(InvokeRequest::builder() + let revision = ExecutionRevision::builder() .resource_id(record.resource_id) .revision_id(version.revision_id.clone()) .image_config(version.image_config.clone()) .ephemeral_storage_mb(version.ephemeral_storage_size) - .function_arn(function_arn) .function_name(record.name.clone()) .qualifier(version.version.clone()) .runtime(version.runtime.clone()) @@ -1085,9 +1152,13 @@ impl RustackLambda { .environment(env) .timeout(Duration::from_secs(u64::from(version.timeout))) .memory_mb(version.memory_size) - .payload(Bytes::copy_from_slice(payload)) - .capture_logs(false) - .build()) + .backend(self.config.executor) + .build(); + Ok(InvokeRequest::new( + Arc::new(revision), + function_arn, + Bytes::copy_from_slice(payload), + )) } // --------------------------------------------------------------- @@ -3053,6 +3124,15 @@ fn validate_s3_field(name: &str, value: &str, max_bytes: usize) -> Result<(), La } /// Build an `InvalidParameter` service error. +fn validate_invoke_payload(payload: &[u8], limit: usize) -> Result<(), LambdaServiceError> { + if payload.len() > limit { + return Err(LambdaServiceError::RequestTooLarge { + message: format!("Invocation payload exceeds {limit} bytes"), + }); + } + Ok(()) +} + fn invalid_parameter(message: &str) -> LambdaServiceError { LambdaServiceError::InvalidParameter { message: message.to_owned(), @@ -3256,6 +3336,51 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_should_reject_cross_scope_arns_and_conflicting_invocation_qualifiers() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("scoped")) + .await?; + let arn = provider.get_record("scoped")?.arn; + for reference in [ + arn.replace("us-east-1", "us-west-2"), + arn.replace("000000000000", "111111111111"), + arn.replace("arn:aws:", "arn:aws-cn:"), + ] { + assert!(matches!( + provider + .invoke(&reference, None, b"{}", InvokeKind::DryRun) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + } + assert!(matches!( + provider + .invoke( + &format!("{arn}:$LATEST"), + Some("1"), + b"{}", + InvokeKind::DryRun + ) + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(matches!( + provider + .invoke( + &format!("{arn}:$LATEST"), + Some("$LATEST"), + b"{}", + InvokeKind::DryRun + ) + .await?, + InvokeOutcome::DryRun + )); + Ok(()) + } + #[tokio::test] async fn test_should_preserve_invoked_alias_separately_from_resolved_version() -> Result<(), LambdaServiceError> { @@ -3286,9 +3411,9 @@ mod tests { let expected_arn = qualifier.map_or_else(|| record.arn.clone(), |q| format!("{}:{q}", record.arn)); assert_eq!(request.function_arn, expected_arn); - assert_eq!(request.qualifier, resolved); - assert_eq!(request.resource_id, record.resource_id); - assert_eq!(request.revision_id, version.revision_id); + assert_eq!(request.revision.qualifier, resolved); + assert_eq!(request.revision.resource_id, record.resource_id); + assert_eq!(request.revision.revision_id, version.revision_id); } Ok(()) } @@ -3321,8 +3446,9 @@ mod tests { .await?; version.ephemeral_storage_size = 4096; version.image_config = None; - assert_eq!(request.ephemeral_storage_mb, 2048); + assert_eq!(request.revision.ephemeral_storage_mb, 2048); let image = request + .revision .image_config .as_ref() .ok_or_else(|| invalid_parameter("Image configuration was lost"))?; @@ -3333,7 +3459,10 @@ mod tests { assert_eq!(image.command.as_deref(), Some([].as_slice())); assert_eq!(image.working_directory.as_deref(), Some("/function")); assert_eq!(request.payload, Bytes::from_static(b"{\"value\":1}")); - assert_ne!(request.ephemeral_storage_mb, version.ephemeral_storage_size); + assert_ne!( + request.revision.ephemeral_storage_mb, + version.ephemeral_storage_size + ); assert!(version.image_config.is_none()); Ok(()) } diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs index cda05c8b..4ebbab8c 100644 --- a/crates/rustack-lambda-core/src/work.rs +++ b/crates/rustack-lambda-core/src/work.rs @@ -1,36 +1,89 @@ -//! Bounded, supervised invocation lifecycle. No detached Event tasks. +//! Bounded invocation admission and a single owner for execution/cleanup futures. + use std::{ + collections::HashMap, sync::{ - Arc, + Arc, OnceLock, atomic::{AtomicBool, AtomicUsize, Ordering}, }, time::Duration, }; -use dashmap::DashMap; use tokio::{ - sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore, oneshot}, - task::{AbortHandle, JoinSet}, + sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinHandle, JoinSet}, + time::{Instant, timeout}, }; +use typed_builder::TypedBuilder; use crate::{ error::LambdaServiceError, - executor::{Executor, InvokeRequest, InvokeResponse}, + executor::{Executor, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::{InvocationContext, InvocationHandle, Settlement}, }; +const ACCEPTED_LIMIT: u32 = 128; +const EXECUTION_LIMIT: usize = 32; + #[derive(Debug)] pub(crate) struct WorkManager { - tasks: Mutex>, - accepting: AtomicBool, + accepting: Arc, capacity: Arc, global: Arc, - functions: DashMap>, + drain: Arc, + supervisor: OnceLock, +} + +#[derive(Debug)] +struct SupervisorHandle { + inbox: mpsc::Sender, + stop: watch::Sender, + state: watch::Receiver, + task: Mutex>>, } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SupervisorState { + Running, + Stopped { failed: bool }, +} + +#[derive(Debug, TypedBuilder)] +struct Submission { + executor: Arc, + request: InvokeRequest, + context: InvocationContext, + cancellation: watch::Sender, + reserved: usize, + asynchronous: bool, + slot: OwnedSemaphorePermit, + admitted: oneshot::Sender>, + response: oneshot::Sender>, + settlement: watch::Sender, +} + +#[derive(Debug, TypedBuilder)] +struct Supervisor { + inbox: mpsc::Receiver, + stop: watch::Receiver, + drain: Arc, + accepting: Arc, + global: Arc, + state: watch::Sender, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + cancellations: HashMap>, + #[builder(default)] + functions: HashMap>, +} + #[derive(Debug, Default)] struct FunctionCapacity { active: AtomicUsize, changed: Notify, } + #[derive(Debug)] struct FunctionPermit(Arc); impl Drop for FunctionPermit { @@ -39,8 +92,11 @@ impl Drop for FunctionPermit { self.0.changed.notify_waiters(); } } + +#[cfg(test)] #[derive(Debug)] -pub(crate) struct CancelOnDrop(pub(crate) AbortHandle); +pub(crate) struct CancelOnDrop(pub(crate) tokio::task::AbortHandle); +#[cfg(test)] impl Drop for CancelOnDrop { fn drop(&mut self) { self.0.abort(); @@ -52,9 +108,6 @@ fn unavailable(message: &str) -> LambdaServiceError { message: message.into(), } } -fn throttled() -> LambdaServiceError { - LambdaServiceError::TooManyRequests -} impl FunctionCapacity { fn try_acquire(self: &Arc, limit: usize) -> Result { @@ -66,9 +119,10 @@ impl FunctionCapacity { None } }) - .map_err(|_| throttled())?; + .map_err(|_| LambdaServiceError::TooManyRequests)?; Ok(FunctionPermit(Arc::clone(self))) } + async fn acquire(self: &Arc, limit: usize) -> FunctionPermit { loop { let changed = self.changed.notified(); @@ -85,244 +139,602 @@ impl FunctionCapacity { impl WorkManager { pub(crate) fn new() -> Self { Self { - tasks: Mutex::new(JoinSet::new()), - accepting: AtomicBool::new(true), - capacity: Arc::new(Semaphore::new(128)), - global: Arc::new(Semaphore::new(32)), - functions: DashMap::new(), + accepting: Arc::new(AtomicBool::new(true)), + capacity: Arc::new(Semaphore::new(ACCEPTED_LIMIT as usize)), + global: Arc::new(Semaphore::new(EXECUTION_LIMIT)), + drain: Arc::new(Notify::new()), + supervisor: OnceLock::new(), } } + fn supervisor(&self) -> &SupervisorHandle { + self.supervisor.get_or_init(|| { + let (inbox, receiver) = mpsc::channel(ACCEPTED_LIMIT as usize); + let (stop, cancellation) = watch::channel(false); + let (state, snapshot) = watch::channel(SupervisorState::Running); + let supervisor = Supervisor::builder() + .inbox(receiver) + .stop(cancellation) + .drain(Arc::clone(&self.drain)) + .accepting(Arc::clone(&self.accepting)) + .global(Arc::clone(&self.global)) + .state(state) + .build(); + let task = tokio::spawn(supervisor.run()); + SupervisorHandle { + inbox, + stop, + state: snapshot, + task: Mutex::new(Some(task)), + } + }) + } + pub(crate) async fn submit( &self, executor: Arc, - req: InvokeRequest, + request: InvokeRequest, reserved: Option, asynchronous: bool, - ) -> Result< - ( - oneshot::Receiver>, - AbortHandle, - ), - LambdaServiceError, - > { - executor.available()?; - let limit = usize::try_from(reserved.unwrap_or(8)) + caller_deadline: Option, + ) -> Result { + executor.available(&request)?; + if !self.accepting.load(Ordering::Acquire) { + return Err(unavailable("Lambda is quiescing")); + } + let reserved = usize::try_from(reserved.unwrap_or(8)) .map_err(|_| unavailable("Invalid reserved concurrency"))?; - if limit == 0 { - return Err(throttled()); + if reserved == 0 { + return Err(LambdaServiceError::TooManyRequests); } let slot = Arc::clone(&self.capacity) .try_acquire_owned() - .map_err(|_| throttled())?; - self.functions.retain(|_, capacity| { - Arc::strong_count(capacity) > 1 || capacity.active.load(Ordering::Acquire) != 0 + .map_err(|_| LambdaServiceError::TooManyRequests)?; + let total = request + .revision + .timeout + .saturating_add(Duration::from_secs(10)); + let deadline = Instant::now() + .checked_add(total) + .ok_or_else(|| unavailable("Invalid invocation budget"))?; + let deadline = caller_deadline.map_or(deadline, |caller| caller.min(deadline)); + let (cancellation, context) = InvocationContext::channel(deadline); + context.check()?; + let (admitted, accepted) = oneshot::channel(); + let (response, receiver) = oneshot::channel(); + let (settlement, settled) = watch::channel(Settlement::Pending); + let handle = + InvocationHandle::new(context.clone(), cancellation.clone(), receiver, settled); + let submission = Submission::builder() + .executor(executor) + .request(request) + .context(context.clone()) + .cancellation(cancellation) + .reserved(reserved) + .asynchronous(asynchronous) + .slot(slot) + .admitted(admitted) + .response(response) + .settlement(settlement) + .build(); + self.supervisor() + .inbox + .try_send(submission) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => LambdaServiceError::TooManyRequests, + mpsc::error::TrySendError::Closed(_) => unavailable("Lambda supervisor stopped"), + })?; + tokio::select! { + biased; + reason = context.cancelled() => Err(LambdaServiceError::from(reason.error())), + result = accepted => { + result.map_err(|_| unavailable("Lambda admission supervisor stopped"))??; + Ok(handle) + } + } + } + + pub(crate) fn cancel_all(&self) { + self.accepting.store(false, Ordering::Release); + if let Some(supervisor) = self.supervisor.get() { + supervisor.stop.send_replace(true); + } + self.drain.notify_one(); + } + + pub(crate) async fn quiesce(&self, budget: Duration) -> Result<(), LambdaServiceError> { + self.accepting.store(false, Ordering::Release); + self.drain.notify_one(); + let Some(supervisor) = self.supervisor.get() else { + return Ok(()); + }; + let mut state = supervisor.state.clone(); + let clean = { + let result = timeout( + budget, + state.wait_for(|state| matches!(state, SupervisorState::Stopped { .. })), + ) + .await; + match result { + Ok(Ok(state)) => matches!(*state, SupervisorState::Stopped { failed: false }), + _ => false, + } + }; + if !clean { + supervisor.stop.send_replace(true); + return Err(unavailable( + "Lambda quiesce did not confirm all invocation cleanup", + )); + } + let mut task = supervisor.task.lock().await; + if let Some(task) = task.as_mut() { + task.await + .map_err(|_| unavailable("Lambda supervisor failed"))?; + } + task.take(); + Ok(()) + } +} + +impl Drop for WorkManager { + fn drop(&mut self) { + // Fallback only: the detached actor keeps cleanup futures alive until settlement. + // Normal provider shutdown awaits quiescence before dropping this owner. + self.cancel_all(); + } +} + +impl Supervisor { + fn cancel_all(&self) { + for cancellation in self.cancellations.values() { + cancellation.send_replace(true); + } + } + + fn admit(&mut self, submission: Submission) { + let result = if self.accepting.load(Ordering::Acquire) { + submission.context.check().map_err(LambdaServiceError::from) + } else { + Err(unavailable("Lambda is quiescing")) + }; + if let Err(error) = result { + reject(submission, error); + return; + } + self.functions.retain(|_, function| { + Arc::strong_count(function) > 1 || function.active.load(Ordering::Acquire) != 0 }); let function = Arc::clone( self.functions - .entry(req.function_name.clone()) - .or_insert_with(|| Arc::new(FunctionCapacity::default())) - .value(), + .entry(submission.request.revision.function_name.clone()) + .or_default(), ); - let permits = if asynchronous { + let permits = if submission.asynchronous { None } else { - Some(( - function.try_acquire(limit)?, - Arc::clone(&self.global) - .try_acquire_owned() - .map_err(|_| throttled())?, - )) - }; - let mut tasks = self.tasks.lock().await; - while let Some(result) = tasks.try_join_next() { - if let Err(error) = result { - tracing::error!(%error, "Lambda worker failed"); + match function + .try_acquire(submission.reserved) + .and_then(|function| { + Arc::clone(&self.global) + .try_acquire_owned() + .map(|global| (function, global)) + .map_err(|_| LambdaServiceError::TooManyRequests) + }) { + Ok(permits) => Some(permits), + Err(error) => { + reject(submission, error); + return; + } } - } - if !self.accepting.load(Ordering::Acquire) { - return Err(unavailable("Lambda is quiescing")); - } - let (tx, rx) = oneshot::channel(); + }; + let Submission { + executor, + request, + context, + cancellation, + reserved, + slot, + admitted, + response, + settlement, + .. + } = submission; let global = Arc::clone(&self.global); - let handle = tasks.spawn(async move { - let _slot = slot; - let result = execute(executor, req, function, global, limit, permits).await; - match &result { - Ok(_) => tracing::debug!("Lambda invocation completed"), - Err(error) => tracing::warn!(%error, "Lambda invocation failed"), - } - if tx.send(result).is_err() { - tracing::debug!("Lambda invocation completed without a waiting caller"); - } + let worker = self.jobs.spawn(async move { + let result = execute( + executor, request, context, function, global, reserved, permits, + ) + .await; + let clean = !matches!(&result, Err(ExecutorError::CleanupFailed(_))); + // Backend cleanup completes before permits and the admission slot are released. + drop(slot); + settlement.send_replace(if clean { + Settlement::Clean + } else { + Settlement::Uncertain + }); + let _ = response.send(result); + clean }); - Ok((rx, handle)) + self.cancellations.insert(worker.id(), cancellation); + let _ = admitted.send(Ok(())); } - pub(crate) async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { - self.accepting.store(false, Ordering::Release); - // Own the JoinSet across awaits so cancellation of quiesce drops/aborts every task. - let mut tasks = std::mem::take(&mut *self.tasks.lock().await); - let drained = tokio::time::timeout(timeout, async { - while let Some(result) = tasks.join_next().await { - if let Err(error) = result { - return Err(unavailable(&format!("Lambda task failed: {error}"))); - } + async fn run(mut self) { + let mut failed = false; + loop { + if !self.accepting.load(Ordering::Acquire) || *self.stop.borrow() { + self.inbox.close(); } - // Includes workers aborted by a previously cancelled quiesce future. - let _all_capacity = Arc::clone(&self.capacity) - .acquire_many_owned(128) - .await - .map_err(|_| unavailable("Lambda accepted-work capacity closed"))?; - Ok(()) - }) - .await; - match drained { - Ok(Ok(())) => Ok(()), - result => { - tasks.abort_all(); - while let Some(result) = tasks.join_next().await { - if let Err(error) = result { - tracing::warn!(%error, "Lambda task cancelled during quiesce"); + if *self.stop.borrow() { + self.cancel_all(); + } + if self.inbox.is_closed() && self.inbox.is_empty() && self.jobs.is_empty() { + break; + } + tokio::select! { + biased; + _ = self.stop.changed(), if !*self.stop.borrow() => { self.accepting.store(false, Ordering::Release); } + () = self.drain.notified() => {} + completed = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => { + match completed { + Some(Ok((id, clean))) => { + self.cancellations.remove(&id); + if !clean { failed = true; } + } + Some(Err(error)) => { + self.cancellations.remove(&error.id()); + failed = true; + tracing::error!("Lambda invocation worker terminated without confirmed settlement"); + } + None => {} } + if failed { self.accepting.store(false, Ordering::Release); self.cancel_all(); } } - match result { - Ok(Err(error)) => Err(error), - _ => Err(unavailable( - "Lambda quiesce deadline exceeded; remaining invocations cancelled", - )), + submission = self.inbox.recv(), if !self.inbox.is_closed() || !self.inbox.is_empty() => { + if let Some(submission) = submission { self.admit(submission); } } } } + self.accepting.store(false, Ordering::Release); + self.state.send_replace(SupervisorState::Stopped { failed }); } } +fn reject(submission: Submission, error: LambdaServiceError) { + let Submission { + slot, + admitted, + response, + settlement, + .. + } = submission; + drop(slot); + settlement.send_replace(Settlement::Clean); + let _ = response.send(Err(ExecutorError::Cancelled)); + let _ = admitted.send(Err(error)); +} + async fn execute( executor: Arc, - req: InvokeRequest, + request: InvokeRequest, + context: InvocationContext, function: Arc, global: Arc, limit: usize, permits: Option<(FunctionPermit, OwnedSemaphorePermit)>, -) -> Result { - let (_function, _global) = match permits { - Some(permits) => permits, - None => ( - function.acquire(limit).await, - global - .acquire_owned() - .await - .map_err(|_| unavailable("Lambda execution closed"))?, - ), +) -> Result { + let acquired = async { + match permits { + Some(permits) => Ok(permits), + None => Ok(( + function.acquire(limit).await, + global + .acquire_owned() + .await + .map_err(|_| ExecutorError::Cancelled)?, + )), + } }; - let deadline = req.timeout.saturating_add(Duration::from_secs(10)); - match tokio::time::timeout(deadline, executor.invoke(req)).await { - Ok(result) => result.map_err(LambdaServiceError::from), - Err(_) => Err(unavailable("Lambda invocation deadline exceeded")), - } + let (_function, _global) = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = acquired => result?, + }; + context.check()?; + executor.invoke(request, context).await } #[cfg(test)] mod tests { + use std::sync::atomic::AtomicBool; + use async_trait::async_trait; + use tokio::sync::Semaphore; use super::*; - use crate::executor::ExecutorError; #[derive(Debug)] - struct WaitingExecutor; + struct ProbeExecutor { + started: Semaphore, + cancelling: Semaphore, + cleanup: Semaphore, + hold_cleanup: AtomicBool, + calls: AtomicUsize, + } + impl ProbeExecutor { + fn new(hold_cleanup: bool) -> Self { + Self { + started: Semaphore::new(0), + cancelling: Semaphore::new(0), + cleanup: Semaphore::new(0), + hold_cleanup: AtomicBool::new(hold_cleanup), + calls: AtomicUsize::new(0), + } + } + async fn started(&self, count: u32) { + timeout(Duration::from_secs(2), self.started.acquire_many(count)) + .await + .unwrap() + .unwrap() + .forget(); + } + async fn cancelling(&self) { + timeout(Duration::from_secs(2), self.cancelling.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + } + } #[async_trait] - impl Executor for WaitingExecutor { - async fn invoke(&self, _request: InvokeRequest) -> Result { - std::future::pending().await + impl Executor for ProbeExecutor { + async fn invoke( + &self, + _request: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.calls.fetch_add(1, Ordering::AcqRel); + self.started.add_permits(1); + let reason = context.cancelled().await; + self.cancelling.add_permits(1); + if self.hold_cleanup.load(Ordering::Acquire) { + self.cleanup + .acquire() + .await + .map_err(|_| ExecutorError::CleanupFailed("Probe cleanup closed".into()))? + .forget(); + } + Err(reason.error()) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) } - async fn shutdown(&self) {} } fn request(name: &str) -> InvokeRequest { let mut request = InvokeRequest::fixture(); - request.function_name = name.into(); - request.timeout = Duration::from_mins(1); + let revision = Arc::make_mut(&mut request.revision); + revision.function_name = name.into(); + revision.timeout = Duration::from_secs(60); request } + #[tokio::test] - async fn test_should_bound_event_set_before_accepting_and_release_on_cancel() { + async fn test_should_bound_event_admission_and_cancel_queued_work_without_starting_it() { let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); for _ in 0..128 { - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor.clone(), request("bounded"), None, true, None) .await - .unwrap(); + .unwrap() + .detach(); } + executor.started(8).await; + assert_eq!(executor.calls.load(Ordering::Acquire), 8); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor.clone(), request("bounded"), None, true, None) .await, Err(LambdaServiceError::TooManyRequests) )); - tokio::task::yield_now().await; - assert!( - work.functions - .get("bounded") - .unwrap() - .active - .load(Ordering::Acquire) - <= 8 - ); - assert!(work.global.available_permits() >= 24); - assert!(work.quiesce(Duration::ZERO).await.is_err()); + work.cancel_all(); + work.quiesce(Duration::from_secs(2)).await.unwrap(); assert_eq!(work.capacity.available_permits(), 128); assert_eq!(work.global.available_permits(), 32); - assert_eq!( - work.functions - .get("bounded") - .unwrap() - .active - .load(Ordering::Acquire), - 0 - ); + assert_eq!(executor.calls.load(Ordering::Acquire), 8); assert!( - work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + work.submit(executor, request("bounded"), None, true, None) .await .is_err() ); } + #[tokio::test] - async fn test_should_enforce_function_global_and_zero_reserved_execution_limits() { + async fn test_should_enforce_shared_function_global_and_zero_reserved_execution_limits() { let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("zero"), Some(0), true) + work.submit(executor.clone(), request("zero"), Some(0), true, None) .await, Err(LambdaServiceError::TooManyRequests) )); + let mut handles = Vec::new(); for name in ["a", "b", "c", "d"] { for _ in 0..8 { - work.submit(Arc::new(WaitingExecutor), request(name), None, false) - .await - .unwrap(); + handles.push( + work.submit(executor.clone(), request(name), None, false, None) + .await + .unwrap(), + ); } assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request(name), None, false) + work.submit(executor.clone(), request(name), None, false, None) .await, Err(LambdaServiceError::TooManyRequests) )); } + assert_eq!(work.global.available_permits(), 0); assert!(matches!( - work.submit(Arc::new(WaitingExecutor), request("e"), Some(32), false) + work.submit(executor, request("e"), Some(32), false, None) .await, Err(LambdaServiceError::TooManyRequests) )); - assert_eq!(work.global.available_permits(), 0); - assert!(work.quiesce(Duration::ZERO).await.is_err()); + drop(handles); + work.quiesce(Duration::from_secs(2)).await.unwrap(); assert_eq!(work.global.available_permits(), 32); } + #[tokio::test] - async fn test_should_release_capacity_when_sync_caller_cancels() { + async fn test_should_hold_permits_until_cancelled_backend_cleanup_is_confirmed() { let work = WorkManager::new(); - let (_response, handle) = work - .submit(Arc::new(WaitingExecutor), request("cancel"), None, false) + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit(executor.clone(), request("cleanup"), Some(1), false, None) + .await + .unwrap(); + executor.started(1).await; + handle.cancel(); + executor.cancelling().await; + assert_eq!(work.capacity.available_permits(), 127); + assert_eq!(work.global.available_permits(), 31); + assert!(matches!( + work.submit(executor.clone(), request("cleanup"), Some(1), false, None) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + executor.cleanup.add_permits(1); + handle + .cancel_and_wait(Instant::now() + Duration::from_secs(2)) .await .unwrap(); - drop(CancelOnDrop(handle)); - assert!(work.quiesce(Duration::from_secs(1)).await.is_err()); assert_eq!(work.capacity.available_permits(), 128); assert_eq!(work.global.available_permits(), 32); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + } + + #[tokio::test] + async fn test_should_keep_cleanup_owned_after_handle_deadline_and_aborted_quiesce_waiter() { + let work = Arc::new(WorkManager::new()); + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit(executor.clone(), request("retained"), None, false, None) + .await + .unwrap(); + executor.started(1).await; + assert!(matches!( + handle.cancel_and_wait(Instant::now()).await, + Err(ExecutorError::CleanupFailed(_)) + )); + executor.cancelling().await; + let waiting = Arc::clone(&work); + let task = tokio::spawn(async move { waiting.quiesce(Duration::from_secs(30)).await }); + tokio::task::yield_now().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(work.capacity.available_permits(), 127); + executor.cleanup.add_permits(1); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[tokio::test] + async fn test_should_cancel_on_handle_drop_and_reject_already_expired_admission() { + let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(false)); + assert!(matches!( + work.submit( + executor.clone(), + request("expired"), + None, + false, + Some(Instant::now()) + ) + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + let handle = work + .submit(executor.clone(), request("drop"), None, false, None) + .await + .unwrap(); + executor.started(1).await; + drop(handle); + executor.cancelling().await; + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[tokio::test(start_paused = true)] + async fn test_should_return_at_caller_deadline_while_retaining_backend_cleanup_capacity() { + let work = WorkManager::new(); + let executor = Arc::new(ProbeExecutor::new(true)); + let handle = work + .submit( + executor.clone(), + request("deadline"), + None, + false, + Some(Instant::now() + Duration::from_secs(1)), + ) + .await + .unwrap(); + executor.started(1).await; + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!( + handle.wait().await, + Err(ExecutorError::DeadlineExceeded) + )); + executor.cancelling().await; + assert_eq!(work.global.available_permits(), 31); + assert_eq!(work.capacity.available_permits(), 127); + assert!(work.quiesce(Duration::ZERO).await.is_err()); + executor.cleanup.add_permits(1); + work.quiesce(Duration::from_secs(2)).await.unwrap(); + assert_eq!(work.global.available_permits(), 32); + assert_eq!(work.capacity.available_permits(), 128); + } + + #[derive(Debug)] + struct PanickingExecutor; + #[async_trait] + impl Executor for PanickingExecutor { + async fn invoke( + &self, + _request: InvokeRequest, + _context: InvocationContext, + ) -> Result { + panic!("intentional supervised panic"); + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } + } + #[tokio::test] + async fn test_should_never_report_successful_quiescence_after_a_worker_panic() { + let work = WorkManager::new(); + let handle = work + .submit( + Arc::new(PanickingExecutor), + request("panic"), + None, + false, + None, + ) + .await + .unwrap(); + assert!(matches!( + handle.wait().await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(work.quiesce(Duration::from_secs(2)).await.is_err()); + assert!( + work.submit( + Arc::new(ProbeExecutor::new(false)), + request("after-panic"), + None, + false, + None + ) + .await + .is_err() + ); } } diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 8187d059..90808729 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -93,6 +93,36 @@ An Image variant carries ResolvedImage, never a mutable tag alone. Its validated Artifact replacement does not change public AWS wire shapes. Native and Squib accept only Zip and consume its lease; Docker accepts Image in M0 and adds Zip in M4. The resolved architecture is a closed x86_64/arm64 enum, not an unchecked list. Capture validates numeric settings and full image overrides before work admission. No per-invoke image inspection or pulling is introduced by this contract. +### 2.6 Execution revision ownership + +ExecutionRevision groups resource/revision identity, resolved version, artifact, architecture, runtime/handler, complete image overrides, environment, timeout, memory, ephemeral storage and selected backend policy. InvokeRequest carries an Arc of that captured revision plus the requested invocation ARN, payload Bytes and log-capture flag. Cloning a request shares revision storage; aliases and subsequent function updates cannot modify captured configuration. Runtime API metadata uses the requested ARN while backend results use the revision's resolved version. Warm identity includes the captured backend policy as well as resource/configuration identity. Debug on both types redacts user environment, argv, code and payload data. Native/Squib consume the same revision object. + +### 2.7 Cancellable invocation and supervisor contract + +Provider start_invocation returns an InvocationHandle after shared admission. The handle exposes its attempt UUID, wait, cancel, cancel_and_wait(cleanup deadline) and detach. Waiting returns typed ExecutorError outcomes; the existing HTTP Invoke maps these through its existing Lambda service error conversion. Event Invoke detaches only after admission succeeds. Dropping an attached handle signals cancellation, never aborts the worker. A wait deadline may return before cleanup; cancel_and_wait reports success only after cleanup and permit release. Its timeout returns CleanupFailed while ownership remains with the supervisor. + +InvocationContext carries the same attempt UUID to the backend/Runtime API, an absolute monotonic deadline and a dedicated cancellation watch channel. The total deadline starts at admission, includes queue/cold start and is bounded by any caller Task deadline. Handler timeout remains separate. Admission checks backend availability against the captured request before accepting work. Unsupported backend/artifact combinations cannot return an accepted Event response. + +A lazily started WorkSupervisor actor owns the bounded submission mailbox, JoinSet and cancellation registry. Existing limits remain128 accepted jobs,32 executing jobs and default8 per logical function across versions. A priority shutdown signal cannot be starved by submissions. Draining closes admission, finishes accepted work, and requests cancellation if its deadline expires. The actor continues owning cleanup after a quiesce waiter is dropped or times out. Worker panic or uncertain cleanup stops admission and prevents successful quiescence. Supervisor Drop signals stop; normal shutdown awaits settlement. + +Native cancellation kills and reaps the owned child, drains log tasks and releases its process reservation before signaling clean settlement. Successful instances rejected by a full/disabled warm pool also retire before invocation settlement. Admission rejects declared host-architecture mismatches before accepting an Event; executable-file validation remains asynchronous. Executor Drop initiates retained process cleanup rather than aborting child supervisors. + +Squib cancellation must retain its startup/VM ownership and await VM shutdown; never cancel/drop SquibBuilder::spawn or Squib::shutdown as a cleanup strategy. Inspection of the pinned Squib0.2.0 facade shows startup and shutdown temporarily own join handles across awaits, so dropping those futures is not a join/cleanup proof. A caller deadline does not authorize losing those futures. Squib guest cancellation retires/poisons that VM rather than reusing a possibly busy guest. Idle and busy VM owners use the same retained shutdown path, including executor Drop. Cleanup owners capture the parent Tokio runtime handle so transfer does not require Drop to run in an entered runtime context; normal service shutdown must await cleanup before stopping that runtime. The configured Squib shutdown budget bounds waiters, never the owned shutdown future. Expiry returns CleanupFailed, retains shutdown ownership and prevents successful quiescence. + +```text +Caller / HTTP WorkSupervisor actor Backend supervisor + │ start ───────────────▶│ reserve / capture attempt │ + │◀── handle + UUID ─────│ spawn owned worker ──────────────────▶│ + │ drop/cancel ──────────┼── watch signal ──────────────────────▶│ + │ wait deadline → error │ worker remains owned │ + │ cancel_and_wait ─────▶│ kill / shutdown / reap + │ │◀── confirmed cleanup ────────────────│ + │ │ release execution + accepted permits │ + │◀── clean settlement ─│ │ + │ cleanup deadline │ retain worker or backend quarantine │ + │◀── CleanupFailed ────│ prevent false successful quiescence │ +``` + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 33da02f9..5ab4c2ac 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -140,7 +140,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| 1.1 | In progress | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented and independently reviewed. Standard workspace suite: 1,752 passing tests; Lambda: 203 unit tests, twelve doctests, six native/DryRun checks. ExecutionRevision ownership and cooperative cancellation remain required. B01/B03 remain open. | +| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task 1.1; P0 does not advertise Docker execution as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task1.2, extracting the shared runtime session and bounded relay protocol; B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index e0c474cf..286860a8 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -43,6 +43,18 @@ The independent review against commit `7915281` identified two implementation fi The slice also preserves extracted ZIP roots in Native warm keys and retains code in Native child supervision through exit. The standard workspace suite passes 1,752 tests (392 preexisting ignored), including 203 Lambda core tests; twelve Lambda doctests and six native/DryRun cases pass. The separately gated Squib case selected by the native Make target is not Squib execution evidence. New winnow/rstest dependency use introduces no audit finding; the same 16 baseline advisories remain B01. Strict production boundary lint debt remains B03; no ignore was added. ExecutionRevision ownership, cooperative invocation cancellation, deployment resolution and Docker remain required before P1/M0 completion. +## P1 task1.1 cancellation/revision slice review + +The independent read-only review found three lifecycle/admission defects. All were fixed; its final recheck found no further concrete in-slice finding. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W13 | P1; `crates/rustack-lambda-core/src/executor/squib.rs:170` | Dropping an idle Squib executor relied on the facade's best-effort Drop and detached VMM handle. OwnedVm now transfers idle/busy VM shutdown into retained CleanupTask, using the captured parent Tokio handle | Generic owned-VM seam test drops off-runtime, expires a cleanup waiter and proves allocation retention until shutdown completes. This is not actual VM execution evidence | +| W14 | P2; `crates/rustack-lambda-core/src/executor/instance.rs:276` | A full/disabled warm pool dropped a successful instance before reaping. Release now returns rejected ownership; Invoke awaits API shutdown and backend termination before settlement | Two overlapping real Native calls with warm retention0/1 require exactly0/1 surviving child after both responses | +| W15 | P2; `crates/rustack-lambda-core/src/executor/native.rs:129` | Known host-architecture mismatch failed only after Event acceptance. Availability now checks architecture before work admission | Real fixture Event rejects the mismatch, then quiesces cleanly without starting the foreign bootstrap | + +ExecutionRevision uses shared Arc ownership; InvocationHandle and the bounded work actor preserve UUID/deadline/cancellation through backend cleanup. Waiter cancellation never aborts backend workers. Native child supervisors retain artifact leases and process reservations through reaping, including executor Drop. Tests cover late-result rejection, cleanup timeout/aborted waiters, worker panic,128 accepted/32 running/default8 function limits and permit release after cleanup. The workspace suite passes1,767 tests (392 preexisting ignored), including215 Lambda core tests and five real Native fixture tests. Lambda doctests, standard workspace build/Clippy/rustdoc and the native SDK target pass. Its separately gated Squib test remains excluded as VM evidence. Full Docker and workflow milestones, plus B01/B03, remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 1b1afbb3..8fddff11 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -100,6 +100,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [artifact representation and ownership sequence](./rustack-workflows-20-lambda-runtime-design.md#25-captured-artifact-representation). - Date: 2026-10-25. +## D14 — Separate cancellation, caller response and resource settlement + +- Alternatives: abort backend futures at the HTTP/Task deadline; release capacity as soon as a caller returns; rely on Squib's best-effort RAII shutdown. +- Decision: immutable Arc execution revisions, one attempt UUID/deadline, attached invocation handles and a bounded work actor. Backend cleanup survives dropped/expired waiters, and clean settlement follows resource cleanup and permit release. Idle VM Drop and Native warm-retention rejection follow the same ownership rule. +- Why: a caller timeout does not prove a process/VM has stopped. The pinned Squib0.2.0 startup/shutdown futures temporarily own VMM join handles; its raw Drop only attempts a shutdown message. Cancelling those futures cannot establish cleanup. Capture the parent Tokio handle for retained cleanup and shut down that runtime only after service settlement. +- Pinned by: [execution revision](./rustack-workflows-20-lambda-runtime-design.md#26-execution-revision-ownership), [cancellation and supervisor contract](./rustack-workflows-20-lambda-runtime-design.md#27-cancellable-invocation-and-supervisor-contract). +- Date: 2026-10-25. + ## Glossary | Term | Meaning in this specification | diff --git a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs index 58fa5fbc..09393f34 100644 --- a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs +++ b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs @@ -213,3 +213,162 @@ async fn test_should_cancel_and_reap_native_event_on_quiesce_deadline() { ); provider.shutdown().await; } + +#[cfg(unix)] +#[tokio::test] +async fn test_should_reap_cancelled_invocations_and_share_attempt_identity_with_runtime_api() { + let root = tempfile::tempdir().unwrap(); + let started = root.path().join("started"); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + let mut input = create("cancellable"); + input.timeout = Some(120); + input.environment = Some(Environment { + variables: Some(HashMap::from([ + ("SLEEP_SECS".into(), "60".into()), + ( + "STARTED_FILE".into(), + started.to_string_lossy().into_owned(), + ), + ])), + }); + provider.create_function(input).await.unwrap(); + let handle = provider + .start_invocation("revision-test", None, b"{}", None) + .await + .unwrap(); + let pid = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match tokio::fs::read_to_string(&started).await { + Ok(pid) if !pid.is_empty() => break pid, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => {} + Err(error) => panic!("Read start marker: {error}"), + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + handle + .cancel_and_wait(tokio::time::Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(pid.trim()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!( + !status.success(), + "cancel acknowledgement must follow process reaping" + ); + + provider + .update_function_configuration( + "revision-test", + &UpdateFunctionConfigurationInput { + environment: Some(Environment { + variables: Some(HashMap::new()), + }), + ..Default::default() + }, + ) + .unwrap(); + let handle = provider + .start_invocation("revision-test", None, br#"{"value":42}"#, None) + .await + .unwrap(); + let id = handle.id().to_string(); + let result = handle.wait().await.unwrap(); + let response: serde_json::Value = serde_json::from_slice(&result.payload).unwrap(); + assert_eq!(response["request_id"], id); + assert_eq!(response["echo"]["value"], 42); + assert_ne!(response["pid"].as_u64().unwrap().to_string(), pid.trim()); + provider.quiesce(Duration::from_secs(2)).await.unwrap(); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_reap_successful_instances_before_returning_when_warm_retention_is_full() { + for max_warm_instances in [0, 1] { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + max_warm_instances, + ..Default::default() + }, + ); + let mut input = create("retirement"); + input.environment = Some(Environment { + variables: Some(HashMap::from([("SLEEP_SECS".into(), "1".into())])), + }); + provider.create_function(input).await.unwrap(); + let first = provider + .start_invocation("revision-test", None, b"{}", None) + .await + .unwrap(); + let second = provider + .start_invocation("revision-test", None, b"{}", None) + .await + .unwrap(); + let (first, second) = tokio::join!(first.wait(), second.wait()); + let mut alive = 0; + for result in [first.unwrap(), second.unwrap()] { + let response: serde_json::Value = serde_json::from_slice(&result.payload).unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(response["pid"].as_u64().unwrap().to_string()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + alive += usize::from(status.success()); + } + assert_eq!( + alive, max_warm_instances, + "nonretained instances must already be reaped" + ); + provider.shutdown().await; + } +} + +#[tokio::test] +async fn test_should_reject_native_event_architecture_mismatch_before_acceptance() { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + let mut input = create("mismatched"); + input.architectures = Some(vec![ + if cfg!(target_arch = "aarch64") { + "x86_64" + } else { + "arm64" + } + .into(), + ]); + provider.create_function(input).await.unwrap(); + assert!( + provider + .invoke("revision-test", None, b"{}", InvokeKind::Event) + .await + .is_err() + ); + provider.quiesce(Duration::from_secs(1)).await.unwrap(); + provider.shutdown().await; +} From 4da7df30001be3a20e1dc73a92cb2edb31640298 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 17:28:22 -0700 Subject: [PATCH 005/106] phase 1: add bounded shared Lambda control protocol Specify and implement typed environment-scoped CBOR frames with allocation preflight, validated metadata, redacted binary payloads and absolute IO deadlines. Add the host-portable runtime crate and discoverable test-lambda-runtime target; session and relay consumers remain next. Independent codec review W16 fixed/rechecked. Workspace build, 1806 tests, fmt, Clippy and rustdoc pass; 39 runtime tests, eight doctests and strict runtime boundary checks pass. New std-only proptest dev dependencies add no audit finding. The same 16 B01 baseline advisories and B03 debt remain mandatory P1 gates; no full P1/M0 completion is claimed. --- Cargo.lock | 45 ++ Cargo.toml | 2 + Makefile | 10 + crates/rustack-lambda-runtime/Cargo.toml | 24 + crates/rustack-lambda-runtime/src/lib.rs | 9 + .../src/protocol/mod.rs | 271 +++++++++++ .../src/protocol/preflight.rs | 140 ++++++ .../src/protocol/tests.rs | 423 ++++++++++++++++++ .../src/protocol/types.rs | 348 ++++++++++++++ specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 30 ++ .../rustack-workflows-70-verification-plan.md | 4 +- specs/rustack-workflows-90-impl-plan.md | 3 +- specs/rustack-workflows-93-review.md | 10 + 14 files changed, 1318 insertions(+), 3 deletions(-) create mode 100644 crates/rustack-lambda-runtime/Cargo.toml create mode 100644 crates/rustack-lambda-runtime/src/lib.rs create mode 100644 crates/rustack-lambda-runtime/src/protocol/mod.rs create mode 100644 crates/rustack-lambda-runtime/src/protocol/preflight.rs create mode 100644 crates/rustack-lambda-runtime/src/protocol/tests.rs create mode 100644 crates/rustack-lambda-runtime/src/protocol/types.rs diff --git a/Cargo.lock b/Cargo.lock index edd0bb5e..121a6df4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2780,6 +2780,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand", + "rand_xorshift", + "regex-syntax", + "unarray", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -2896,6 +2912,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3848,6 +3873,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-lambda-runtime" +version = "0.9.0" +dependencies = [ + "bytes", + "ciborium", + "proptest", + "rstest", + "serde", + "thiserror", + "tokio", + "uuid", +] + [[package]] name = "rustack-logs-core" version = "0.11.0" @@ -5414,6 +5453,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 591a2d15..0c1c578f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ rustack-sns-core = { version = "0.11.0", path = "crates/rustack-sns-core" } rustack-lambda-model = { version = "0.9.0", path = "crates/rustack-lambda-model" } rustack-lambda-http = { version = "0.11.0", path = "crates/rustack-lambda-http" } rustack-lambda-core = { version = "0.11.0", path = "crates/rustack-lambda-core" } +rustack-lambda-runtime = { version = "0.9.0", path = "crates/rustack-lambda-runtime" } rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } rustack-logs-http = { version = "0.11.0", path = "crates/rustack-logs-http" } @@ -176,6 +177,7 @@ form_urlencoded = "1.2" # Testing rstest = { version = "0.28", default-features = false } +proptest = { version = "1.12", default-features = false, features = ["std"] } tokio-test = "0.4" aws-config = { version = "1.12", default-features = false, features = [ "credentials-process", diff --git a/Makefile b/Makefile index 62309eea..7e413ff2 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,16 @@ smoke: integration: @cargo test -p rustack-integration -- --ignored +# Shared runtime protocol tests and strict production boundary checks. +test-lambda-runtime: + @cargo test -p rustack-lambda-runtime + @cargo clippy -p rustack-lambda-runtime --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-runtime + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml new file mode 100644 index 00000000..98ac428e --- /dev/null +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "rustack-lambda-runtime" +description = "Shared Lambda runtime session and bounded relay protocol for Rustack" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +bytes = { workspace = true, features = ["serde"] } +ciborium = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs new file mode 100644 index 00000000..531a4ed1 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -0,0 +1,9 @@ +//! Shared, host-portable Lambda runtime contracts. +//! +//! The control protocol validates untrusted CBOR before deserialization and +//! preserves raw payload bytes without logging their contents. This crate has +//! no dependency on a function provider, VM implementation or Docker client. + +#![forbid(unsafe_code)] + +pub mod protocol; diff --git a/crates/rustack-lambda-runtime/src/protocol/mod.rs b/crates/rustack-lambda-runtime/src/protocol/mod.rs new file mode 100644 index 00000000..7846abdc --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/mod.rs @@ -0,0 +1,271 @@ +//! Length-prefixed, bounded CBOR for one environment's attach connection. +//! +//! A partial read/write failure invalidates the connection. Callers own the IO +//! task and must retire that transport rather than retrying a partially read +//! frame or replaying an invocation. + +mod preflight; +#[cfg(test)] +mod tests; +mod types; + +use std::{io, num::TryFromIntError}; + +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + time::{Instant, timeout_at}, +}; +pub use types::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, FunctionArn, Message, Outcome, + Payload, TraceId, +}; + +/// Maximum event or result payload, in bytes. +pub const MAX_PAYLOAD_BYTES: usize = 6 * 1024 * 1024; +/// Maximum CBOR envelope metadata, excluding raw payload bytes. +pub const MAX_METADATA_BYTES: usize = 64 * 1024; +/// Maximum frame body, excluding its four-byte prefix. +pub const MAX_FRAME_BYTES: usize = MAX_PAYLOAD_BYTES + MAX_METADATA_BYTES; + +/// Protocol failures contain classes and IO sources, never invalid peer values. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ProtocolError { + /// A declared/allotted length exceeds a protocol budget. + #[error("control protocol size budget exceeded")] + TooLarge, + /// The byte structure is invalid, unsupported, duplicated or truncated. + #[error("invalid control CBOR structure")] + InvalidStructure, + /// The valid CBOR does not match the typed message schema. + #[error("invalid control message schema")] + InvalidSchema, + /// A version other than one was supplied. + #[error("unsupported control protocol version")] + UnsupportedVersion, + /// A validated primitive has invalid syntax or range. + #[error("invalid control metadata")] + InvalidMetadata, + /// IO failed, including truncated transport input. + #[error("control transport IO failed")] + Io(#[from] io::Error), + /// An absolute IO deadline expired; the transport must be retired. + #[error("control transport deadline exceeded")] + Deadline, +} + +impl From for ProtocolError { + fn from(_: TryFromIntError) -> Self { + Self::TooLarge + } +} + +/// A validated, environment-scoped control envelope. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", try_from = "RawFrame")] +#[non_exhaustive] +pub struct Frame { + version: u32, + environment: EnvironmentId, + message: Message, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RawFrame { + version: u32, + environment: EnvironmentId, + message: Message, +} + +impl TryFrom for Frame { + type Error = ProtocolError; + + fn try_from(raw: RawFrame) -> Result { + if raw.version != 1 { + return Err(ProtocolError::UnsupportedVersion); + } + Ok(Self::new(raw.environment, raw.message)) + } +} + +impl Frame { + /// Wrap a typed message in protocol version one. + /// + /// ``` + /// use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Ready); + /// assert_eq!(frame.message(), &Message::Ready); + /// ``` + #[must_use] + pub fn new(environment: EnvironmentId, message: Message) -> Self { + Self { + version: 1, + environment, + message, + } + } + + /// Identity that must match the connection's owned environment. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let id = EnvironmentId::default(); + /// assert_eq!(Frame::new(id, Message::Close).environment(), id); + /// ``` + #[must_use] + pub fn environment(&self) -> EnvironmentId { + self.environment + } + + /// Borrow the typed message without cloning its payload. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Close); + /// assert!(matches!(frame.message(), Message::Close)); + /// ``` + #[must_use] + pub fn message(&self) -> &Message { + &self.message + } + + /// Move the message into the owning session or transport actor. + /// + /// ``` + /// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; + /// let frame = Frame::new(EnvironmentId::default(), Message::Close); + /// assert_eq!(frame.into_message(), Message::Close); + /// ``` + #[must_use] + pub fn into_message(self) -> Message { + self.message + } +} + +/// Encode one complete frame, including the big-endian length prefix. +/// +/// # Errors +/// Rejects excessive metadata or an encoding/size failure. +/// +/// ``` +/// # use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message, encode}; +/// let bytes = encode(&Frame::new(EnvironmentId::default(), Message::Ready))?; +/// assert!(bytes.len() > 4); +/// # Ok::<(), rustack_lambda_runtime::protocol::ProtocolError>(()) +/// ``` +pub fn encode(frame: &Frame) -> Result { + let mut output = vec![0; 4]; + ciborium::into_writer(frame, &mut output).map_err(|_| ProtocolError::InvalidSchema)?; + let body = output.get(4..).ok_or(ProtocolError::InvalidStructure)?; + preflight::validate(body)?; + let length = u32::try_from(body.len())?.to_be_bytes(); + output + .get_mut(..4) + .ok_or(ProtocolError::InvalidStructure)? + .copy_from_slice(&length); + Ok(Bytes::from(output)) +} + +/// Decode one bounded frame body, without its length prefix. +/// +/// # Errors +/// Rejects malformed CBOR, unsupported schema/version or invalid metadata. +/// No untrusted text is included in the returned error. +/// +/// ``` +/// use rustack_lambda_runtime::protocol::decode; +/// assert!(decode(&[]).is_err()); +/// ``` +pub fn decode(body: &[u8]) -> Result { + preflight::validate(body)?; + let raw: RawFrame = ciborium::from_reader(body).map_err(|_| ProtocolError::InvalidSchema)?; + Frame::try_from(raw) +} + +/// Read a frame under one absolute deadline; None denotes clean boundary EOF. +/// +/// # Errors +/// Returns IO/truncation, deadline or protocol validation errors. A failed or +/// cancelled read invalidates the connection, including any consumed prefix. +/// +/// ```no_run +/// # async fn read(mut input: impl tokio::io::AsyncRead + Unpin) -> Result<(), rustack_lambda_runtime::protocol::ProtocolError> { +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// let frame = rustack_lambda_runtime::protocol::read_frame(&mut input, deadline).await?; +/// # Ok(()) } +/// ``` +pub async fn read_frame( + reader: &mut R, + deadline: Instant, +) -> Result, ProtocolError> { + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + let frame = timeout_at(deadline, async { + let mut prefix = [0; 4]; + let count = reader.read(&mut prefix).await?; + if count == 0 { + return Ok(None); + } + reader + .read_exact( + prefix + .get_mut(count..) + .ok_or(ProtocolError::InvalidStructure)?, + ) + .await?; + let length = usize::try_from(u32::from_be_bytes(prefix))?; + if length > MAX_FRAME_BYTES { + return Err(ProtocolError::TooLarge); + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).await?; + decode(&body).map(Some) + }) + .await + .map_err(|_| ProtocolError::Deadline)??; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + Ok(frame) +} + +/// Encode, write and flush a frame under one absolute deadline. +/// +/// # Errors +/// Returns encoding, IO or deadline failure. Partial writes require retiring +/// the connection; retrying a business invocation is never implicit. +/// +/// ```no_run +/// # async fn write(mut output: impl tokio::io::AsyncWrite + Unpin, frame: rustack_lambda_runtime::protocol::Frame) -> Result<(), rustack_lambda_runtime::protocol::ProtocolError> { +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// rustack_lambda_runtime::protocol::write_frame(&mut output, &frame, deadline).await?; +/// # Ok(()) } +/// ``` +pub async fn write_frame( + writer: &mut W, + frame: &Frame, + deadline: Instant, +) -> Result<(), ProtocolError> { + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + let encoded = encode(frame)?; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + timeout_at(deadline, async { + writer.write_all(&encoded).await?; + writer.flush().await + }) + .await + .map_err(|_| ProtocolError::Deadline)??; + if Instant::now() >= deadline { + return Err(ProtocolError::Deadline); + } + Ok(()) +} diff --git a/crates/rustack-lambda-runtime/src/protocol/preflight.rs b/crates/rustack-lambda-runtime/src/protocol/preflight.rs new file mode 100644 index 00000000..e387550e --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/preflight.rs @@ -0,0 +1,140 @@ +//! Allocation-bounded CBOR structural validation before Serde materialization. + +use std::{collections::HashSet, str}; + +use super::{MAX_FRAME_BYTES, MAX_METADATA_BYTES, MAX_PAYLOAD_BYTES, ProtocolError}; + +#[derive(Debug)] +struct Scanner<'a> { + remaining: &'a [u8], + members: usize, + payload: Option, +} + +pub(super) fn validate(body: &[u8]) -> Result<(), ProtocolError> { + if body.len() > MAX_FRAME_BYTES { + return Err(ProtocolError::TooLarge); + } + let mut scanner = Scanner { + remaining: body, + members: 0, + payload: None, + }; + scanner.item(1, false)?; + if !scanner.remaining.is_empty() { + return Err(ProtocolError::InvalidStructure); + } + if body.len().saturating_sub(scanner.payload.unwrap_or(0)) > MAX_METADATA_BYTES { + return Err(ProtocolError::TooLarge); + } + Ok(()) +} + +impl<'a> Scanner<'a> { + fn take(&mut self, length: usize) -> Result<&'a [u8], ProtocolError> { + let (value, rest) = self + .remaining + .split_at_checked(length) + .ok_or(ProtocolError::InvalidStructure)?; + self.remaining = rest; + Ok(value) + } + + fn header(&mut self) -> Result<(u8, u64), ProtocolError> { + let (&initial, rest) = self + .remaining + .split_first() + .ok_or(ProtocolError::InvalidStructure)?; + self.remaining = rest; + let major = initial >> 5; + if major == 7 && initial != 0xf4 && initial != 0xf5 { + return Err(ProtocolError::InvalidStructure); + } + let additional = initial & 31; + let value = match additional { + 0..=23 => u64::from(additional), + 24 => u64::from(u8::from_be_bytes( + self.take(1)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 25 => u64::from(u16::from_be_bytes( + self.take(2)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 26 => u64::from(u32::from_be_bytes( + self.take(4)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + )), + 27 => u64::from_be_bytes( + self.take(8)? + .try_into() + .map_err(|_| ProtocolError::InvalidStructure)?, + ), + _ => return Err(ProtocolError::InvalidStructure), + }; + Ok((major, value)) + } + + fn text(&mut self, key: bool) -> Result<&'a str, ProtocolError> { + let (major, length) = self.header()?; + if major != 3 { + return Err(ProtocolError::InvalidStructure); + } + let length = usize::try_from(length)?; + if length > if key { 64 } else { 2048 } { + return Err(ProtocolError::TooLarge); + } + str::from_utf8(self.take(length)?).map_err(|_| ProtocolError::InvalidStructure) + } + + fn item(&mut self, depth: usize, payload: bool) -> Result<(), ProtocolError> { + if depth > 16 { + return Err(ProtocolError::TooLarge); + } + let (major, value) = self.header()?; + match major { + 0 => Ok(()), + 2 if payload && self.payload.is_none() => { + let length = usize::try_from(value)?; + if length > MAX_PAYLOAD_BYTES { + return Err(ProtocolError::TooLarge); + } + self.take(length)?; + self.payload = Some(length); + Ok(()) + } + 3 => { + let length = usize::try_from(value)?; + if length > 2048 { + return Err(ProtocolError::TooLarge); + } + str::from_utf8(self.take(length)?).map_err(|_| ProtocolError::InvalidStructure)?; + Ok(()) + } + 5 => { + let count = usize::try_from(value)?; + self.members = self + .members + .checked_add(count) + .ok_or(ProtocolError::TooLarge)?; + if self.members > 128 { + return Err(ProtocolError::TooLarge); + } + let mut keys = HashSet::with_capacity(count); + for _ in 0..count { + let key = self.text(true)?; + if !keys.insert(key) { + return Err(ProtocolError::InvalidStructure); + } + self.item(depth + 1, key == "payload")?; + } + Ok(()) + } + 7 if value == 20 || value == 21 => Ok(()), + _ => Err(ProtocolError::InvalidStructure), + } + } +} diff --git a/crates/rustack-lambda-runtime/src/protocol/tests.rs b/crates/rustack-lambda-runtime/src/protocol/tests.rs new file mode 100644 index 00000000..01d26c56 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/tests.rs @@ -0,0 +1,423 @@ +use std::{num::NonZeroU64, time::Duration}; + +use ciborium::Value; +use proptest::{collection, prelude::*}; +use rstest::rstest; +use tokio::io::{duplex, empty}; + +use super::*; + +fn payload() -> Payload { + Bytes::from_static(b"secret-payload\0\xff") + .try_into() + .unwrap() +} + +fn messages() -> Vec { + let attempt = AttemptId::default(); + vec![ + Message::Hello { + architecture: Architecture::Arm64, + }, + Message::Hello { + architecture: Architecture::X86_64, + }, + Message::Ready, + Message::BootstrapReady, + Message::Invoke { + attempt, + deadline_unix_ms: NonZeroU64::new(1_800_000_000_000).unwrap(), + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + trace_id: "Root=1-00000000-000000000000000000000000;Sampled=0" + .parse() + .unwrap(), + payload: payload(), + }, + Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: payload(), + }, + Message::InvocationResult { + attempt, + outcome: Outcome::FunctionError, + payload: payload(), + }, + Message::InitError { payload: payload() }, + Message::Cancel { attempt }, + Message::Close, + Message::Ack(Acknowledgement::Close), + Message::Ack(Acknowledgement::Cancel(attempt)), + ] +} + +fn body(value: &Value) -> Vec { + let mut bytes = Vec::new(); + ciborium::into_writer(value, &mut bytes).unwrap(); + bytes +} + +fn value(message: Message) -> Value { + let encoded = encode(&Frame::new(EnvironmentId::default(), message)).unwrap(); + ciborium::from_reader(encoded.get(4..).unwrap()).unwrap() +} + +fn field<'a>(value: &'a mut Value, key: &str) -> &'a mut Value { + value + .as_map_mut() + .unwrap() + .iter_mut() + .find_map(|(k, v)| (k.as_text() == Some(key)).then_some(v)) + .unwrap() +} + +#[test] +fn test_should_round_trip_every_variant_and_redact_payload_debug() { + for message in messages() { + let frame = Frame::new(EnvironmentId::default(), message); + let bytes = encode(&frame).unwrap(); + let length = u32::from_be_bytes(bytes.get(..4).unwrap().try_into().unwrap()); + assert_eq!(usize::try_from(length).unwrap(), bytes.len() - 4); + assert_eq!(decode(bytes.get(4..).unwrap()).unwrap(), frame); + assert!(!format!("{frame:?}").contains("secret-payload")); + } + let encoded = value(Message::InitError { payload: payload() }); + let mut encoded = encoded; + let data = field(field(&mut encoded, "message"), "data"); + assert!(matches!(field(data, "payload"), Value::Bytes(_))); +} + +#[rstest] +#[case("00000000-0000-0000-0000-000000000000")] +#[case("00112233-4455-6677-8899-AABBCCDDEEFF")] +#[case("00112233445566778899aabbccddeeff")] +#[case("00112233-4455-6677-8899-aabbccddeeff\n")] +#[case("not-an-id")] +fn test_should_reject_noncanonical_or_nil_identities(#[case] value: &str) { + assert!(value.parse::().is_err()); + assert!(value.parse::().is_err()); +} + +#[rstest] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:$LATEST")] +#[case("arn:aws-cn:lambda:cn-north-1:123456789012:function:echo:1")] +#[case("arn:aws-us-gov:lambda:us-gov-west-1:123456789012:function:echo:prod")] +fn test_should_accept_scoped_function_metadata(#[case] value: &str) { + assert_eq!(value.parse::().unwrap().as_ref(), value); +} + +#[rstest] +#[case("")] +#[case("arn:aws:s3:us-east-1:000000000000:function:echo")] +#[case("arn:aws:lambda::000000000000:function:echo")] +#[case("arn:aws:lambda:us-east-1:0000:function:echo")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:0")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:18446744073709551616")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo:prod:extra")] +#[case("arn:aws:lambda:us-east-1:000000000000:function:echo\r\nsecret")] +fn test_should_reject_invalid_header_metadata_without_echoing_it(#[case] value: &str) { + let error = value.parse::().unwrap_err(); + assert_eq!(error.to_string(), "invalid control metadata"); + assert!(!format!("{error:?}").contains("secret")); + assert!("trace\nsecret".parse::().is_err()); + assert!("é".parse::().is_err()); + assert!("x".repeat(257).parse::().is_err()); + + let mut packet = self::value(messages().into_iter().nth(4).unwrap()); + *field(field(field(&mut packet, "message"), "data"), "functionArn") = Value::Text(value.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn test_should_never_panic_on_bounded_malformed_control_input(bytes in collection::vec(any::(), 0..8192)) { + if let Ok(frame) = decode(&bytes) { + let encoded = encode(&frame).unwrap(); + prop_assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + } + } + + #[test] + fn test_should_preserve_arbitrary_binary_payloads(bytes in collection::vec(any::(), 0..16384)) { + let frame = Frame::new(EnvironmentId::default(), Message::InitError { + payload: Payload::try_from(Bytes::from(bytes)).unwrap(), + }); + let encoded = encode(&frame).unwrap(); + prop_assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + } + + #[test] + fn test_should_handle_mutated_valid_envelopes_without_panicking(position in any::(), mask in any::()) { + let frame = Frame::new(EnvironmentId::default(), Message::InitError { payload: payload() }); + let mut encoded = encode(&frame).unwrap().get(4..).unwrap().to_vec(); + let position = position % encoded.len(); + *encoded.get_mut(position).unwrap() ^= mask; + if let Ok(frame) = decode(&encoded) { + let bytes = encode(&frame).unwrap(); + prop_assert_eq!(decode(bytes.get(4..).unwrap()).unwrap(), frame); + } + } +} + +#[test] +fn test_should_reject_unknown_duplicate_and_mismatched_variant_fields() { + let frame = value(Message::Ready); + let mut unknown = frame.clone(); + unknown + .as_map_mut() + .unwrap() + .push((Value::Text("secret-unrecognized".into()), Value::Bool(true))); + let error = decode(&body(&unknown)).unwrap_err(); + assert!(matches!(error, ProtocolError::InvalidSchema)); + assert!(!format!("{error:?} {error}").contains("secret-unrecognized")); + let mut duplicate = frame.clone(); + let first = duplicate.as_map().unwrap().first().unwrap().clone(); + duplicate.as_map_mut().unwrap().push(first); + assert!(matches!( + decode(&body(&duplicate)), + Err(ProtocolError::InvalidStructure) + )); + let mut unknown_kind = frame.clone(); + *field(field(&mut unknown_kind, "message"), "kind") = Value::Text("secret-variant".into()); + assert!(matches!( + decode(&body(&unknown_kind)), + Err(ProtocolError::InvalidSchema) + )); + let mut version = frame; + *field(&mut version, "version") = Value::Integer(2.into()); + assert!(matches!( + decode(&body(&version)), + Err(ProtocolError::UnsupportedVersion) + )); + + let mut ack = value(Message::Ack(Acknowledgement::Cancel(AttemptId::default()))); + *field(field(field(&mut ack, "message"), "data"), "operation") = Value::Text("close".into()); + assert!(decode(&body(&ack)).is_err()); + let data = field(field(&mut ack, "message"), "data"); + data.as_map_mut() + .unwrap() + .retain(|(k, _)| k.as_text() != Some("attempt")); + *field(data, "operation") = Value::Text("cancel".into()); + assert!(decode(&body(&ack)).is_err()); + + for mut message in [ + value(Message::InitError { payload: payload() }), + value(Message::InvocationResult { + attempt: AttemptId::default(), + outcome: Outcome::Success, + payload: payload(), + }), + ] { + let data = field(field(&mut message, "message"), "data"); + let duplicate = data.as_map().unwrap().first().unwrap().clone(); + data.as_map_mut().unwrap().push(duplicate); + assert!(matches!( + decode(&body(&message)), + Err(ProtocolError::InvalidStructure) + )); + } +} + +#[test] +fn test_should_reject_extra_data_fields_for_every_message_variant_and_zero_deadlines() { + for message in messages() { + let mut packet = value(message); + let message = field(&mut packet, "message"); + if let Some((_, data)) = message + .as_map_mut() + .unwrap() + .iter_mut() + .find(|(key, _)| key.as_text() == Some("data")) + { + data.as_map_mut() + .unwrap() + .push((Value::Text("secret-extra".into()), Value::Bool(true))); + } else { + message + .as_map_mut() + .unwrap() + .push((Value::Text("data".into()), Value::Bool(true))); + } + let error = decode(&body(&packet)).unwrap_err(); + assert!(!format!("{error} {error:?}").contains("secret-extra")); + } + let mut packet = value(messages().into_iter().nth(4).unwrap()); + *field( + field(field(&mut packet, "message"), "data"), + "deadlineUnixMs", + ) = Value::Integer(0.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} + +#[test] +fn test_should_bound_payload_metadata_nesting_and_collection_counts_before_deserialization() { + let maximum = Payload::try_from(Bytes::from(vec![0xff; MAX_PAYLOAD_BYTES])).unwrap(); + let frame = Frame::new( + EnvironmentId::default(), + Message::InitError { payload: maximum }, + ); + let encoded = encode(&frame).unwrap(); + assert_eq!(decode(encoded.get(4..).unwrap()).unwrap(), frame); + assert!(Payload::try_from(Bytes::from(vec![0; MAX_PAYLOAD_BYTES + 1])).is_err()); + let mut too_large = value(Message::InitError { payload: payload() }); + *field(field(field(&mut too_large, "message"), "data"), "payload") = + Value::Bytes(vec![0; MAX_PAYLOAD_BYTES + 1]); + assert!(matches!( + decode(&body(&too_large)), + Err(ProtocolError::TooLarge) + )); + let metadata = Value::Map( + (0..65) + .map(|i| (Value::Text(format!("k{i}")), Value::Text("x".repeat(1024)))) + .collect(), + ); + assert!(matches!( + preflight::validate(&body(&metadata)), + Err(ProtocolError::TooLarge) + )); + let members = Value::Map( + (0..129) + .map(|i| (Value::Text(format!("k{i}")), Value::Bool(true))) + .collect(), + ); + assert!(matches!( + preflight::validate(&body(&members)), + Err(ProtocolError::TooLarge) + )); + let mut nested = Value::Bool(true); + for _ in 0..17 { + nested = Value::Map(vec![(Value::Text("x".into()), nested)]); + } + assert!(matches!( + preflight::validate(&body(&nested)), + Err(ProtocolError::TooLarge) + )); + assert!(matches!( + preflight::validate(&body(&Value::Map(vec![( + Value::Text("k".repeat(65)), + Value::Bool(true) + )]))), + Err(ProtocolError::TooLarge) + )); +} + +#[rstest] +#[case(&[0x9f, 0xff])] +#[case(&[0xbf, 0xff])] +#[case(&[0x5f, 0x40, 0xff])] +#[case(&[0xc0, 0x00])] +#[case(&[0x20])] +#[case(&[0x80])] +#[case(&[0xf9, 0, 20])] +#[case(&[0xfa, 0, 0, 0, 20])] +#[case(&[0xfb, 0, 0, 0, 0, 0, 0, 0, 20])] +#[case(&[0xf6])] +#[case(&[0xa1, 0x61, b'x', 0x7b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])] +fn test_should_reject_forbidden_cbor_shapes_and_huge_declared_lengths(#[case] bytes: &[u8]) { + assert!(preflight::validate(bytes).is_err()); +} + +#[test] +fn test_should_reject_trailing_data_and_payload_arrays_or_text() { + let frame = Frame::new(EnvironmentId::default(), Message::Ready); + let mut bytes = encode(&frame).unwrap().get(4..).unwrap().to_vec(); + bytes.push(0x00); + assert!(matches!( + decode(&bytes), + Err(ProtocolError::InvalidStructure) + )); + for invalid in [Value::Array(vec![]), Value::Text("base64-secret".into())] { + let mut message = value(Message::InitError { payload: payload() }); + *field(field(field(&mut message, "message"), "data"), "payload") = invalid; + assert!(decode(&body(&message)).is_err()); + } +} + +#[tokio::test] +async fn test_should_round_trip_fragmented_transport_and_distinguish_clean_and_truncated_eof() { + let frame = Frame::new( + EnvironmentId::default(), + Message::InitError { payload: payload() }, + ); + let encoded = encode(&frame).unwrap(); + let (mut writer, mut reader) = duplex(3); + let expected = frame.clone(); + let send = tokio::spawn(async move { + write_frame(&mut writer, &frame, Instant::now() + Duration::from_secs(2)) + .await + .unwrap(); + }); + assert_eq!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(2)) + .await + .unwrap() + .unwrap(), + expected + ); + send.await.unwrap(); + assert!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(2)) + .await + .unwrap() + .is_none() + ); + assert!( + read_frame(&mut empty(), Instant::now() + Duration::from_secs(1)) + .await + .unwrap() + .is_none() + ); + for length in [1, 2, 3, 4, encoded.len() - 1] { + let mut truncated = encoded.get(..length).unwrap(); + assert!( + matches!(read_frame(&mut truncated, Instant::now() + Duration::from_secs(1)).await, Err(ProtocolError::Io(error)) if error.kind() == io::ErrorKind::UnexpectedEof) + ); + } + let oversized = u32::MAX.to_be_bytes(); + assert!(matches!( + read_frame( + &mut oversized.as_slice(), + Instant::now() + Duration::from_secs(1) + ) + .await, + Err(ProtocolError::TooLarge) + )); +} + +#[tokio::test(start_paused = true)] +async fn test_should_enforce_absolute_deadlines_for_partial_reads_and_stalled_writes() { + let (mut writer, mut reader) = duplex(2); + writer.write_all(&[0, 0]).await.unwrap(); + assert!(matches!( + read_frame(&mut reader, Instant::now() + Duration::from_secs(1)).await, + Err(ProtocolError::Deadline) + )); + let frame = Frame::new(EnvironmentId::default(), Message::Ready); + assert!(matches!( + write_frame(&mut writer, &frame, Instant::now() + Duration::from_secs(1)).await, + Err(ProtocolError::Deadline) + )); + let encoded = encode(&frame).unwrap(); + let mut unread = encoded.as_ref(); + assert!(matches!( + read_frame(&mut unread, Instant::now()).await, + Err(ProtocolError::Deadline) + )); + assert_eq!( + unread, + encoded.as_ref(), + "expired reads must not consume a ready frame" + ); +} diff --git a/crates/rustack-lambda-runtime/src/protocol/types.rs b/crates/rustack-lambda-runtime/src/protocol/types.rs new file mode 100644 index 00000000..ebd63cad --- /dev/null +++ b/crates/rustack-lambda-runtime/src/protocol/types.rs @@ -0,0 +1,348 @@ +//! Validated metadata and closed control-message variants. + +use std::{fmt, num::NonZeroU64, str::FromStr}; + +use bytes::Bytes; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use uuid::Uuid; + +use super::{MAX_PAYLOAD_BYTES, ProtocolError}; + +macro_rules! identity { + ($name:ident, $doc:literal) => { + #[doc = $doc] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(try_from = "String", into = "String")] + pub struct $name(Uuid); + + impl Default for $name { + fn default() -> Self { + Self(Uuid::new_v4()) + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + impl FromStr for $name { + type Err = ProtocolError; + fn from_str(value: &str) -> Result { + if value.len() != 36 { + return Err(ProtocolError::InvalidMetadata); + } + let id = Uuid::parse_str(value).map_err(|_| ProtocolError::InvalidMetadata)?; + if id.is_nil() || id.to_string() != value { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(id)) + } + } + impl TryFrom for $name { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } + } + impl From<$name> for String { + fn from(value: $name) -> Self { + value.to_string() + } + } + }; +} + +identity!( + EnvironmentId, + "Canonical nonnil environment UUID; parse with `FromStr`, generate with `Default`." +); +identity!( + AttemptId, + "Canonical nonnil invocation attempt UUID; parse with `FromStr`, generate with `Default`." +); + +/// Architecture advertised by the environment-local relay. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Architecture { + /// Linux x86-64. + #[serde(rename = "x86_64")] + X86_64, + /// Linux `AArch64`. + #[serde(rename = "arm64")] + Arm64, +} + +/// Validated Lambda invocation ARN, including an optional version/alias. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct FunctionArn(String); + +impl FromStr for FunctionArn { + type Err = ProtocolError; + + fn from_str(value: &str) -> Result { + if value.len() > 2048 { + return Err(ProtocolError::InvalidMetadata); + } + let mut parts = value.split(':'); + let valid = matches!(parts.next(), Some("arn")) + && matches!(parts.next(), Some("aws" | "aws-cn" | "aws-us-gov")) + && matches!(parts.next(), Some("lambda")) + && parts + .next() + .is_some_and(|region| identifier(region, 64, false)) + && parts.next().is_some_and(|account| { + account.len() == 12 && account.bytes().all(|b| b.is_ascii_digit()) + }) + && matches!(parts.next(), Some("function")) + && parts.next().is_some_and(|name| identifier(name, 64, true)); + if !valid { + return Err(ProtocolError::InvalidMetadata); + } + if let Some(qualifier) = parts.next() { + if qualifier != "$LATEST" && !identifier(qualifier, 128, true) { + return Err(ProtocolError::InvalidMetadata); + } + if qualifier.bytes().all(|byte| byte.is_ascii_digit()) + && !qualifier.parse::().is_ok_and(|version| version != 0) + { + return Err(ProtocolError::InvalidMetadata); + } + } + if parts.next().is_some() { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(value.into())) + } +} + +fn identifier(value: &str, maximum: usize, underscore: bool) -> bool { + !value.is_empty() + && value.len() <= maximum + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || (underscore && b == b'_')) +} + +impl TryFrom for FunctionArn { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: FunctionArn) -> Self { + value.0 + } +} +impl AsRef for FunctionArn { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// Nonempty printable ASCII trace metadata, at most256 bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct TraceId(String); + +impl FromStr for TraceId { + type Err = ProtocolError; + fn from_str(value: &str) -> Result { + if value.is_empty() || value.len() > 256 || !value.bytes().all(|b| (32..=126).contains(&b)) + { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(value.into())) + } +} +impl TryFrom for TraceId { + type Error = ProtocolError; + fn try_from(value: String) -> Result { + value.parse() + } +} +impl From for String { + fn from(value: TraceId) -> Self { + value.0 + } +} +impl AsRef for TraceId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// A size-validated raw event/result whose Debug never displays contents. +#[derive(Clone, PartialEq, Eq)] +pub struct Payload(Bytes); + +impl TryFrom for Payload { + type Error = ProtocolError; + fn try_from(value: Bytes) -> Result { + if value.len() > MAX_PAYLOAD_BYTES { + return Err(ProtocolError::TooLarge); + } + Ok(Self(value)) + } +} +impl From for Bytes { + fn from(value: Payload) -> Self { + value.0 + } +} +impl AsRef<[u8]> for Payload { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} +impl fmt::Debug for Payload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Payload") + .field("bytes", &self.0.len()) + .finish_non_exhaustive() + } +} +impl Serialize for Payload { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.0) + } +} +impl<'de> Deserialize<'de> for Payload { + fn deserialize>(deserializer: D) -> Result { + struct PayloadVisitor; + impl de::Visitor<'_> for PayloadVisitor { + type Value = Payload; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded CBOR byte string") + } + fn visit_bytes(self, bytes: &[u8]) -> Result { + if bytes.len() > MAX_PAYLOAD_BYTES { + return Err(E::custom("payload size budget exceeded")); + } + Ok(Payload(Bytes::copy_from_slice(bytes))) + } + fn visit_byte_buf(self, bytes: Vec) -> Result { + Payload::try_from(Bytes::from(bytes)).map_err(E::custom) + } + } + deserializer.deserialize_byte_buf(PayloadVisitor) + } +} + +/// Acknowledgement scope; cancellation always names exactly one attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "AckFields", into = "AckFields")] +pub enum Acknowledgement { + /// The session invalidated this attempt; user-code exit is not proven. + Cancel(AttemptId), + /// The control session accepted closure. + Close, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +enum AckOperation { + Cancel, + Close, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct AckFields { + operation: AckOperation, + #[serde(skip_serializing_if = "Option::is_none")] + attempt: Option, +} +impl TryFrom for Acknowledgement { + type Error = ProtocolError; + fn try_from(value: AckFields) -> Result { + match (value.operation, value.attempt) { + (AckOperation::Cancel, Some(attempt)) => Ok(Self::Cancel(attempt)), + (AckOperation::Close, None) => Ok(Self::Close), + _ => Err(ProtocolError::InvalidMetadata), + } + } +} +impl From for AckFields { + fn from(value: Acknowledgement) -> Self { + match value { + Acknowledgement::Cancel(attempt) => Self { + operation: AckOperation::Cancel, + attempt: Some(attempt), + }, + Acknowledgement::Close => Self { + operation: AckOperation::Close, + attempt: None, + }, + } + } +} + +/// Whether the function posted a success response or a handler error. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Outcome { + /// Runtime /response. + Success, + /// Runtime /error, distinct from a transport or initialization failure. + FunctionError, +} + +/// Closed control schema; payload and invocation metadata are validated types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + content = "data", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +#[non_exhaustive] +pub enum Message { + /// First relay frame advertising the compiled platform. + Hello { + /// Linux image architecture. + architecture: Architecture, + }, + /// Relay listener is bound; does not prove bootstrap readiness. + Ready, + /// The function issued its first valid GET /next. + BootstrapReady, + /// One host-owned invocation; never replay automatically. + Invoke { + /// Shared caller/backend attempt UUID. + attempt: AttemptId, + /// Absolute wall deadline, checked and converted by the receiver. + deadline_unix_ms: NonZeroU64, + /// Requested ARN, preserving an invoked alias. + function_arn: FunctionArn, + /// Runtime trace header. + trace_id: TraceId, + /// Raw event bytes. + payload: Payload, + }, + /// Function completion for one delivered attempt. + InvocationResult { + /// Completed attempt UUID. + attempt: AttemptId, + /// /response versus /error. + outcome: Outcome, + /// Raw result or error bytes. + payload: Payload, + }, + /// Fatal initialization failure before event delivery. + InitError { + /// Raw initialization error document. + payload: Payload, + }, + /// Invalidate an attempt; the host still terminates its function. + Cancel { + /// Invalidated attempt UUID. + attempt: AttemptId, + }, + /// Close the environment's sole control session. + Close, + /// Cancellation or closure acknowledgement, scoped by operation. + Ack(Acknowledgement), +} diff --git a/specs/index.md b/specs/index.md index 64b63d82..206a2392 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; no new runtime implementation is claimed. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation foundations are implemented. Docker execution and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 90808729..c48358e1 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -155,6 +155,36 @@ Provider Environment actor Docker/relay RIC Ready means the relay is listening; bootstrap-ready means the RIC issued its first GET /next. They are different signals. Image deployment, relay start, function initialization, handler execution and cleanup have separate budgets. Synchronous Invoke's outer total includes cold start. Function handler timeout starts at event delivery and is additionally bounded by the caller's Task absolute deadline: cold start must not extend that Task deadline. Idle `/next` uses the environment idle/shutdown lease, not the short control-request timeout; this is bounded long polling. +### 3.1 Shared runtime crate and control schema + +Add the host-portable library and Linux relay binary in `crates/rustack-lambda-runtime`. Lambda core depends on its library; the runtime crate never depends on lambda-core, Squib, the AWS SDK or Docker. Its protocol module owns framing and validated message types, its session module owns the single-invocation state machine, and transport adapters own HTTP/stdio IO and supervision. Deliver these contracts before the Docker consumer. + +Protocol version1 encodes one definite-length CBOR map with exactly `version` (unsigned1), `environment` (canonical nonnil UUID string) and `message` (adjacently tagged `kind`/`data` map). Unit messages omit data. All identifiers are canonical lowercase hyphenated UUID strings. The message kind and data schema are: + +| kind | Direction | data fields | +|---|---|---| +| hello | Relay→host | architecture: x86_64/arm64 | +| ready | Relay→host | None | +| bootstrapReady | Relay→host | None | +| invoke | Host→relay | attempt (UUID), deadlineUnixMs (nonzero u64), functionArn (validated Lambda ARN), traceId (bounded printable ASCII), payload (CBOR byte string≤6 MiB) | +| invocationResult | Relay→host | attempt, outcome: success/functionError, payload | +| initError | Relay→host | payload | +| cancel | Host→relay | attempt | +| close | Host→relay | None | +| ack | Relay→host | operation: cancel/close; attempt is required for cancel and forbidden for close | + +Unknown fields/variants, duplicate keys, absent required values and mismatched per-variant fields reject. The decoder preflights the complete bounded byte slice before Serde allocation: definite maps only, at most16 nesting levels and128 total map members; text≤2,048 bytes; text map keys≤64 bytes; no arrays, negative integers, tags, floats, indefinite strings/collections or trailing items. Byte strings are allowed only as a single payload field; its bytes are excluded from the64 KiB metadata budget. Enforce the6 MiB payload and6 MiB+64 KiB frame limits separately. UUID, ARN, trace and deadline validation runs again during deserialization. Bytes uses serialize_bytes; a CBOR array/base64 string is not a valid payload. Debug and protocol-error displays omit payload bytes and invalid peer input. + +Read/write helpers operate on an owned reader/writer task. Clean EOF is distinct from a truncated prefix/body; a partial-frame timeout invalidates that transport rather than retrying from the middle. Callers supply an absolute IO deadline consistent with startup, invocation or idle leases. Cancellation of a helper authorizes retirement of that connection only; reconnect/replay is forbidden. The protocol has no registry credentials or arbitrary commands. Cancel Ack proves session invalidation, not termination of user code; the environment owner must still kill/reap the function before cleanup settlement. + +The existing workspace [Ciborium0.2.2](https://docs.rs/ciborium/0.2.2/ciborium/) provides CBOR/Serde encoding after preflight. Byte payload serialization uses the existing Bytes implementation; no second byte-buffer abstraction is required. Tests cover every variant, binary payloads, fragmented transport, clean/truncated EOF, malicious lengths/depth/duplicates/unknown fields, metadata/payload limits, canonical identities, timed partial IO and redacted Debug/errors. + +### 3.2 Runtime session ownership + +RuntimeSession is the single owner of idle/queued/delivered/closed invocation state and the first-poll readiness signal. HTTP adapters validate routes/headers/body budgets before sending typed commands over a bounded mailbox; the relay adapter validates protocol/environment/attempt before submitting the same commands. Only one queued or delivered attempt and one waiting GET /next are retained per environment. Concurrent polls/invocations reject explicitly. The first valid GET emits BootstrapReady exactly once, even when no job is queued. Completing an unknown/expired/already completed attempt cannot consume another attempt's response sender. Init failure and cancellation close the session; cancelled environments cannot accept another invocation. + +The native HTTP adapter keeps its127.0.0.1 ephemeral socket and public executor module reexports. The relay binds127.0.0.1:9001. Both use identical session metadata/result/deadline rules, and adapter shutdown joins all connection/session tasks. Native init timeout waits for first-poll readiness rather than treating process creation as readiness. The relay's outer process supervisor owns stdio/HTTP tasks and exits on malformed control input, EOF, parent lease expiry or closed session; no transport failure silently recreates a session. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 9f16a773..7462281c 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -67,7 +67,9 @@ Every row needs positive and failure-path assertions where applicable. Tests inv Add future Makefile targets: `codegen-stepfunctions`, `test-stepfunctions`, `test-lambda-docker`, `smoke-workflows`, and `verify-workflows`. Add image provisioning/build targets instead of shell scripts. Pin fixture-image digests/content IDs and expected test manifest entries. `verify-workflows` runs the exact required suites and checks that none were silently skipped. Reuse the existing build/fmt/clippy/audit/deny targets. -For Rust-relevant implementation work run cargo build, cargo test, cargo +nightly fmt, and cargo clippy -- -D warnings, plus the repository's stricter pedantic/boundary lint selection and existing applicable targets. Run cargo audit and cargo deny check when dependencies land. This documents future implementation gates; the current spec-only task changes no executable/toolchain behavior and does not require Rust compilation. +For Rust-relevant implementation work run cargo build, cargo test, cargo +nightly fmt, and cargo clippy -- -D warnings, plus the repository's stricter pedantic/boundary lint selection and existing applicable targets. Run cargo audit and cargo deny check when dependencies land. Specification-only edits use text/link checks per AGENTS.md. + +The shared runtime contract suite is available as `make test-lambda-runtime`; it runs the library tests/doctests and strict production boundary Clippy without lint ignores. Protocol property tests use [proptest1.12](https://docs.rs/proptest/1.12.0/proptest/) with512 cases per property, exercising bounded malformed bytes, mutations of valid envelopes and arbitrary binary payloads. Only its std feature is enabled; subprocess/fork machinery is unnecessary for these deterministic bounded parser checks. Runtime-session/relay tests extend this same target as their consumers land; a passing codec suite alone does not satisfy Docker acceptance. ## 4. Security acceptance diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 5ab4c2ac..acc1a384 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -141,6 +141,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | -| 1.2–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | +| 1.2 | In progress | Cancellation/revision foundation committed as93bd1ec. Shared runtime crate now implements strict typed CBOR framing, preflight before materialization, redacted payloads, deadline/EOF handling and all message variants. Independent W16 fix rechecked. Workspace:1,806 passing tests; runtime:39 tests (including512-case properties), eight doctests and strict production-boundary Clippy pass. make test-lambda-runtime passes. Shared RuntimeSession, HTTP migration, relay executable and startup probes remain required; no Docker runtime acceptance yet. | +| 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task1.2, extracting the shared runtime session and bounded relay protocol; B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 286860a8..72bdbf74 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -55,6 +55,16 @@ The independent read-only review found three lifecycle/admission defects. All we ExecutionRevision uses shared Arc ownership; InvocationHandle and the bounded work actor preserve UUID/deadline/cancellation through backend cleanup. Waiter cancellation never aborts backend workers. Native child supervisors retain artifact leases and process reservations through reaping, including executor Drop. Tests cover late-result rejection, cleanup timeout/aborted waiters, worker panic,128 accepted/32 running/default8 function limits and permit release after cleanup. The workspace suite passes1,767 tests (392 preexisting ignored), including215 Lambda core tests and five real Native fixture tests. Lambda doctests, standard workspace build/Clippy/rustdoc and the native SDK target pass. Its separately gated Squib test remains excluded as VM evidence. Full Docker and workflow milestones, plus B01/B03, remain required. +## P1 task1.2 protocol slice review + +The independent read-only review verified bounded preflight before Serde, typed variants, duplicate/unknown-field rejection, independent metadata/payload budgets, canonical identities, redacted diagnostics, absolute IO deadlines and clean/truncated EOF handling. Its final recheck found no remaining in-slice finding. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W16 | P3; `crates/rustack-lambda-runtime/src/protocol/types.rs`, FunctionArn parsing | Numeric qualifier zero/overflow differed from the provider's validated qualifier. Runtime ARN parsing now enforces the same positive-u64 rule | Direct parse and full envelope deserialization reject0 and18446744073709551616 | + +The shared runtime crate has39 passing codec tests, including512-case properties for malformed bytes, mutated valid envelopes and arbitrary binary payloads, plus eight passing doctests. The workspace suite passes1,806 tests (392 preexisting ignored); build/fmt/Clippy/rustdoc and strict runtime production-boundary checks pass. `make test-lambda-runtime` runs the reusable contract suite. Proptest1.12 uses only std; its three added dev packages introduce no audit finding. The same16 B01 advisories remain; deny bans/licenses/sources pass. This approval covers the protocol only: shared RuntimeSession, HTTP migration, relay executable, Docker execution and workflow engine remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | From 2c50c2d2f4d1b53d004661f988329040e448c0c3 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 17:41:23 -0700 Subject: [PATCH 006/106] phase 1: own runtime sessions and confirm event delivery Add the bounded shared RuntimeSession actor, per-attempt tickets, first-poll initialization signal, deadline/cancellation handling and joined shutdown. Retain offered events until poll receipt; reconcile receipts and attempt validity before state transitions. Independent review W17-W19 races fixed and rechecked. Workspace build, 1822 tests, fmt, Clippy and rustdoc pass; runtime 55 tests, 23 doctests and strict boundary checks pass. Native HTTP migration, relay/socket client, Docker executor and ASL remain required. B01/B03 security and strict-lint gates remain open; P1/M0 is not complete. --- Cargo.lock | 1 + crates/rustack-lambda-runtime/Cargo.toml | 1 + crates/rustack-lambda-runtime/src/lib.rs | 1 + crates/rustack-lambda-runtime/src/session.rs | 925 ++++++++++++++++++ .../src/session/tests.rs | 608 ++++++++++++ ...tack-workflows-20-lambda-runtime-design.md | 26 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 12 + specs/rustack-workflows-99-decisions.md | 8 + 9 files changed, 1584 insertions(+), 2 deletions(-) create mode 100644 crates/rustack-lambda-runtime/src/session.rs create mode 100644 crates/rustack-lambda-runtime/src/session/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 121a6df4..46a46983 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3884,6 +3884,7 @@ dependencies = [ "serde", "thiserror", "tokio", + "typed-builder", "uuid", ] diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml index 98ac428e..6a64cd4e 100644 --- a/crates/rustack-lambda-runtime/Cargo.toml +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -15,6 +15,7 @@ bytes = { workspace = true, features = ["serde"] } ciborium = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +typed-builder = { workspace = true } tokio = { workspace = true } uuid = { workspace = true } diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index 531a4ed1..356f9629 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -7,3 +7,4 @@ #![forbid(unsafe_code)] pub mod protocol; +pub mod session; diff --git a/crates/rustack-lambda-runtime/src/session.rs b/crates/rustack-lambda-runtime/src/session.rs new file mode 100644 index 00000000..b646609e --- /dev/null +++ b/crates/rustack-lambda-runtime/src/session.rs @@ -0,0 +1,925 @@ +//! A single actor owns each environment's invocation and long-poll state. +//! +//! This module owns only in-process state/channels. Transport adapters own HTTP +//! connections and the environment supervisor owns its process/container/VM. +//! Session cancellation invalidates an attempt; it never proves user-code exit. + +use std::{fmt, future::pending, mem::replace, time::Duration}; + +use thiserror::Error; +use tokio::{ + runtime::Handle, + sync::{ + mpsc, + oneshot::{self, error::TryRecvError}, + watch, + }, + task::JoinHandle, + time::{Instant, sleep_until, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use crate::protocol::{AttemptId, FunctionArn, Outcome, Payload, TraceId}; + +const MAILBOX_CAPACITY: usize = 8; +const MAX_IDLE_LEASE: Duration = Duration::from_hours(24); + +/// Bounded session errors never include function inputs or peer-supplied values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum SessionError { + /// Actor stopped or an owner disappeared. + #[error("runtime session unavailable")] + Unavailable, + /// One invocation/poll is already active, or the mailbox is full. + #[error("runtime session busy")] + Busy, + /// Completion does not identify the delivered active attempt. + #[error("unknown runtime invocation attempt")] + UnknownAttempt, + /// Command, idle lease or attempt deadline expired. + #[error("runtime session deadline exceeded")] + Deadline, + /// Caller or environment owner cancelled the session. + #[error("runtime session cancelled")] + Cancelled, + /// A lease/deadline cannot be represented or is outside its range. + #[error("invalid runtime session metadata")] + InvalidMetadata, + /// The owned actor panicked or failed to join. + #[error("runtime session supervisor failed")] + SupervisorFailed, +} + +/// Captured, validated event and runtime metadata for a single attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Invocation { + attempt: AttemptId, + deadline: Instant, + function_arn: FunctionArn, + trace_id: TraceId, + payload: Payload, +} + +impl Invocation { + /// Capture one event and its absolute monotonic deadline. + /// + /// # Errors + /// Returns Deadline if the attempt has already expired. + /// + /// ``` + /// # use rustack_lambda_runtime::{protocol::{AttemptId, Payload}, session::Invocation}; + /// # fn example() -> Result<(), Box> { + /// let call = Invocation::new( + /// AttemptId::default(), tokio::time::Instant::now() + std::time::Duration::from_secs(5), + /// "arn:aws:lambda:us-east-1:000000000000:function:echo".parse()?, + /// "Root=1-00000000-000000000000000000000000;Sampled=0".parse()?, + /// Payload::try_from(bytes::Bytes::from_static(b"{}"))?, + /// )?; + /// # Ok(()) } + /// ``` + pub fn new( + attempt: AttemptId, + deadline: Instant, + function_arn: FunctionArn, + trace_id: TraceId, + payload: Payload, + ) -> Result { + check_deadline(deadline)?; + Ok(Self { + attempt, + deadline, + function_arn, + trace_id, + payload, + }) + } + + /// Borrow all event metadata and payload without copying bytes. + /// + /// ``` + /// # fn inspect(call: &rustack_lambda_runtime::session::Invocation) { + /// let (attempt, deadline, arn, trace, payload) = call.parts(); + /// # } + /// ``` + #[must_use] + pub fn parts(&self) -> (AttemptId, Instant, &FunctionArn, &TraceId, &Payload) { + ( + self.attempt, + self.deadline, + &self.function_arn, + &self.trace_id, + &self.payload, + ) + } +} + +/// One accepted runtime result; raw payload diagnostics are always redacted. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum RuntimeResult { + /// Function /response. + Success(Payload), + /// Function /error. + FunctionError(Payload), + /// Fatal initialization error before any event delivery. + InitError(Payload), +} + +/// Attached ownership of a session attempt; Drop invalidates only this attempt. +#[non_exhaustive] +pub struct InvocationTicket { + attempt: AttemptId, + deadline: Instant, + cancellation: watch::Sender, + response: oneshot::Receiver>, + attached: bool, +} + +impl fmt::Debug for InvocationTicket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvocationTicket") + .field("attempt", &self.attempt) + .field("deadline", &self.deadline) + .finish_non_exhaustive() + } +} + +impl InvocationTicket { + /// Identity used by the delivered Runtime API event. + /// + /// ``` + /// # fn inspect(ticket: &rustack_lambda_runtime::session::InvocationTicket) { + /// let attempt = ticket.attempt(); + /// # } + /// ``` + #[must_use] + pub fn attempt(&self) -> AttemptId { + self.attempt + } + + /// Request invalidation; the environment supervisor still owns termination. + /// + /// ``` + /// # fn stop(ticket: &rustack_lambda_runtime::session::InvocationTicket) { + /// ticket.cancel(); + /// # } + /// ``` + pub fn cancel(&self) { + self.cancellation.send_replace(true); + } + + /// Await a single result within the attempt's absolute deadline. + /// + /// # Errors + /// Returns cancellation/expiry, session failure or disconnected supervision. + /// + /// ```no_run + /// # async fn finish(ticket: rustack_lambda_runtime::session::InvocationTicket) { + /// let result = ticket.wait().await; + /// # } + /// ``` + pub async fn wait(mut self) -> Result { + check_deadline(self.deadline)?; + if *self.cancellation.borrow() { + return Err(SessionError::Cancelled); + } + let result = timeout_at(self.deadline, &mut self.response) + .await + .map_err(|_| SessionError::Deadline)? + .map_err(|_| SessionError::Unavailable)?; + check_deadline(self.deadline)?; + if *self.cancellation.borrow() { + return Err(SessionError::Cancelled); + } + self.attached = false; + result + } +} + +impl Drop for InvocationTicket { + fn drop(&mut self) { + if self.attached { + self.cancel(); + } + } +} + +/// Cloneable bounded command port; no mutable invocation state is shared. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct SessionHandle { + inbox: mpsc::Sender, + stop: watch::Sender, + bootstrap: watch::Receiver, +} + +/// Owner of the actor and take-once initialization-error notification. +#[derive(Debug)] +#[non_exhaustive] +pub struct RuntimeSession { + handle: SessionHandle, + task: Option>, + initialization_error: Option>, + settlement: Option>, +} + +impl RuntimeSession { + /// Start a single-invocation actor with a bounded idle lease. + /// + /// # Errors + /// Rejects a zero/over24h lease, unrepresentable deadline or missing Tokio runtime. + /// + /// ```no_run + /// # async fn example() -> Result<(), rustack_lambda_runtime::session::SessionError> { + /// let mut session = rustack_lambda_runtime::session::RuntimeSession::start(std::time::Duration::from_secs(60))?; + /// session.shutdown().await?; + /// # Ok(()) } + /// ``` + pub fn start(idle_lease: Duration) -> Result { + if idle_lease.is_zero() || idle_lease > MAX_IDLE_LEASE { + return Err(SessionError::InvalidMetadata); + } + let runtime = Handle::try_current().map_err(|_| SessionError::Unavailable)?; + let idle_deadline = Instant::now() + .checked_add(idle_lease) + .ok_or(SessionError::InvalidMetadata)?; + let (inbox, commands) = mpsc::channel(MAILBOX_CAPACITY); + let (stop, stopped) = watch::channel(false); + let (bootstrap, initialized) = watch::channel(false); + let (initialization_error, error) = oneshot::channel(); + let actor = SessionActor::builder() + .inbox(commands) + .stop(stopped) + .bootstrap(bootstrap) + .initialization_error(Some(initialization_error)) + .idle_lease(idle_lease) + .idle_deadline(idle_deadline) + .build(); + Ok(Self { + handle: SessionHandle { + inbox, + stop, + bootstrap: initialized, + }, + task: Some(runtime.spawn(actor.run())), + initialization_error: Some(error), + settlement: None, + }) + } + + /// Borrow the actor's command port. + /// + /// ``` + /// # fn inspect(session: &rustack_lambda_runtime::session::RuntimeSession) { + /// let handle = session.handle().clone(); + /// # } + /// ``` + #[must_use] + pub fn handle(&self) -> &SessionHandle { + &self.handle + } + + /// Transfer the initialization notification exactly once. + /// + /// ``` + /// # fn inspect(session: &mut rustack_lambda_runtime::session::RuntimeSession) { + /// let notification = session.take_initialization_error(); + /// assert!(session.take_initialization_error().is_none()); + /// # } + /// ``` + pub fn take_initialization_error(&mut self) -> Option> { + self.initialization_error.take() + } + + /// Signal stop and join the owned actor; cancelling a waiter retains its handle. + /// + /// # Errors + /// Returns `SupervisorFailed` if the actor panicked or failed to join. + /// + /// ```no_run + /// # async fn stop(session: &mut rustack_lambda_runtime::session::RuntimeSession) { + /// let clean = session.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), SessionError> { + self.handle.close(); + if let Some(result) = self.settlement { + return result; + } + let result = if let Some(task) = self.task.as_mut() { + task.await.map_err(|_| SessionError::SupervisorFailed) + } else { + Ok(()) + }; + self.task.take(); + self.settlement = Some(result); + result + } +} + +impl Drop for RuntimeSession { + fn drop(&mut self) { + self.handle.close(); + } +} + +impl SessionHandle { + fn send(&self, command: Command, deadline: Instant) -> Result<(), SessionError> { + check_deadline(deadline)?; + if *self.stop.borrow() { + return Err(SessionError::Unavailable); + } + self.inbox + .try_send(Envelope { deadline, command }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => SessionError::Busy, + mpsc::error::TrySendError::Closed(_) => SessionError::Unavailable, + }) + } + + /// Admit one event; a dropped admission future cancels its own attempt. + /// + /// # Errors + /// Returns Busy, cancellation/expiry or unavailable supervision. + /// + /// ```no_run + /// # async fn submit(handle: &rustack_lambda_runtime::session::SessionHandle, call: rustack_lambda_runtime::session::Invocation) { + /// let deadline = call.parts().1; + /// let ticket = handle.submit(call, deadline).await; + /// # } + /// ``` + pub async fn submit( + &self, + invocation: Invocation, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + check_deadline(invocation.deadline)?; + let (cancellation, cancelled) = watch::channel(false); + let (response, result) = oneshot::channel(); + let (admitted, accepted) = oneshot::channel(); + let ticket = InvocationTicket { + attempt: invocation.attempt, + deadline: invocation.deadline, + cancellation, + response: result, + attached: true, + }; + self.send( + Command::Submit { + invocation, + cancellation: cancelled, + response, + admitted, + }, + deadline, + )?; + receive(accepted, deadline).await??; + Ok(ticket) + } + + /// Wait for the next event, allowing exactly one outstanding poll. + /// + /// # Errors + /// Returns Busy for a concurrent poll/active delivery, expiry or closed session. + /// + /// ```no_run + /// # async fn poll(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(60); + /// let event = handle.next(deadline).await; + /// # } + /// ``` + pub async fn next(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send(Command::Next { reply }, deadline)?; + let delivery = receive(response, deadline).await??; + delivery + .receipt + .send(()) + .map_err(|()| SessionError::Unavailable)?; + Ok(delivery.invocation) + } + + /// Validate and settle exactly one delivered attempt. + /// + /// # Errors + /// Rejects stale/mismatched identities, deadline expiry, overload or closure. + /// + /// ```no_run + /// # async fn complete(handle: &rustack_lambda_runtime::session::SessionHandle, attempt: rustack_lambda_runtime::protocol::AttemptId, payload: rustack_lambda_runtime::protocol::Payload) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let result = handle.complete(attempt, None, rustack_lambda_runtime::protocol::Outcome::Success, payload, deadline).await; + /// # } + /// ``` + pub async fn complete( + &self, + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + deadline: Instant, + ) -> Result<(), SessionError> { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send( + Command::Complete { + attempt, + invocation_id, + outcome, + payload, + reply, + }, + deadline, + )?; + receive(response, deadline).await? + } + + /// Report a fatal initialization failure before any event delivery. + /// + /// # Errors + /// Returns Busy after delivery, or command deadline/overload/session failure. + /// + /// ```no_run + /// # async fn fail(handle: &rustack_lambda_runtime::session::SessionHandle, payload: rustack_lambda_runtime::protocol::Payload) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let result = handle.initialization_error(payload, deadline).await; + /// # } + /// ``` + pub async fn initialization_error( + &self, + payload: Payload, + deadline: Instant, + ) -> Result<(), SessionError> { + check_deadline(deadline)?; + let (reply, response) = oneshot::channel(); + self.send(Command::InitError { payload, reply }, deadline)?; + receive(response, deadline).await? + } + + /// Await the first valid GET /next; listener readiness cannot satisfy this. + /// + /// # Errors + /// Returns deadline expiry or closure before the first poll. + /// + /// ```no_run + /// # async fn ready(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + /// let ready = handle.bootstrap_ready(deadline).await; + /// # } + /// ``` + pub async fn bootstrap_ready(&self, deadline: Instant) -> Result<(), SessionError> { + check_deadline(deadline)?; + let mut bootstrap = self.bootstrap.clone(); + timeout_at(deadline, bootstrap.wait_for(|ready| *ready)) + .await + .map_err(|_| SessionError::Deadline)? + .map_err(|_| SessionError::Unavailable)?; + check_deadline(deadline) + } + + /// Signal priority shutdown without waiting for the ordinary command mailbox. + /// + /// ``` + /// # fn stop(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// handle.close(); + /// # } + /// ``` + pub fn close(&self) { + self.stop.send_replace(true); + } +} + +fn check_deadline(deadline: Instant) -> Result<(), SessionError> { + if Instant::now() >= deadline { + Err(SessionError::Deadline) + } else { + Ok(()) + } +} + +async fn receive(response: oneshot::Receiver, deadline: Instant) -> Result { + let result = timeout_at(deadline, response) + .await + .map_err(|_| SessionError::Deadline)? + .map_err(|_| SessionError::Unavailable)?; + check_deadline(deadline)?; + Ok(result) +} + +#[derive(Debug)] +struct Envelope { + deadline: Instant, + command: Command, +} + +#[derive(Debug)] +enum Command { + Submit { + invocation: Invocation, + cancellation: watch::Receiver, + response: oneshot::Sender>, + admitted: oneshot::Sender>, + }, + Next { + reply: oneshot::Sender>, + }, + Complete { + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + reply: oneshot::Sender>, + }, + InitError { + payload: Payload, + reply: oneshot::Sender>, + }, +} + +#[derive(Debug)] +struct Pending { + attempt: AttemptId, + deadline: Instant, + cancellation: watch::Receiver, + response: oneshot::Sender>, +} + +impl Pending { + fn check(&self) -> Result<(), SessionError> { + check_deadline(self.deadline)?; + if *self.cancellation.borrow() || self.cancellation.has_changed().is_err() { + Err(SessionError::Cancelled) + } else { + Ok(()) + } + } +} + +#[derive(Debug)] +struct PollDelivery { + invocation: Invocation, + receipt: oneshot::Sender<()>, +} + +#[derive(Debug)] +enum PhaseEvent { + Cancelled, + Receipt(bool), +} + +#[derive(Debug, Default)] +enum Phase { + #[default] + Idle, + Queued { + invocation: Invocation, + pending: Pending, + }, + Offering { + invocation: Invocation, + pending: Pending, + receipt: oneshot::Receiver<()>, + }, + Delivered(Pending), + Closed, +} + +impl Phase { + fn pending(&self) -> Option<&Pending> { + match self { + Self::Queued { pending, .. } + | Self::Offering { pending, .. } + | Self::Delivered(pending) => Some(pending), + _ => None, + } + } + + async fn event(&mut self) -> PhaseEvent { + match self { + Self::Offering { + pending, receipt, .. + } => { + tokio::select! { + biased; + () = async { let _ = pending.cancellation.wait_for(|cancelled| *cancelled).await; } => PhaseEvent::Cancelled, + result = receipt => PhaseEvent::Receipt(result.is_ok()), + } + } + Self::Queued { pending, .. } | Self::Delivered(pending) => { + let _ = pending.cancellation.wait_for(|cancelled| *cancelled).await; + PhaseEvent::Cancelled + } + _ => pending().await, + } + } +} + +#[derive(Debug, TypedBuilder)] +struct SessionActor { + inbox: mpsc::Receiver, + stop: watch::Receiver, + bootstrap: watch::Sender, + initialization_error: Option>, + idle_lease: Duration, + idle_deadline: Instant, + #[builder(default)] + phase: Phase, + #[builder(default)] + poll: Option>>, + #[builder(default)] + ever_delivered: bool, +} + +impl SessionActor { + async fn run(mut self) { + loop { + let deadline = self + .phase + .pending() + .map_or(self.idle_deadline, |pending| pending.deadline); + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => { self.close(SessionError::Cancelled); break; } + () = sleep_until(deadline) => { self.close(SessionError::Deadline); break; } + event = self.phase.event() => { + match event { + PhaseEvent::Cancelled => { self.close(SessionError::Cancelled); break; } + PhaseEvent::Receipt(received) => self.received(received), + } + } + command = self.inbox.recv() => { + let Some(Envelope { command, deadline }) = command else { self.close(SessionError::Unavailable); break; }; + if let Err(error) = check_deadline(deadline) { reject(command, error); continue; } + if self.command(command) { + let Some(deadline) = Instant::now().checked_add(self.idle_lease) else { self.close(SessionError::InvalidMetadata); break; }; + self.idle_deadline = deadline; + } + if matches!(self.phase, Phase::Closed) { break; } + self.deliver(); + } + } + } + } + + fn command(&mut self, command: Command) -> bool { + self.reconcile_receipt(); + let error = if *self.stop.borrow() { + Some(SessionError::Cancelled) + } else if matches!(self.phase, Phase::Closed) { + Some(SessionError::Unavailable) + } else { + self.phase + .pending() + .and_then(|pending| pending.check().err()) + }; + if let Some(error) = error { + self.close(error); + reject(command, error); + return false; + } + match command { + Command::Submit { + invocation, + cancellation, + response, + admitted, + } => { + let error = if !matches!(self.phase, Phase::Idle) { + Some(SessionError::Busy) + } else if check_deadline(invocation.deadline).is_err() { + Some(SessionError::Deadline) + } else if *cancellation.borrow() || cancellation.has_changed().is_err() { + Some(SessionError::Cancelled) + } else { + None + }; + if let Some(error) = error { + let _ = admitted.send(Err(error)); + return false; + } + let pending = Pending { + attempt: invocation.attempt, + deadline: invocation.deadline, + cancellation, + response, + }; + self.phase = Phase::Queued { + invocation, + pending, + }; + let _ = admitted.send(Ok(())); + true + } + Command::Next { reply } => { + if matches!(self.phase, Phase::Offering { .. } | Phase::Delivered(_)) + || self.poll.as_ref().is_some_and(|poll| !poll.is_closed()) + { + let _ = reply.send(Err(SessionError::Busy)); + return false; + } + if reply.is_closed() { + return false; + } + self.bootstrap.send_if_modified(|ready| { + let changed = !*ready; + *ready = true; + changed + }); + self.poll = Some(reply); + true + } + Command::Complete { + attempt, + invocation_id, + outcome, + payload, + reply, + } => self.complete(attempt, invocation_id, outcome, payload, reply), + Command::InitError { payload, reply } => { + if let Some(error) = self + .phase + .pending() + .and_then(|pending| pending.check().err()) + { + self.close(error); + let _ = reply.send(Err(error)); + return false; + } + if self.ever_delivered { + let _ = reply.send(Err(SessionError::Busy)); + return false; + } + if let Some(notification) = self.initialization_error.take() { + let _ = notification.send(payload.clone()); + } + if let Some(pending) = self.take_pending() { + let _ = pending.response.send(Ok(RuntimeResult::InitError(payload))); + } + self.close(SessionError::Unavailable); + let _ = reply.send(Ok(())); + true + } + } + } + + fn reconcile_receipt(&mut self) { + let received = match &mut self.phase { + Phase::Offering { receipt, .. } => match receipt.try_recv() { + Ok(()) => Some(true), + Err(TryRecvError::Closed) => Some(false), + Err(TryRecvError::Empty) => None, + }, + _ => None, + }; + if let Some(received) = received { + self.received(received); + } + } + + fn complete( + &mut self, + attempt: AttemptId, + invocation_id: Option, + outcome: Outcome, + payload: Payload, + reply: oneshot::Sender>, + ) -> bool { + let valid = matches!(&self.phase, Phase::Delivered(pending) if pending.attempt == attempt) + && invocation_id.is_none_or(|id| id == attempt); + if !valid { + let _ = reply.send(Err(SessionError::UnknownAttempt)); + return false; + } + let Phase::Delivered(pending) = replace(&mut self.phase, Phase::Idle) else { + let _ = reply.send(Err(SessionError::Unavailable)); + return false; + }; + let result = if let Err(error) = pending.check() { + let _ = pending.response.send(Err(error)); + self.close(error); + Err(error) + } else { + let result = match outcome { + Outcome::Success => RuntimeResult::Success(payload), + Outcome::FunctionError => RuntimeResult::FunctionError(payload), + }; + if pending.response.send(Ok(result)).is_err() { + self.close(SessionError::Cancelled); + Err(SessionError::Cancelled) + } else { + Ok(()) + } + }; + let accepted = result.is_ok(); + let _ = reply.send(result); + accepted + } + + fn deliver(&mut self) { + if !matches!(self.phase, Phase::Queued { .. }) { + return; + } + let Some(poll) = self.poll.take() else { + return; + }; + let Phase::Queued { + invocation, + pending, + } = replace(&mut self.phase, Phase::Idle) + else { + return; + }; + if let Err(error) = pending.check() { + let _ = poll.send(Err(error)); + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + let (receipt, received) = oneshot::channel(); + match poll.send(Ok(PollDelivery { + invocation: invocation.clone(), + receipt, + })) { + Ok(()) => { + self.phase = Phase::Offering { + invocation, + pending, + receipt: received, + }; + } + Err(_) => { + self.phase = Phase::Queued { + invocation, + pending, + }; + } + } + } + + fn received(&mut self, received: bool) { + let Phase::Offering { + invocation, + pending, + .. + } = replace(&mut self.phase, Phase::Idle) + else { + return; + }; + if let Err(error) = pending.check() { + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + if received { + self.ever_delivered = true; + self.phase = Phase::Delivered(pending); + } else { + self.phase = Phase::Queued { + invocation, + pending, + }; + } + } + + fn take_pending(&mut self) -> Option { + match replace(&mut self.phase, Phase::Closed) { + Phase::Queued { pending, .. } + | Phase::Offering { pending, .. } + | Phase::Delivered(pending) => Some(pending), + _ => None, + } + } + + fn close(&mut self, error: SessionError) { + if let Some(pending) = self.take_pending() { + let _ = pending.response.send(Err(error)); + } + if let Some(poll) = self.poll.take() { + let _ = poll.send(Err(error)); + } + self.inbox.close(); + } +} + +fn reject(command: Command, error: SessionError) { + match command { + Command::Submit { + admitted, response, .. + } => { + let _ = admitted.send(Err(error)); + let _ = response.send(Err(error)); + } + Command::Next { reply } => { + let _ = reply.send(Err(error)); + } + Command::Complete { reply, .. } | Command::InitError { reply, .. } => { + let _ = reply.send(Err(error)); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/session/tests.rs b/crates/rustack-lambda-runtime/src/session/tests.rs new file mode 100644 index 00000000..169285a2 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/session/tests.rs @@ -0,0 +1,608 @@ +use std::{ + future::Future, + task::{Context, Poll, Waker}, +}; + +use bytes::Bytes; + +use super::*; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} + +fn payload() -> Payload { + Payload::try_from(Bytes::from_static(b"secret-payload")).unwrap() +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + "Root=1-00000000-000000000000000000000000;Sampled=0" + .parse() + .unwrap(), + payload(), + ) + .unwrap() +} + +fn poll_once(future: &mut F) -> Poll { + Future::poll( + std::pin::Pin::new(future), + &mut Context::from_waker(Waker::noop()), + ) +} + +#[tokio::test] +async fn test_should_deliver_once_validate_completion_and_reuse_after_success_and_function_error() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let first = invocation(); + let ticket = handle.submit(first.clone(), deadline()).await.unwrap(); + assert_eq!(ticket.attempt(), first.parts().0); + assert_eq!( + handle.submit(invocation(), deadline()).await.unwrap_err(), + SessionError::Busy + ); + assert_eq!(handle.next(deadline()).await.unwrap(), first); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Busy + ); + assert_eq!( + handle + .complete( + AttemptId::default(), + None, + Outcome::Success, + payload(), + deadline() + ) + .await, + Err(SessionError::UnknownAttempt) + ); + assert_eq!( + handle + .complete( + ticket.attempt(), + Some(AttemptId::default()), + Outcome::Success, + payload(), + deadline() + ) + .await, + Err(SessionError::UnknownAttempt) + ); + handle + .complete( + ticket.attempt(), + Some(ticket.attempt()), + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + let old = ticket.attempt(); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::Success(payload()) + ); + let second = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + assert_eq!( + handle + .complete(old, None, Outcome::Success, payload(), deadline()) + .await, + Err(SessionError::UnknownAttempt) + ); + handle + .complete( + second.attempt(), + None, + Outcome::FunctionError, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + second.wait().await.unwrap(), + RuntimeResult::FunctionError(payload()) + ); + session.shutdown().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_distinguish_listener_readiness_first_poll_and_cancelled_poll_replacement() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut readiness = handle.bootstrap.clone(); + assert_eq!( + handle + .bootstrap_ready(Instant::now() + Duration::from_millis(1)) + .await, + Err(SessionError::Deadline) + ); + let mut first = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut first).is_pending()); + handle.bootstrap_ready(deadline()).await.unwrap(); + assert!(*readiness.borrow_and_update()); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Busy + ); + drop(first); + let call = invocation(); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + assert_eq!(handle.next(deadline()).await.unwrap(), call); + assert!( + !readiness.has_changed().unwrap(), + "readiness must be emitted once" + ); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_report_initialization_failure_once_before_delivery_and_close_pending_work() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let notification = session.take_initialization_error().unwrap(); + assert!(session.take_initialization_error().is_none()); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle + .initialization_error(payload(), deadline()) + .await + .unwrap(); + assert_eq!(notification.await.unwrap(), payload()); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::InitError(payload()) + ); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_reject_initialization_error_after_any_delivery_without_consuming_active_result() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + assert_eq!( + handle.initialization_error(payload(), deadline()).await, + Err(SessionError::Busy) + ); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + assert_eq!( + handle.initialization_error(payload(), deadline()).await, + Err(SessionError::Busy) + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_invalidate_dropped_tickets_and_expired_attempts_without_reuse() { + for delivered in [false, true] { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + if delivered { + handle.next(deadline()).await.unwrap(); + } + drop(ticket); + tokio::task::yield_now().await; + assert_eq!( + handle.submit(invocation(), deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); + } + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(ticket.wait().await.unwrap_err(), SessionError::Deadline); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_cancel_attached_waiters_and_never_cancel_new_work_when_an_old_ticket_drops() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let old = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + handle + .complete(old.attempt(), None, Outcome::Success, payload(), deadline()) + .await + .unwrap(); + let current = handle.submit(invocation(), deadline()).await.unwrap(); + drop(old); + handle.next(deadline()).await.unwrap(); + handle + .complete( + current.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + current.wait().await.unwrap(); + + let cancelled = handle.submit(invocation(), deadline()).await.unwrap(); + let mut waiting = Box::pin(cancelled.wait()); + assert!(poll_once(&mut waiting).is_pending()); + drop(waiting); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_retain_actor_join_ownership_after_a_cancelled_shutdown_waiter() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut shutdown = Box::pin(session.shutdown()); + assert!(poll_once(&mut shutdown).is_pending()); + drop(shutdown); + assert!(session.task.is_some()); + session.shutdown().await.unwrap(); + assert!(session.task.is_none()); + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); +} + +#[tokio::test] +async fn test_should_cancel_admitted_work_after_the_admission_waiter_is_dropped() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut admission = Box::pin(handle.submit(invocation(), deadline())); + assert!(poll_once(&mut admission).is_pending()); + tokio::task::yield_now().await; + drop(admission); + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_idle_and_command_deadlines_and_prioritize_stop_over_a_full_mailbox() { + assert!(matches!( + RuntimeSession::start(Duration::ZERO), + Err(SessionError::InvalidMetadata) + )); + assert!(matches!( + RuntimeSession::start(Duration::from_hours(25)), + Err(SessionError::InvalidMetadata) + )); + let mut session = RuntimeSession::start(Duration::from_secs(1)).unwrap(); + let handle = session.handle().clone(); + assert_eq!( + handle.next(Instant::now()).await.unwrap_err(), + SessionError::Deadline + ); + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert_eq!( + handle.next(deadline()).await.unwrap_err(), + SessionError::Unavailable + ); + session.shutdown().await.unwrap(); + + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut receivers = Vec::new(); + for _ in 0..MAILBOX_CAPACITY { + let (reply, receiver) = oneshot::channel(); + handle.send(Command::Next { reply }, deadline()).unwrap(); + receivers.push(receiver); + } + let (reply, _) = oneshot::channel(); + assert_eq!( + handle.send(Command::Next { reply }, deadline()), + Err(SessionError::Busy) + ); + handle.close(); + session.shutdown().await.unwrap(); + assert!( + !*handle.bootstrap.borrow(), + "priority stop must beat queued polls" + ); + for receiver in receivers { + assert!(receiver.await.is_err()); + } +} + +#[tokio::test] +async fn test_should_never_report_success_after_actor_supervision_failure() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + session.task.as_ref().unwrap().abort(); + assert_eq!( + session.shutdown().await, + Err(SessionError::SupervisorFailed) + ); + assert_eq!( + session.shutdown().await, + Err(SessionError::SupervisorFailed) + ); +} + +#[test] +fn test_should_reject_start_without_a_tokio_runtime() { + assert!(matches!( + RuntimeSession::start(Duration::from_secs(60)), + Err(SessionError::Unavailable) + )); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_expired_mailbox_commands_without_delivering_or_settling_work() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let (reply, response) = oneshot::channel(); + handle + .send( + Command::Next { reply }, + Instant::now() + Duration::from_millis(1), + ) + .unwrap(); + tokio::time::advance(Duration::from_millis(1)).await; + assert!(matches!( + response.await.unwrap(), + Err(SessionError::Deadline) + )); + assert!(!*handle.bootstrap.borrow()); + + let ticket = handle.submit(invocation(), deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + let (reply, response) = oneshot::channel(); + handle + .send( + Command::Complete { + attempt: ticket.attempt(), + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply, + }, + Instant::now() + Duration::from_millis(1), + ) + .unwrap(); + tokio::time::advance(Duration::from_millis(1)).await; + assert_eq!(response.await.unwrap(), Err(SessionError::Deadline)); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + ticket.wait().await.unwrap(), + RuntimeResult::Success(payload()) + ); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_signal_owned_actor_cleanup_when_the_session_owner_is_dropped() { + let session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation(); + assert!( + !format!("{call:?} {:?}", RuntimeResult::Success(payload())).contains("secret-payload") + ); + let ticket = handle.submit(call, deadline()).await.unwrap(); + assert!(!format!("{ticket:?}").contains("secret-payload")); + drop(session); + assert_eq!(ticket.wait().await, Err(SessionError::Cancelled)); + assert_eq!( + handle.next(deadline()).await, + Err(SessionError::Unavailable) + ); +} + +#[tokio::test] +async fn test_should_requeue_an_offered_event_if_the_poll_future_drops_before_receiving_it() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation(); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + let mut poll = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut poll).is_pending()); + // The actor sends the event, but this poll future never consumes the reply. + tokio::task::yield_now().await; + drop(poll); + assert_eq!(handle.next(deadline()).await.unwrap(), call); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + ticket.wait().await.unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_reconcile_ready_poll_receipts_before_completion_or_initialization_commands() { + for initialization in [false, true] { + let mut actor = transition_actor(); + let invocation = invocation(); + let attempt = invocation.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: invocation.deadline, + cancellation, + response, + }; + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation, + pending, + receipt: acknowledged, + }; + receipt.send(()).unwrap(); + let (reply, result) = oneshot::channel(); + if initialization { + assert!(!actor.command(Command::InitError { + payload: payload(), + reply + })); + assert_eq!(result.await.unwrap(), Err(SessionError::Busy)); + assert!(actor.ever_delivered); + assert!(matches!(actor.phase, Phase::Delivered(_))); + let (reply, _) = oneshot::channel(); + assert!(actor.complete(attempt, None, Outcome::Success, payload(), reply)); + } else { + assert!(actor.command(Command::Complete { + attempt, + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply + })); + assert_eq!(result.await.unwrap(), Ok(())); + } + assert_eq!( + received.await.unwrap(), + Ok(RuntimeResult::Success(payload())) + ); + } +} + +fn transition_actor() -> SessionActor { + let (_, inbox) = mpsc::channel(MAILBOX_CAPACITY); + let (_, stop) = watch::channel(false); + let (bootstrap, _) = watch::channel(false); + SessionActor::builder() + .inbox(inbox) + .stop(stop) + .bootstrap(bootstrap) + .initialization_error(None) + .idle_lease(Duration::from_secs(60)) + .idle_deadline(Instant::now() + Duration::from_secs(60)) + .build() +} + +#[tokio::test(start_paused = true)] +async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_delivery_transition() +{ + for transition in ["deliver", "complete", "receipt"] { + for condition in ["cancelled", "closed", "expired"] { + let mut actor = transition_actor(); + let invocation = invocation(); + let attempt = invocation.attempt; + let (cancel, cancellation) = watch::channel(false); + let mut cancel = Some(cancel); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: invocation.deadline, + cancellation, + response, + }; + match condition { + "cancelled" => { + cancel.as_ref().unwrap().send_replace(true); + } + "closed" => { + cancel.take(); + } + "expired" => tokio::time::advance(Duration::from_secs(5)).await, + _ => panic!("unknown test case"), + } + let expected = if condition == "expired" { + SessionError::Deadline + } else { + SessionError::Cancelled + }; + match transition { + "deliver" => { + actor.phase = Phase::Queued { + invocation, + pending, + }; + let (poll, received) = oneshot::channel(); + actor.poll = Some(poll); + actor.deliver(); + assert!(matches!(received.await.unwrap(), Err(error) if error == expected)); + } + "complete" => { + actor.phase = Phase::Delivered(pending); + let (reply, received) = oneshot::channel(); + assert!(!actor.complete(attempt, None, Outcome::Success, payload(), reply)); + assert_eq!(received.await.unwrap(), Err(expected)); + } + "receipt" => { + let (_, receipt) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation, + pending, + receipt, + }; + actor.received(true); + } + _ => panic!("unknown test case"), + } + assert_eq!(received.await.unwrap(), Err(expected)); + assert!(matches!(actor.phase, Phase::Closed)); + assert!(actor.inbox.is_closed()); + } + } +} diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index c48358e1..c621f39c 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -185,6 +185,32 @@ RuntimeSession is the single owner of idle/queued/delivered/closed invocation st The native HTTP adapter keeps its127.0.0.1 ephemeral socket and public executor module reexports. The relay binds127.0.0.1:9001. Both use identical session metadata/result/deadline rules, and adapter shutdown joins all connection/session tasks. Native init timeout waits for first-poll readiness rather than treating process creation as readiness. The relay's outer process supervisor owns stdio/HTTP tasks and exits on malformed control input, EOF, parent lease expiry or closed session; no transport failure silently recreates a session. +The shared session API has a RuntimeSession owner, cloneable SessionHandle and InvocationTicket. start(idle lease) creates one actor with an eight-command mailbox; the owner exposes its handle, first-poll readiness and a take-once initialization-error receiver. submit(validated Invocation, command deadline) returns a ticket only after actor admission. next(command deadline) permits one outstanding poll and returns the delivered Invocation. complete(attempt, optional Invocation-Id, outcome, payload, command deadline) validates both identities and active deadline atomically before resolving the ticket. initialization_error(payload, command deadline) is legal only before any event delivery, reports once and closes the session. Transport adapters convert SessionError into their own HTTP/control outcomes without logging invalid values. + +Invocation has validated attempt/ARN/trace/payload fields and an absolute monotonic deadline. Ticket wait returns a typed success/function-error/init-error result. Dropping an attached ticket, including cancellation of its wait/admission future, signals a dedicated per-attempt cancellation receiver. A ticket's cancel signal never targets a later attempt. Cancellation/expiry invalidates the session, resolves the current ticket/poll once, and forbids reuse. Concurrent calls receive Busy, stale/foreign completion receives UnknownAttempt without consuming active ownership, and RPC queue overload is explicit. A cancelled long-poll waiter may be replaced without losing a queued invocation. Result delivery to a disconnected ticket closes the session rather than accepting further work. + +Offering an event through a oneshot channel is not delivery proof: retain the queued Invocation until next() consumes its response and synchronously acknowledges a per-poll receipt. A dropped response future closes that receipt and restores the queued event. Prioritize receipt processing before later mailbox commands so immediate completion observes Delivered. Before dispatching each selected mailbox command, synchronously reconcile any ready/closed receipt and recheck stop/attempt validity: biased select alone cannot order concurrently arriving messages on separate channels. Once the adapter receives an event, failed HTTP transmission requires retiring its environment rather than silently replaying; the receipt does not prove user-code execution. + +```text +┌─ RuntimeSession owner ───────────────┐ ┌─ SessionHandle / adapter ─────┐ +│ priority stop + retained join │ │ submit / next / complete │ +└──────────────────┬─────────────────┘ └──────────────┬────────────────┘ + │ stop watch │ bounded mailbox8 +┌─ Session actor ───▼────────────────────────────────────▼───────────────┐ +│ Idle ──submit──▶ Queued ──offer to poll──▶ Offering │ +│ ▲ │ per-poll receipt │ +│ └── receiver disappears ──┤ │ +│ └── consumed ─▶ Delivered│ +│ Idle ◀──────── matching, active completion ────────────────────┘ │ +│ stop / cancellation / expiry ─▶ Closed: fail pending ticket + poll │ +│ recheck attempt validity at offer, receipt and completion transitions │ +└──────────────────────────────────────────────────────────────────────┘ + InvocationTicket drop → per-attempt cancellation watch + Closed session → environment owner terminates external resources +``` + +The actor prioritizes owner stop, active attempt cancellation/deadline and idle lease expiry over mailbox traffic. Activity refreshes the idle lease only at valid session commands; a running attempt uses its own deadline. Each command envelope carries its absolute RPC deadline, checked again before state mutation. Shutdown signals the actor and awaits its stored join handle by mutable borrow, so a cancelled shutdown waiter cannot lose the task. Cache failed joins so repeated shutdown never reports false success. Owner Drop signals stop and leaves the actor to finish its in-process channel cleanup; adapters still own/await their HTTP tasks and external execution resources. The parent Tokio runtime must remain alive until normal service shutdown has joined these owners. Paused-clock and contention tests pin single admission/delivery/settlement, cancelled poll replacement, stale-completion isolation, old-ticket cancellation, init/readiness, idle/attempt deadlines, overload and dropped shutdown/admission waiters. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index acc1a384..e6cb5163 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -141,7 +141,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | -| 1.2 | In progress | Cancellation/revision foundation committed as93bd1ec. Shared runtime crate now implements strict typed CBOR framing, preflight before materialization, redacted payloads, deadline/EOF handling and all message variants. Independent W16 fix rechecked. Workspace:1,806 passing tests; runtime:39 tests (including512-case properties), eight doctests and strict production-boundary Clippy pass. make test-lambda-runtime passes. Shared RuntimeSession, HTTP migration, relay executable and startup probes remain required; no Docker runtime acceptance yet. | +| 1.2 | In progress | Cancellation/revision foundation committed as93bd1ec; bounded protocol committed as4da7df3. Shared RuntimeSession actor now implements bounded commands, first-poll readiness, initialization failure, per-attempt cancellation/deadlines, receipt-confirmed delivery, stale completion rejection and cancel-safe joined shutdown. W17–W19 races fixed/rechecked. Workspace:1,822 passing tests; runtime:55 tests (including512-case properties),23 doctests and strict production-boundary Clippy pass. make test-lambda-runtime passes. Native HTTP migration, relay executable/socket client and startup probes remain required; no Docker runtime acceptance yet. | | 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production task is P1 task1.2, extracting the shared runtime session and bounded relay protocol; B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: migrate Native HTTP to the shared session and build the relay/socket client and startup probes. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 72bdbf74..0bcc0ae7 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -65,6 +65,18 @@ The independent read-only review verified bounded preflight before Serde, typed The shared runtime crate has39 passing codec tests, including512-case properties for malformed bytes, mutated valid envelopes and arbitrary binary payloads, plus eight passing doctests. The workspace suite passes1,806 tests (392 preexisting ignored); build/fmt/Clippy/rustdoc and strict runtime production-boundary checks pass. `make test-lambda-runtime` runs the reusable contract suite. Proptest1.12 uses only std; its three added dev packages introduce no audit finding. The same16 B01 advisories remain; deny bans/licenses/sources pass. This approval covers the protocol only: shared RuntimeSession, HTTP migration, relay executable, Docker execution and workflow engine remain required. +## P1 task1.2 shared-session slice review + +The independent read-only review found three cancellation/order races. All were fixed and the final recheck found no remaining concrete in-slice finding. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W17 | P2; `crates/rustack-lambda-runtime/src/session.rs:821` | A sent oneshot response was treated as delivered before its waiter consumed it. Offering now retains Invocation until a per-poll receipt; abandoned receipts recover queued ownership | Deterministic manual-poll test drops next() after send but before consumption, then receives the same event exactly once through a replacement poll | +| W18 | P2; `crates/rustack-lambda-runtime/src/session.rs:782` | Biased selection did not recheck cancellation arriving between branch polls. Offer/receipt/completion and command dispatch now revalidate cancellation, owner closure and deadline before mutation | Direct transition tests cover cancelled/closed/expired attempts across offer, receipt and completion and require Closed with failed pending results | +| W19 | P2; `crates/rustack-lambda-runtime/src/session.rs:768` | A receipt and following completion could arrive between select branch polls, leaving stale Offering/initialization state. Command dispatch now synchronously reconciles ready/closed receipts before matching commands | Ready-receipt→Complete succeeds; ready-receipt→InitError returns Busy and preserves the delivered attempt | + +The actor owns an eight-command mailbox, one queued/offered/delivered attempt and one poll; per-attempt tickets prevent old cancellation from targeting new work. RPC envelopes carry their own deadline, readiness changes once, initialization notifies once, shutdown retains its join handle across cancelled waiters and caches failed settlement. Sixteen actor tests and the39 codec tests pass;23 runtime doctests and strict production-boundary Clippy pass. Workspace build/fmt/Clippy/rustdoc and1,822 tests pass (392 preexisting ignored). This review covers the shared actor only: the Native HTTP adapter still uses its existing session implementation until migration; the relay/socket client, Docker executor and ASL engine are not implemented by this slice. B01/B03 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 8fddff11..f4d51a70 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -108,6 +108,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [execution revision](./rustack-workflows-20-lambda-runtime-design.md#26-execution-revision-ownership), [cancellation and supervisor contract](./rustack-workflows-20-lambda-runtime-design.md#27-cancellable-invocation-and-supervisor-contract). - Date: 2026-10-25. +## D15 — Confirm poll consumption before committing runtime delivery + +- Alternatives: treat a successful oneshot send as delivery; rely solely on biased select ordering across receipt and mailbox channels; leave an unconsumed event in Delivered until timeout. +- Decision: retain Invocation ownership in Offering until next() consumes its reply and acknowledges a per-poll receipt. Recover the queued event if that receipt closes. Reconcile ready receipts and revalidate stop/cancellation/deadline synchronously before each command/transition. +- Why: a sent response can be dropped without being consumed, and a receipt arriving between select branch polls can race a completion command. Explicit receipt ownership prevents lost events and false UnknownAttempt/initialization outcomes. Transport adapters still retire environments on failed HTTP delivery; session receipt is not proof of user-code execution. +- Pinned by: [shared session ownership and state diagram](./rustack-workflows-20-lambda-runtime-design.md#32-runtime-session-ownership). +- Date: 2026-10-25. + ## Glossary | Term | Meaning in this specification | From 95282c4ef0d3a4e315557eba9d0bb53602ec9e69 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 18:06:34 -0700 Subject: [PATCH 007/106] phase 1: migrate Native execution to shared runtime HTTP sessions Replace the old pending-map API with the bounded shared adapter. Wait for first bootstrap polling, preserve handler budgets across cold start and join API/process retirement across updates, cancellation and concurrent shutdown. Enforce header/body/trailer validation and task-bound Hyper diagnostic filtering. Independent review W20-W22 fixed/rechecked. Workspace build, 1841 tests, fmt, Clippy and rustdoc pass; runtime 71 tests/34 doctests and strict boundary checks, core 212 tests/26 doctests, 11 real Native regressions and native SDK checks pass. Audit remains at the same 16 B01 baseline findings; B03 and relay/Docker/ASL work remain required before full completion. --- Cargo.lock | 8 + Cargo.toml | 1 + crates/rustack-lambda-core/Cargo.toml | 1 + .../src/executor/instance.rs | 415 ++++++++++---- .../src/executor/native.rs | 51 +- .../src/executor/runtime_api.rs | 520 +---------------- crates/rustack-lambda-runtime/Cargo.toml | 7 + .../rustack-lambda-runtime/src/diagnostics.rs | 76 +++ crates/rustack-lambda-runtime/src/http.rs | 538 ++++++++++++++++++ .../rustack-lambda-runtime/src/http/tests.rs | 502 ++++++++++++++++ crates/rustack-lambda-runtime/src/lib.rs | 3 + crates/rustack-lambda-runtime/src/metadata.rs | 134 +++++ crates/rustack-lambda-runtime/src/session.rs | 23 + ...tack-workflows-20-lambda-runtime-design.md | 12 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 14 + specs/rustack-workflows-99-decisions.md | 8 + .../echo-bootstrap/src/main.rs | 15 + .../echo-bootstrap/tests/hardening.rs | 242 ++++++++ 19 files changed, 1913 insertions(+), 661 deletions(-) create mode 100644 crates/rustack-lambda-runtime/src/diagnostics.rs create mode 100644 crates/rustack-lambda-runtime/src/http.rs create mode 100644 crates/rustack-lambda-runtime/src/http/tests.rs create mode 100644 crates/rustack-lambda-runtime/src/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 46a46983..80fcf0b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3814,6 +3814,7 @@ dependencies = [ "rustack-core", "rustack-lambda-http", "rustack-lambda-model", + "rustack-lambda-runtime", "serde", "serde_json", "sha2 0.11.0", @@ -3879,11 +3880,18 @@ version = "0.9.0" dependencies = [ "bytes", "ciborium", + "http 1.5.0", + "http-body-util", + "hyper", + "hyper-util", "proptest", "rstest", "serde", "thiserror", "tokio", + "tracing", + "tracing-core", + "tracing-subscriber", "typed-builder", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 0c1c578f..feeaa24f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -154,6 +154,7 @@ uuid = { version = "1.26", features = ["v4"] } # Logging / tracing tracing = "0.1" +tracing-core = "0.1.36" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } # Async utilities diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 8d88cb18..06222d6a 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -14,6 +14,7 @@ workspace = true rustack-core = { workspace = true } rustack-lambda-model = { workspace = true } rustack-lambda-http = { workspace = true } +rustack-lambda-runtime = { workspace = true } rustack-auth = { workspace = true } squib = { workspace = true } http = { workspace = true } diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 305f15df..6a5e9d28 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -1,24 +1,36 @@ //! A single warm execution instance and its pool. //! -//! Used by both the native and Docker backends. Each instance owns its own +//! Used by host bootstrap backends. Each instance owns its own //! [`runtime_api::RuntimeApiHandle`] and a way to start / kill the //! corresponding bootstrap (process or container). Pool entries are scoped //! per `(function_name, qualifier)` because code can diverge between //! versions. use std::{ + mem::take, + net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; +use bytes::Bytes; use dashmap::DashMap; -use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot}; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, Payload}, +}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore, oneshot}, + task::{JoinHandle, JoinSet}, + time::{Instant as TokioInstant, timeout_at}, +}; use tracing::debug; +use typed_builder::TypedBuilder; use super::{ error::ExecutorError, - runtime_api::{self, Job, RuntimeApiHandle, RuntimeResult}, + runtime_api::{self, Invocation, RuntimeApiHandle, RuntimeResult, SessionError}, types::{InvokeRequest, InvokeResponse}, }; use crate::{artifact::DeploymentArtifact, invocation::InvocationContext}; @@ -37,7 +49,8 @@ pub(crate) trait InstanceBackend: Send + Sync + std::fmt::Debug { } /// Spawn a bootstrap pointing at `runtime_api_addr` for the given function. - /// Returns a handle the pool will keep alive until the instance is reaped. + /// Register external ownership before an await after creation. Cancellation + /// before returning a handle must leave no unowned process. async fn spawn( &self, req: &InvokeRequest, @@ -93,7 +106,52 @@ struct Instance { backend: BackendHandle, last_used: Instant, idle_permit: Option, - init_error: Option>, + init_error: Option>, +} + +impl Instance { + async fn run( + &mut self, + invocation: Invocation, + context: &InvocationContext, + timeout: Duration, + ) -> Result { + let response = async { + let ticket = self + .api + .session() + .submit(invocation, context.deadline()) + .await + .map_err(|error| invocation_error(error, timeout))?; + match self.init_error.take() { + Some(mut init) => tokio::select! { + biased; + result = ticket.wait() => result.map_err(|error| invocation_error(error, timeout)), + error = &mut init => error.map(RuntimeResult::InitError).map_err(|_| ExecutorError::RuntimeExited("Runtime initialization channel closed".into())), + }, + None => ticket + .wait() + .await + .map_err(|error| invocation_error(error, timeout)), + } + }; + tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = response => result, + exit = self.backend.wait() => exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before completing invocation".into()))), + } + } + + async fn retire(mut self) -> Result<(), ExecutorError> { + let api = self + .api + .shutdown() + .await + .map_err(|_| ExecutorError::CleanupFailed("Runtime API shutdown failed".into())); + let backend = self.backend.terminate().await; + backend.and(api) + } } /// Pool of warm instances per `(function, qualifier)` key. @@ -101,7 +159,7 @@ struct Instance { /// `acquire` fast-paths a warm instance and otherwise asks the backend to spawn /// a new one. `release` returns the instance to the pool subject to /// `max_warm`. `reap_idle` evicts instances older than `idle_timeout`. -#[derive(Debug)] +#[derive(Debug, TypedBuilder)] pub(crate) struct InstancePool { backend: Arc, max_warm: usize, @@ -118,14 +176,14 @@ impl InstancePool { idle_timeout: Duration, init_timeout: Duration, ) -> Self { - Self { - backend, - max_warm, - idle_timeout, - init_timeout, - pools: DashMap::new(), - idle_capacity: Arc::new(Semaphore::new(32)), - } + Self::builder() + .backend(backend) + .max_warm(max_warm) + .idle_timeout(idle_timeout) + .init_timeout(init_timeout) + .pools(DashMap::new()) + .idle_capacity(Arc::new(Semaphore::new(32))) + .build() } pub(crate) fn key(req: &InvokeRequest) -> PoolKey { @@ -172,195 +230,306 @@ impl InstancePool { context: InvocationContext, ) -> Result { context.check()?; + let attempt: AttemptId = context + .id() + .to_string() + .parse() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation identity".into()))?; + let arn = req + .function_arn + .parse() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation ARN".into()))?; + let trace = local_trace_id() + .map_err(|_| ExecutorError::InvalidCode("Invalid invocation clock metadata".into()))?; + let payload = Payload::try_from(req.payload.clone()) + .map_err(|_| ExecutorError::InvalidCode("Invocation payload too large".into()))?; let key = Self::key(&req); - let mut instance = if let Some(instance) = self.try_acquire(&key) { - instance - } else { - tokio::select! { - biased; - reason = context.cancelled() => return Err(reason.error()), - result = self.spawn_new(&req) => result?, + let (instance, retired) = self.try_acquire(&key); + retire_all(retired).await?; + let instance = match instance { + Some(instance) if instance.backend.running() && !instance.api.session().is_closed() => { + Some(instance) } + Some(instance) => { + instance.retire().await?; + None + } + None => None, }; - let Some(handler_deadline) = Instant::now().checked_add(req.revision.timeout) else { - instance.api.shutdown().await; - instance.backend.terminate().await?; - return Err(ExecutorError::InvalidCode("Invalid handler timeout".into())); - }; - let deadline = handler_deadline.min(context.deadline().into_std()); - let (resp_tx, resp_rx) = oneshot::channel(); - let job = Job { - request_id: context.id().to_string(), - function_arn: req.function_arn.clone(), - deadline, - payload: req.payload.clone(), - response_tx: resp_tx, + let mut instance = match instance { + Some(instance) => instance, + None => self.spawn_new(&req, &context).await?, }; - let result = { - let response = async { - instance - .api - .submit(job) - .await - .map_err(|error| ExecutorError::Io(error.into()))?; - let response = async { - match instance.init_error.take() { - Some(mut init) => { - tokio::select! { response = resp_rx => response, error = &mut init => error.map(RuntimeResult::InitError) } - } - None => resp_rx.await, - } - }; - match tokio::time::timeout(req.revision.timeout, response).await { - Ok(Ok(response)) => Ok(response), - Ok(Err(_)) => Err(ExecutorError::RuntimeExited( - "Runtime response channel closed".into(), - )), - Err(_) => Err(ExecutorError::Timeout(req.revision.timeout)), - } - }; - tokio::select! { - biased; - reason = context.cancelled() => Err(reason.error()), - result = response => result, - exit = instance.backend.wait() => { - exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before completing invocation".into()))) - } - } + let Some(handler_deadline) = TokioInstant::now().checked_add(req.revision.timeout) else { + instance.retire().await?; + return Err(ExecutorError::InvalidCode("Invalid handler timeout".into())); }; - let result = match context.check() { - Ok(()) => result, - Err(error) => Err(error), + let deadline = handler_deadline.min(context.deadline()); + let Ok(invocation) = Invocation::new(attempt, deadline, arn, trace, payload) else { + instance.retire().await?; + context.check()?; + return Err(ExecutorError::Timeout(req.revision.timeout)); }; - let reusable = - matches!(&result, Ok(RuntimeResult::Success(_))) && instance.backend.running(); + let result = instance + .run(invocation, &context, req.revision.timeout) + .await; + let result = context.check().and(result); + let reusable = matches!(&result, Ok(RuntimeResult::Success(_))) + && instance.backend.running() + && !instance.api.session().is_closed(); if reusable { instance.last_used = Instant::now(); - if let Some(mut instance) = self.release(key, instance) { - instance.api.shutdown().await; - instance.backend.terminate().await?; + if let Some(instance) = self.release(key, instance) { + instance.retire().await?; } } else { - instance.api.shutdown().await; - instance.backend.terminate().await?; + instance.retire().await?; } match result? { RuntimeResult::Success(payload) => Ok(InvokeResponse::success( - payload, + Bytes::from(payload), req.revision.qualifier.clone(), )), - RuntimeResult::Error(payload) => Ok(InvokeResponse { + RuntimeResult::FunctionError(payload) => Ok(InvokeResponse { status: 200, - payload, + payload: Bytes::from(payload), function_error: Some("Unhandled".into()), log_tail: None, executed_version: req.revision.qualifier.clone(), }), - RuntimeResult::InitError(payload) => Err(ExecutorError::InitFailed( - String::from_utf8_lossy(&payload).into_owned(), + RuntimeResult::InitError(_) => Err(ExecutorError::InitFailed( + "Bootstrap reported initialization failure".into(), + )), + _ => Err(ExecutorError::RuntimeExited( + "Unsupported runtime result".into(), )), } } - fn try_acquire(&self, key: &PoolKey) -> Option { - let mut bucket = self.pools.get_mut(&key.0)?; - let Some(index) = bucket.iter().position(|(revision, _)| revision == &key.1) else { - bucket.clear(); - return None; + fn try_acquire(&self, key: &PoolKey) -> (Option, Vec) { + let Some(mut bucket) = self.pools.get_mut(&key.0) else { + return (None, Vec::new()); }; - let (_, mut instance) = bucket.swap_remove(index); - instance.idle_permit.take(); - Some(instance) + let result = if let Some(index) = bucket.iter().position(|(revision, _)| revision == &key.1) + { + let (_, mut instance) = bucket.swap_remove(index); + instance.idle_permit.take(); + (Some(instance), Vec::new()) + } else { + ( + None, + take(&mut *bucket) + .into_iter() + .map(|(_, instance)| instance) + .collect(), + ) + }; + drop(bucket); + self.pools.remove_if(&key.0, |_, bucket| bucket.is_empty()); + result } fn release(&self, key: PoolKey, mut instance: Instance) -> Option { let Ok(permit) = Arc::clone(&self.idle_capacity).try_acquire_owned() else { return Some(instance); }; - let mut bucket = self.pools.entry(key.0).or_default(); - if bucket.len() < self.max_warm.min(1) { + let mut bucket = self.pools.entry(key.0.clone()).or_default(); + if !self.idle_capacity.is_closed() && bucket.len() < self.max_warm.min(1) { instance.idle_permit = Some(permit); bucket.push((key.1, instance)); None } else { + drop(bucket); + self.pools.remove_if(&key.0, |_, bucket| bucket.is_empty()); Some(instance) } } - async fn spawn_new(&self, req: &InvokeRequest) -> Result { + async fn spawn_new( + &self, + req: &InvokeRequest, + context: &InvocationContext, + ) -> Result { + context.check()?; if !self.backend.has_capacity() { + let mut evicted = None; for mut bucket in self.pools.iter_mut() { if let Some((_, instance)) = bucket.pop() { - drop(instance); + evicted = Some(instance); break; } } + if let Some(instance) = evicted { + instance.retire().await?; + } } - let api = runtime_api::start() - .await - .map_err(|e| ExecutorError::Io(format!("bind runtime api: {e}")))?; + let deadline = TokioInstant::now() + .checked_add(self.init_timeout) + .ok_or_else(|| ExecutorError::InvalidCode("Invalid initialization budget".into()))? + .min(context.deadline()); + let mut api = tokio::select! { + biased; + reason = context.cancelled() => return Err(reason.error()), + result = timeout_at(deadline, runtime_api::start_at(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), self.idle_timeout)) => { + result.map_err(|_| ExecutorError::Timeout(self.init_timeout))?.map_err(|_| ExecutorError::InitFailed("Runtime API startup failed".into()))? + } + }; let addr = api.addr(); - let init_error = api.take_init_error_rx().await; - - // Race: backend spawn + first /next poll. We don't observe /next here - // directly — we rely on either submit landing on a polling bootstrap - // OR an `/init/error` arriving. To keep liveness, spawn the backend - // within the init window and watch the init-error channel for a - // fast-fail signal. - let backend = tokio::time::timeout(self.init_timeout, self.backend.spawn(req, addr)) - .await - .map_err(|_| ExecutorError::Timeout(self.init_timeout))??; - let inst = Instance { + let init_error = api.take_initialization_error(); + let backend = tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = timeout_at(deadline, self.backend.spawn(req, addr)) => result.map_err(|_| ExecutorError::Timeout(self.init_timeout)).and_then(|result| result), + }; + let backend = match backend { + Ok(backend) => backend, + Err(error) => { + api.shutdown().await.map_err(|_| { + ExecutorError::CleanupFailed("Runtime API startup cleanup failed".into()) + })?; + return Err(error); + } + }; + let mut instance = Instance { api, backend, last_used: Instant::now(), idle_permit: None, init_error, }; - debug!(function = %req.revision.function_name, addr = %addr, "spawned new lambda instance"); - Ok(inst) + let readiness = { + let ready = async { + match instance.init_error.as_mut() { + Some(init) => tokio::select! { + biased; + error = init => match error { + Ok(_) => Err(ExecutorError::InitFailed("Bootstrap reported initialization failure".into())), + Err(_) => Err(ExecutorError::RuntimeExited("Runtime initialization channel closed".into())), + }, + ready = instance.api.session().bootstrap_ready(deadline) => ready.map_err(|error| invocation_error(error, self.init_timeout)), + }, + None => instance + .api + .session() + .bootstrap_ready(deadline) + .await + .map_err(|error| invocation_error(error, self.init_timeout)), + } + }; + tokio::select! { + biased; + reason = context.cancelled() => Err(reason.error()), + result = ready => result, + exit = instance.backend.wait() => exit.and(Err(ExecutorError::RuntimeExited("Bootstrap exited before first Runtime API poll".into()))), + } + }; + if let Err(error) = readiness { + instance.retire().await?; + return Err(error); + } + debug!(function = %req.revision.function_name, "Lambda bootstrap completed first Runtime API poll"); + Ok(instance) } - /// Reap instances idle for longer than `idle_timeout`. Returns count. - pub(crate) fn reap_idle(&self) -> usize { + /// Reap expired idle instances and join their API/process cleanup. + pub(crate) async fn reap_idle(&self) -> Result { let now = Instant::now(); - let idle = self.idle_timeout; - let mut killed = 0usize; + let mut retired = Vec::new(); for mut bucket in self.pools.iter_mut() { - let before = bucket.len(); - bucket.retain(|(_, instance)| now.duration_since(instance.last_used) <= idle); - killed += before.saturating_sub(bucket.len()); + let mut index = 0; + while index < bucket.len() { + let expired = bucket.get(index).is_some_and(|(_, instance)| { + now.duration_since(instance.last_used) > self.idle_timeout + }); + if expired { + let (_, instance) = bucket.swap_remove(index); + retired.push(instance); + } else { + index += 1; + } + } } self.pools.retain(|_, bucket| !bucket.is_empty()); - killed + let count = retired.len(); + retire_all(retired).await?; + Ok(count) } - /// Drain and kill every instance in every pool. - pub(crate) fn shutdown(&self) { + /// Close warm retention before starting owned asynchronous cleanup. + pub(crate) fn stop_retention(&self) { + self.idle_capacity.close(); + } + + /// Stop warm retention and join every idle API/process owner. + pub(crate) async fn shutdown(&self) -> Result<(), ExecutorError> { + self.idle_capacity.close(); + let mut retired = Vec::new(); + for mut bucket in self.pools.iter_mut() { + retired.extend(take(&mut *bucket).into_iter().map(|(_, instance)| instance)); + } self.pools.clear(); + retire_all(retired).await + } +} + +fn invocation_error(error: SessionError, timeout: Duration) -> ExecutorError { + match error { + SessionError::Deadline => ExecutorError::Timeout(timeout), + SessionError::Cancelled => ExecutorError::Cancelled, + SessionError::SupervisorFailed => { + ExecutorError::CleanupFailed("Runtime session supervisor failed".into()) + } + _ => ExecutorError::RuntimeExited("Runtime session is unavailable".into()), } } +async fn retire_all(instances: Vec) -> Result<(), ExecutorError> { + // Idle retention is bounded to32; retire independently so one stalled + // HTTP connection cannot multiply the shutdown budget by the pool size. + let mut tasks = JoinSet::new(); + for instance in instances { + tasks.spawn(instance.retire()); + } + let mut failure = None; + while let Some(result) = tasks.join_next().await { + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => { + failure.get_or_insert(error); + } + Err(_) => { + failure.get_or_insert(ExecutorError::CleanupFailed( + "Idle instance cleanup task failed".into(), + )); + } + } + } + failure.map_or(Ok(()), Err) +} + /// Spawn a periodic idle-reaper background task. Stops when `cancel` flips. pub(crate) fn spawn_reaper( pool: Arc, interval: Duration, mut cancel: tokio::sync::watch::Receiver, -) -> tokio::task::JoinHandle<()> { +) -> JoinHandle> { tokio::spawn(async move { let mut tick = tokio::time::interval(interval); tick.tick().await; // skip the immediate tick loop { tokio::select! { - _ = cancel.changed() => break, + biased; + () = async { let _ = cancel.wait_for(|stopped| *stopped).await; } => break, _ = tick.tick() => { - let n = pool.reap_idle(); - if n > 0 { - debug!(reaped = n, "lambda idle reaper"); + match pool.reap_idle().await { + Ok(n) if n > 0 => debug!(reaped = n, "lambda idle reaper"), + Ok(_) => {}, + Err(error) => { pool.idle_capacity.close(); return Err(error); } } } } } + Ok(()) }) } diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 06463e92..3f97da61 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -42,8 +42,9 @@ use crate::{ pub struct NativeExecutor { pool: Arc, cancel_tx: watch::Sender, - reaper: PMutex>>, + reaper: PMutex>>>, processes: Arc, + cleanup: CleanupTask, } #[derive(Debug)] @@ -80,15 +81,7 @@ impl NativeProcesses { impl Drop for NativeExecutor { fn drop(&mut self) { - self.processes.accepting.store(false, Ordering::Release); - self.cancel_tx.send_replace(true); - self.pool.shutdown(); - if let Some(reaper) = self.reaper.lock().take() { - reaper.abort(); - } - // Process supervisors retain children, artifact leases and reservations - // until exit. Dropping the executor must not abort these owners. - self.processes.start_cleanup(); + self.start_cleanup(); } } @@ -120,8 +113,33 @@ impl NativeExecutor { cancel_tx, reaper: PMutex::new(Some(reaper)), processes, + cleanup: CleanupTask::default(), } } + + fn start_cleanup(&self) { + self.processes.accepting.store(false, Ordering::Release); + self.cancel_tx.send_replace(true); + self.pool.stop_retention(); + self.cleanup.start_on(&self.processes.runtime, || { + let pool = Arc::clone(&self.pool); + let processes = Arc::clone(&self.processes); + let reaper = self.reaper.lock().take(); + async move { + processes.start_cleanup(); + let reaper = match reaper { + Some(reaper) => reaper + .await + .map_err(|_| ExecutorError::CleanupFailed("Native reaper failed".into())) + .and_then(|result| result), + None => Ok(()), + }; + let pool = pool.shutdown().await; + let children = processes.cleanup.wait().await; + reaper.and(pool).and(children) + } + }); + } } #[async_trait] @@ -174,17 +192,8 @@ impl Executor for NativeExecutor { } async fn shutdown(&self) -> Result<(), ExecutorError> { - self.processes.accepting.store(false, Ordering::Release); - self.cancel_tx.send_replace(true); - self.pool.shutdown(); - self.processes.start_cleanup(); - let reaper = self.reaper.lock().take(); - if let Some(reaper) = reaper { - reaper - .await - .map_err(|_| ExecutorError::CleanupFailed("Native reaper failed".into()))?; - } - self.processes.cleanup.wait().await + self.start_cleanup(); + self.cleanup.wait().await } } diff --git a/crates/rustack-lambda-core/src/executor/runtime_api.rs b/crates/rustack-lambda-core/src/executor/runtime_api.rs index c32e7b95..826d19c5 100644 --- a/crates/rustack-lambda-core/src/executor/runtime_api.rs +++ b/crates/rustack-lambda-core/src/executor/runtime_api.rs @@ -1,518 +1,8 @@ -//! In-process Lambda Runtime API server. +//! Shared Lambda Runtime API reexports for existing executor consumers. //! -//! Each warm runtime instance (process or container) gets its own socket bound -//! on `127.0.0.1:0`. The bootstrap inside the instance speaks the standard -//! Lambda Runtime API to it: -//! -//! - `GET /2018-06-01/runtime/invocation/next` — long-poll for the next job. -//! - `POST /2018-06-01/runtime/invocation/{id}/response` — deliver success. -//! - `POST /2018-06-01/runtime/invocation/{id}/error` — deliver function error. -//! - `POST /2018-06-01/runtime/init/error` — fatal init failure. -//! -//! Per AWS, an instance handles one invocation at a time. The receiver is -//! locked once per `/next` call so concurrent polls (which shouldn't happen -//! in practice) serialize cleanly. - -use std::{ - convert::Infallible, - net::SocketAddr, - sync::Arc, - time::{Instant, SystemTime, UNIX_EPOCH}, -}; +//! Native and relay adapters use the same bounded runtime session implementation. -use bytes::Bytes; -use dashmap::DashMap; -use http_body_util::{BodyExt, Full, Limited}; -use hyper::{ - Method, Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, +pub use rustack_lambda_runtime::{ + http::{ApiError, RuntimeApiHandle, start, start_at}, + session::{Invocation, InvocationTicket, RuntimeResult, SessionError}, }; -use hyper_util::rt::TokioIo; -use tokio::{ - net::TcpListener, - sync::{Mutex, mpsc, oneshot, watch}, - task::{JoinHandle, JoinSet}, -}; -use tracing::{debug, warn}; - -/// One unit of work for a runtime instance: payload in, result out. -#[derive(Debug)] -pub struct Job { - /// Synthetic AWS request id (UUID). - pub request_id: String, - /// Function ARN reported via `Lambda-Runtime-Invoked-Function-Arn`. - pub function_arn: String, - /// Wall-clock deadline (used for `Lambda-Runtime-Deadline-Ms`). - pub deadline: Instant, - /// Body delivered as the invocation event. - pub payload: Bytes, - /// Where to send the bootstrap's response (or error). - pub response_tx: oneshot::Sender, -} - -/// Result of a runtime round-trip. -#[derive(Debug, Clone)] -pub enum RuntimeResult { - /// Bootstrap posted to `/response`. - Success(Bytes), - /// Bootstrap posted to `/error`. - Error(Bytes), - /// Bootstrap posted to `/init/error` before invocation. - InitError(Bytes), -} - -/// Handle to a running runtime API socket. Drop or call `shutdown` to stop. -#[derive(Debug)] -pub struct RuntimeApiHandle { - addr: SocketAddr, - job_tx: mpsc::Sender, - shutdown_tx: watch::Sender, - accept_task: Option>, - /// Optional channel that the orchestrator subscribes to in order to be - /// notified of `/init/error` failures. - init_error_rx: Mutex>>, -} - -impl RuntimeApiHandle { - /// Bound socket address. - #[must_use] - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Submit a new invocation job. Fails if the runtime task has stopped. - pub async fn submit(&self, job: Job) -> Result<(), &'static str> { - self.job_tx - .send(job) - .await - .map_err(|_| "runtime api task is gone") - } - - /// Take the init-error receiver. Returns `Some` exactly once. - pub async fn take_init_error_rx(&self) -> Option> { - self.init_error_rx.lock().await.take() - } - - /// Stop the accept loop and wait for it to exit. - pub async fn shutdown(&mut self) { - let _ = self.shutdown_tx.send(true); - if let Some(task) = self.accept_task.take() { - let _ = task.await; - } - } -} - -impl Drop for RuntimeApiHandle { - fn drop(&mut self) { - let _ = self.shutdown_tx.send(true); - if let Some(task) = self.accept_task.take() { - task.abort(); - } - } -} - -/// State shared across connection handlers. -#[derive(Debug)] -struct State { - /// Single-consumer queue of pending jobs. - job_rx: Mutex>, - /// `request_id` -> `oneshot::Sender` for the in-flight job. - pending: DashMap>, - /// One-shot to signal an init-error to the orchestrator. - init_error_tx: Mutex>>, -} - -/// Bind a fresh runtime API socket and return a handle. -/// -/// The bootstrap process should be started **after** this returns, with -/// `AWS_LAMBDA_RUNTIME_API` set to `handle.addr()`. -pub async fn start() -> std::io::Result { - let listener = TcpListener::bind(("127.0.0.1", 0)).await?; - let addr = listener.local_addr()?; - - let (job_tx, job_rx) = mpsc::channel::(8); - let (shutdown_tx, mut shutdown_rx) = watch::channel(false); - let (init_err_tx, init_err_rx) = oneshot::channel::(); - - let state = Arc::new(State { - job_rx: Mutex::new(job_rx), - pending: DashMap::new(), - init_error_tx: Mutex::new(Some(init_err_tx)), - }); - - let accept_task = tokio::spawn(async move { - let mut connections = JoinSet::new(); - loop { - tokio::select! { - result = connections.join_next(), if !connections.is_empty() => { if let Some(Err(error)) = result { warn!(%error, "Runtime API connection task failed"); } } - _ = shutdown_rx.changed() => { - debug!("runtime api: shutdown signalled, exiting accept loop"); - break; - } - accepted = listener.accept() => { - let (stream, _peer) = match accepted { - Ok(c) => c, - Err(e) => { - warn!(error = %e, "runtime api accept failed"); - continue; - } - }; - if connections.len() >= 4 { drop(stream); continue; } - let state = Arc::clone(&state); - connections.spawn(async move { - let io = TokioIo::new(stream); - let svc = service_fn(move |req| { - let state = Arc::clone(&state); - async move { handle(state, req).await } - }); - if let Err(e) = http1::Builder::new() - .serve_connection(io, svc) - .await - { - // EOF / reset is expected when the bootstrap exits. - debug!(error = %e, "runtime api connection closed"); - } - }); - } - } - } - connections.abort_all(); - while let Some(result) = connections.join_next().await { - if let Err(error) = result { - debug!(%error, "Runtime API connection stopped"); - } - } - }); - - Ok(RuntimeApiHandle { - addr, - job_tx, - shutdown_tx, - accept_task: Some(accept_task), - init_error_rx: Mutex::new(Some(init_err_rx)), - }) -} - -/// Service entry point. -async fn handle( - state: Arc, - req: Request, -) -> Result>, Infallible> { - let method = req.method().clone(); - let path = req.uri().path().to_owned(); - debug!(%method, %path, "runtime api request"); - - let result = match (method, path.as_str()) { - (Method::GET, "/2018-06-01/runtime/invocation/next") => handle_next(&state).await, - (Method::POST, p) - if p.starts_with("/2018-06-01/runtime/invocation/") && p.ends_with("/response") => - { - handle_complete(&state, &path, req, RuntimeResult::Success).await - } - (Method::POST, p) - if p.starts_with("/2018-06-01/runtime/invocation/") && p.ends_with("/error") => - { - handle_complete(&state, &path, req, RuntimeResult::Error).await - } - (Method::POST, "/2018-06-01/runtime/init/error") => handle_init_error(&state, req).await, - _ => Ok(simple(StatusCode::NOT_FOUND, b"not found".as_ref())), - }; - - Ok(result.unwrap_or_else(|status| simple(status, b"".as_ref()))) -} - -/// Long-poll for the next job. Returns once a job is queued or shutdown -/// drops the channel. -async fn handle_next(state: &Arc) -> Result>, StatusCode> { - let mut rx = state.job_rx.lock().await; - let Some(job) = rx.recv().await else { - return Err(StatusCode::SERVICE_UNAVAILABLE); - }; - drop(rx); // release before doing any work - - let Job { - request_id, - function_arn, - deadline, - payload, - response_tx, - } = job; - - state.pending.insert(request_id.clone(), response_tx); - - let deadline_ms = deadline_unix_ms(deadline); - // X-Ray-shaped trace id for runtime SDKs that expect the header to be - // present. Format isn't validated downstream — a UUID-derived value is - // fine for local emulation. - let trace_id = format!( - "Root=1-{:08x}-{};Parent={};Sampled=1", - unix_secs_lower32(), - uuid::Uuid::new_v4().simple(), - &uuid::Uuid::new_v4().simple().to_string()[..16], - ); - - let resp = Response::builder() - .status(StatusCode::OK) - .header("Lambda-Runtime-Aws-Request-Id", &request_id) - .header("Lambda-Runtime-Deadline-Ms", deadline_ms.to_string()) - .header("Lambda-Runtime-Invoked-Function-Arn", &function_arn) - .header("Lambda-Runtime-Trace-Id", trace_id) - .header("Content-Type", "application/json") - .body(Full::new(payload)) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(resp) -} - -/// Handle `/response` or `/error` — fire the matching oneshot, return 202. -async fn handle_complete( - state: &Arc, - path: &str, - req: Request, - map_result: F, -) -> Result>, StatusCode> -where - F: FnOnce(Bytes) -> RuntimeResult, -{ - let request_id = parse_request_id(path).ok_or(StatusCode::BAD_REQUEST)?; - let body = collect(req.into_body()).await?; - if let Some((_, sender)) = state.pending.remove(&request_id) { - let _ = sender.send(map_result(body)); - Ok(simple(StatusCode::ACCEPTED, b"".as_ref())) - } else { - Ok(simple( - StatusCode::BAD_REQUEST, - b"unknown request id".as_ref(), - )) - } -} - -async fn handle_init_error( - state: &Arc, - req: Request, -) -> Result>, StatusCode> { - let body = collect(req.into_body()).await?; - if let Some(tx) = state.init_error_tx.lock().await.take() { - let _ = tx.send(body); - } - Ok(simple(StatusCode::ACCEPTED, b"".as_ref())) -} - -fn parse_request_id(path: &str) -> Option { - let after = path.strip_prefix("/2018-06-01/runtime/invocation/")?; - let id = after.split('/').next()?; - if id.is_empty() { - None - } else { - Some(id.to_owned()) - } -} - -fn deadline_unix_ms(deadline: Instant) -> u64 { - let now = Instant::now(); - let remaining = deadline.saturating_duration_since(now); - let now_ms = u64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(), - ) - .unwrap_or(u64::MAX); - now_ms.saturating_add(u64::try_from(remaining.as_millis()).unwrap_or(0)) -} - -#[allow(clippy::cast_possible_truncation)] -fn unix_secs_lower32() -> u32 { - let secs = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - // Wrapping is fine — the trace id is opaque to consumers. - (secs & 0xFFFF_FFFF) as u32 -} - -async fn collect(body: Incoming) -> Result { - tokio::time::timeout( - std::time::Duration::from_secs(5), - Limited::new(body, 6 * 1024 * 1024).collect(), - ) - .await - .map_err(|_| StatusCode::REQUEST_TIMEOUT)? - .map(http_body_util::Collected::to_bytes) - .map_err(|error| { - if error.is::() { - StatusCode::PAYLOAD_TOO_LARGE - } else { - StatusCode::BAD_REQUEST - } - }) -} - -fn simple(status: StatusCode, body: &[u8]) -> Response> { - Response::builder() - .status(status) - .body(Full::new(Bytes::copy_from_slice(body))) - .unwrap_or_else(|_| Response::new(Full::new(Bytes::from_static(b"")))) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpStream, - }; - - use super::*; - - /// Send a raw HTTP/1.1 request, return (status_line, headers_block, body_bytes). - async fn http_request( - addr: SocketAddr, - method: &str, - path: &str, - body: &[u8], - ) -> (String, String, Vec) { - let mut stream = TcpStream::connect(addr).await.unwrap(); - let req = format!( - "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nContent-Length: {len}\r\nConnection: \ - close\r\n\r\n", - len = body.len(), - ); - stream.write_all(req.as_bytes()).await.unwrap(); - if !body.is_empty() { - stream.write_all(body).await.unwrap(); - } - let mut buf = Vec::new(); - stream.read_to_end(&mut buf).await.unwrap(); - - // Split status / headers / body manually. - let split = buf - .windows(4) - .position(|w| w == b"\r\n\r\n") - .expect("header terminator"); - let head_text = String::from_utf8_lossy(&buf[..split]).to_string(); - let body_bytes = buf[split + 4..].to_vec(); - let mut lines = head_text.split("\r\n"); - let status = lines.next().unwrap_or("").to_owned(); - let headers = lines.collect::>().join("\r\n"); - (status, headers, body_bytes) - } - - fn header_value(headers: &str, name: &str) -> Option { - let needle = name.to_ascii_lowercase(); - headers.split("\r\n").find_map(|line| { - let (k, v) = line.split_once(':')?; - if k.trim().to_ascii_lowercase() == needle { - Some(v.trim().to_owned()) - } else { - None - } - }) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_round_trip_invocation_via_runtime_api() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - - let (resp_tx, resp_rx) = oneshot::channel(); - let job = Job { - request_id: "req-1".to_owned(), - function_arn: "arn:fake".to_owned(), - deadline: Instant::now() + Duration::from_secs(3), - payload: Bytes::from_static(b"{\"hi\":1}"), - response_tx: resp_tx, - }; - handle.submit(job).await.unwrap(); - - let bootstrap = tokio::spawn(async move { - let (status, headers, body) = - http_request(addr, "GET", "/2018-06-01/runtime/invocation/next", b"").await; - assert!(status.starts_with("HTTP/1.1 200"), "status: {status}"); - let request_id = - header_value(&headers, "Lambda-Runtime-Aws-Request-Id").expect("request id header"); - assert_eq!(request_id, "req-1"); - assert_eq!(&body[..], b"{\"hi\":1}"); - - let path = format!("/2018-06-01/runtime/invocation/{request_id}/response"); - let (status, _h, _b) = http_request(addr, "POST", &path, b"{\"echo\":1}").await; - assert!(status.starts_with("HTTP/1.1 202"), "post status: {status}"); - }); - - let result = tokio::time::timeout(Duration::from_secs(3), resp_rx) - .await - .expect("oneshot timed out") - .unwrap(); - match result { - RuntimeResult::Success(b) => assert_eq!(&b[..], b"{\"echo\":1}"), - other => panic!("expected Success, got {other:?}"), - } - bootstrap.await.unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_propagate_function_error() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - - let (resp_tx, resp_rx) = oneshot::channel(); - handle - .submit(Job { - request_id: "req-2".to_owned(), - function_arn: "arn:fake".to_owned(), - deadline: Instant::now() + Duration::from_secs(3), - payload: Bytes::from_static(b"{}"), - response_tx: resp_tx, - }) - .await - .unwrap(); - - tokio::spawn(async move { - let (_s, headers, _b) = - http_request(addr, "GET", "/2018-06-01/runtime/invocation/next", b"").await; - let id = header_value(&headers, "Lambda-Runtime-Aws-Request-Id").unwrap(); - let path = format!("/2018-06-01/runtime/invocation/{id}/error"); - http_request( - addr, - "POST", - &path, - br#"{"errorMessage":"boom","errorType":"TestError"}"#, - ) - .await; - }); - - let result = tokio::time::timeout(Duration::from_secs(3), resp_rx) - .await - .unwrap() - .unwrap(); - match result { - RuntimeResult::Error(b) => { - let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); - assert_eq!(v["errorType"], "TestError"); - } - other => panic!("expected Error, got {other:?}"), - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_should_capture_init_error() { - let handle = start().await.unwrap(); - let addr = handle.addr(); - let init_rx = handle.take_init_error_rx().await.unwrap(); - - tokio::spawn(async move { - http_request( - addr, - "POST", - "/2018-06-01/runtime/init/error", - br#"{"errorMessage":"init failed"}"#, - ) - .await; - }); - let body = tokio::time::timeout(Duration::from_secs(3), init_rx) - .await - .unwrap() - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(v["errorMessage"], "init failed"); - } -} diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml index 6a64cd4e..286c1547 100644 --- a/crates/rustack-lambda-runtime/Cargo.toml +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -13,13 +13,20 @@ workspace = true [dependencies] bytes = { workspace = true, features = ["serde"] } ciborium = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } typed-builder = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } +tracing-core = { workspace = true } uuid = { workspace = true } [dev-dependencies] +tracing-subscriber = { workspace = true } proptest = { workspace = true } rstest = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-runtime/src/diagnostics.rs b/crates/rustack-lambda-runtime/src/diagnostics.rs new file mode 100644 index 00000000..32828159 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/diagnostics.rs @@ -0,0 +1,76 @@ +//! Mandatory per-task dependency filter, independent of application `RUST_LOG`. +//! +//! Hyper's chunk decoder can put partial trailer bytes in TRACE fields. Keep +//! its verbose events/spans out before delegating to the application's subscriber. + +use tracing_core::{ + Dispatch, Event, Interest, Level, Metadata, Subscriber, dispatcher, + span::{Attributes, Current, Id, Record}, +}; + +#[derive(Debug)] +struct RuntimeSubscriber { + parent: Dispatch, +} + +fn allowed(metadata: &Metadata<'_>) -> bool { + let target = metadata.target(); + let dependency = ["hyper", "hyper_util"].iter().any(|name| { + target == *name + || target + .strip_prefix(name) + .is_some_and(|suffix| suffix.starts_with("::")) + }); + !dependency || *metadata.level() <= Level::INFO +} + +impl Subscriber for RuntimeSubscriber { + fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest { + if allowed(metadata) { + self.parent.register_callsite(metadata) + } else { + Interest::never() + } + } + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + allowed(metadata) && self.parent.enabled(metadata) + } + fn new_span(&self, attributes: &Attributes<'_>) -> Id { + self.parent.new_span(attributes) + } + fn record(&self, id: &Id, values: &Record<'_>) { + self.parent.record(id, values); + } + fn record_follows_from(&self, id: &Id, follows: &Id) { + self.parent.record_follows_from(id, follows); + } + fn event(&self, event: &Event<'_>) { + // Recheck even when a different live dispatcher enabled this callsite. + if allowed(event.metadata()) { + self.parent.event(event); + } + } + fn enter(&self, id: &Id) { + self.parent.enter(id); + } + fn exit(&self, id: &Id) { + self.parent.exit(id); + } + fn clone_span(&self, id: &Id) -> Id { + self.parent.clone_span(id) + } + fn try_close(&self, id: Id) -> bool { + self.parent.try_close(id) + } + fn current_span(&self) -> Current { + self.parent.current_span() + } +} + +pub(crate) fn runtime_dispatch() -> Dispatch { + dispatcher::get_default(|parent| { + Dispatch::new(RuntimeSubscriber { + parent: parent.clone(), + }) + }) +} diff --git a/crates/rustack-lambda-runtime/src/http.rs b/crates/rustack-lambda-runtime/src/http.rs new file mode 100644 index 00000000..c0ccb3b3 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/http.rs @@ -0,0 +1,538 @@ +//! Loopback Runtime API adapter backed by the shared session actor. +//! +//! One supervisor owns the session, listener and bounded HTTP connection tasks. +//! Infrastructure diagnostics never include peer targets, headers or payloads. + +use std::{ + convert::Infallible, + io, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, + time::Duration, +}; + +use ::http::{ + HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri, + header::{CONNECTION, CONTENT_LENGTH, CONTENT_TYPE}, +}; +use bytes::Bytes; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::{body::Incoming, server::conn::http1, service::service_fn}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use thiserror::Error; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::{oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use crate::{ + diagnostics::runtime_dispatch, + metadata::deadline_unix_ms, + protocol::{AttemptId, MAX_PAYLOAD_BYTES, Outcome, Payload}, + session::{Invocation, RuntimeSession, SessionError, SessionHandle}, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const HEADER_BYTES: usize = 16 * 1024; +const HEADER_COUNT: usize = 64; +const HEADER_VALUE_BYTES: usize = 256; +const TARGET_BYTES: usize = 256; +const CONNECTIONS: usize = 4; +const INVOCATION_ID: &str = "lambda-runtime-invocation-id"; + +/// Adapter/supervision failures without invalid peer data in their diagnostics. +#[derive(Debug, Clone, Error)] +#[non_exhaustive] +pub enum ApiError { + /// Listener IO failure. + #[error("runtime API listener IO failed")] + Io(#[source] Arc), + /// Shared session failed. + #[error(transparent)] + Session(#[from] SessionError), + /// A supervised HTTP task failed or exceeded graceful shutdown. + #[error("runtime API connection failed")] + ConnectionFailed, + /// The server task failed to join. + #[error("runtime API supervisor failed")] + SupervisorFailed, + /// Runtime API listeners must remain environment-local. + #[error("runtime API requires a loopback address")] + NonLoopback, +} + +impl From for ApiError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} + +#[derive(Debug)] +struct ServerTask { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +impl ServerTask { + async fn shutdown(&mut self) -> Result<(), ApiError> { + self.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| ApiError::SupervisorFailed) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settlement = Some(result.clone()); + result + } +} + +impl Drop for ServerTask { + fn drop(&mut self) { + // Fallback: the retained supervisor still owns its session/connections. + // Normal shutdown joins this handle before the parent Tokio runtime stops. + self.stop.send_replace(true); + } +} + +/// Owned loopback listener and shared-session command port. +#[derive(Debug)] +#[non_exhaustive] +pub struct RuntimeApiHandle { + addr: SocketAddr, + session: SessionHandle, + server: ServerTask, + initialization_error: Option>, +} + +impl RuntimeApiHandle { + /// Bound listener address for `AWS_LAMBDA_RUNTIME_API`. + /// + /// ``` + /// # fn inspect(api: &rustack_lambda_runtime::http::RuntimeApiHandle) { + /// assert!(api.addr().ip().is_loopback()); + /// # } + /// ``` + #[must_use] + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Shared session port used by the executor or control relay. + /// + /// ``` + /// # fn inspect(api: &rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let session = api.session().clone(); + /// # } + /// ``` + #[must_use] + pub fn session(&self) -> &SessionHandle { + &self.session + } + + /// Transfer the initialization-error receiver at most once. + /// + /// ``` + /// # fn inspect(api: &mut rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let notification = api.take_initialization_error(); + /// assert!(api.take_initialization_error().is_none()); + /// # } + /// ``` + pub fn take_initialization_error(&mut self) -> Option> { + self.initialization_error.take() + } + + /// Stop admission and join all connection/session tasks. + /// + /// # Errors + /// Returns cached IO/session/supervisor failures; dropping a waiter retains ownership. + /// + /// ```no_run + /// # async fn stop(api: &mut rustack_lambda_runtime::http::RuntimeApiHandle) { + /// let clean = api.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), ApiError> { + self.server.shutdown().await + } +} + +/// Start the default Native loopback adapter with a ten-minute idle lease. +/// +/// # Errors +/// Returns listener/session startup failures. +/// +/// ```no_run +/// # async fn example() -> Result<(), rustack_lambda_runtime::http::ApiError> { +/// let mut api = rustack_lambda_runtime::http::start().await?; +/// api.shutdown().await?; +/// # Ok(()) } +/// ``` +pub async fn start() -> Result { + start_at( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + Duration::from_mins(10), + ) + .await +} + +/// Bind an explicit loopback endpoint, shared by Native and the relay. +/// +/// # Errors +/// Rejects non-loopback addresses or listener/session startup failures. +/// +/// ```no_run +/// # async fn example() -> Result<(), rustack_lambda_runtime::http::ApiError> { +/// let mut api = rustack_lambda_runtime::http::start_at("127.0.0.1:9001".parse().map_err(|_| rustack_lambda_runtime::http::ApiError::NonLoopback)?, std::time::Duration::from_secs(60)).await?; +/// api.shutdown().await?; +/// # Ok(()) } +/// ``` +#[tracing::instrument(skip_all)] +pub async fn start_at( + addr: SocketAddr, + idle_lease: Duration, +) -> Result { + if !addr.ip().is_loopback() { + return Err(ApiError::NonLoopback); + } + let listener = timeout(REQUEST_TIMEOUT, TcpListener::bind(addr)) + .await + .map_err(|_| ApiError::ConnectionFailed)??; + let addr = listener.local_addr()?; + let mut owner = RuntimeSession::start(idle_lease)?; + let session = owner.handle().clone(); + let initialization_error = owner.take_initialization_error(); + let (stop, _) = watch::channel(false); + let task = tokio::spawn( + serve(listener, owner, idle_lease, stop.clone()).with_subscriber(runtime_dispatch()), + ); + Ok(RuntimeApiHandle { + addr, + session, + server: ServerTask { + stop, + task: Some(task), + settlement: None, + }, + initialization_error, + }) +} + +async fn serve( + listener: TcpListener, + mut owner: RuntimeSession, + idle_lease: Duration, + stop: watch::Sender, +) -> Result<(), ApiError> { + let session = owner.handle().clone(); + let mut stopped = stop.subscribe(); + let mut connections = JoinSet::new(); + let mut failure = None; + loop { + tokio::select! { + biased; + () = async { let _ = stopped.wait_for(|stopped| *stopped).await; } => break, + () = session.closed() => break, + completed = connections.join_next(), if !connections.is_empty() => { + if !matches!(completed, Some(Ok(Ok(())))) { failure = Some(ApiError::ConnectionFailed); break; } + } + accepted = listener.accept() => { + match accepted { + Ok((socket, _)) if connections.len() < CONNECTIONS => { + let session = session.clone(); + let stop = stop.subscribe(); + connections.spawn(connection(socket, session, idle_lease, stop).with_current_subscriber()); + } + Ok(_) => {} + Err(error) => { failure = Some(ApiError::from(error)); break; } + } + } + } + } + drop(listener); + stop.send_replace(true); + session.close(); + while let Some(result) = connections.join_next().await { + if !matches!(result, Ok(Ok(()))) { + failure.get_or_insert(ApiError::ConnectionFailed); + } + } + if let Err(error) = owner.shutdown().await { + failure.get_or_insert(ApiError::Session(error)); + } + failure.map_or(Ok(()), Err) +} + +#[tracing::instrument(skip_all)] +async fn connection( + socket: TcpStream, + session: SessionHandle, + idle_lease: Duration, + mut stop: watch::Receiver, +) -> Result<(), ApiError> { + let service_session = session.clone(); + let service = service_fn(move |request| handle(service_session.clone(), idle_lease, request)); + let mut builder = http1::Builder::new(); + builder + .timer(TokioTimer::new()) + .header_read_timeout(REQUEST_TIMEOUT) + .max_headers(HEADER_COUNT) + .max_buf_size(HEADER_BYTES); + let connection = builder.serve_connection(TokioIo::new(socket), service); + tokio::pin!(connection); + let result = tokio::select! { + biased; + () = async { let _ = stop.wait_for(|stopped| *stopped).await; } => { + connection.as_mut().graceful_shutdown(); + timeout(REQUEST_TIMEOUT, &mut connection).await.map_err(|_| ApiError::ConnectionFailed)? + } + result = &mut connection => result, + }; + // Idle keepalive/header timeouts close only that connection. The session's + // own deadline bounds long polls and writes; other IO failures invalidate it. + match result { + Ok(()) => Ok(()), + Err(error) if error.is_timeout() => Ok(()), + Err(_) => { + session.close(); + Err(ApiError::ConnectionFailed) + } + } +} + +#[derive(Debug, Clone, Copy)] +enum Route { + Next, + Complete { + attempt: AttemptId, + outcome: Outcome, + }, + InitError, +} + +fn route(method: &Method, target: &Uri) -> Result { + if target + .path_and_query() + .is_some_and(|value| value.as_str().len() > TARGET_BYTES) + || target.scheme().is_some() + || target.authority().is_some() + || target.query().is_some() + { + return Err(StatusCode::BAD_REQUEST); + } + let path = target.path(); + if path == "/2018-06-01/runtime/invocation/next" { + return if method == Method::GET { + Ok(Route::Next) + } else { + Err(StatusCode::METHOD_NOT_ALLOWED) + }; + } + if path == "/2018-06-01/runtime/init/error" { + return if method == Method::POST { + Ok(Route::InitError) + } else { + Err(StatusCode::METHOD_NOT_ALLOWED) + }; + } + let rest = path + .strip_prefix("/2018-06-01/runtime/invocation/") + .ok_or(StatusCode::NOT_FOUND)?; + let (attempt, suffix) = rest.split_once('/').ok_or(StatusCode::NOT_FOUND)?; + let outcome = match suffix { + "response" => Outcome::Success, + "error" => Outcome::FunctionError, + _ => return Err(StatusCode::NOT_FOUND), + }; + if method != Method::POST { + return Err(StatusCode::METHOD_NOT_ALLOWED); + } + Ok(Route::Complete { + attempt: attempt.parse().map_err(|_| StatusCode::BAD_REQUEST)?, + outcome, + }) +} + +fn validate_headers(headers: &HeaderMap) -> Result<(), StatusCode> { + if headers.len() > HEADER_COUNT { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + let mut total = 0usize; + for (name, value) in headers { + if value.as_bytes().len() > HEADER_VALUE_BYTES + || !value + .as_bytes() + .iter() + .all(|byte| (32..=126).contains(byte)) + { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + total = total + .checked_add(name.as_str().len()) + .and_then(|n| n.checked_add(value.as_bytes().len())) + .ok_or(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE)?; + if total > HEADER_BYTES { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + } + Ok(()) +} + +async fn handle( + session: SessionHandle, + idle_lease: Duration, + request: Request, +) -> Result>, Infallible> { + Ok(dispatch(&session, idle_lease, request) + .await + .unwrap_or_else(error_response)) +} + +async fn dispatch( + session: &SessionHandle, + idle_lease: Duration, + request: Request, +) -> Result>, StatusCode> { + validate_headers(request.headers())?; + let route = route(request.method(), request.uri())?; + let invocation_id = if matches!(route, Route::Complete { .. }) { + if request.headers().get_all(INVOCATION_ID).iter().count() > 1 { + return Ok(invalid_invocation_id()); + } + match request.headers().get(INVOCATION_ID) { + None => None, + Some(value) => match value + .to_str() + .ok() + .and_then(|value| value.parse::().ok()) + { + Some(id) => Some(id), + None => return Ok(invalid_invocation_id()), + }, + } + } else { + None + }; + if matches!(route, Route::Complete { attempt, .. } if invocation_id.is_some_and(|id| id != attempt)) + { + return Ok(invalid_invocation_id()); + } + let limit = if matches!(route, Route::Next) { + 0 + } else { + MAX_PAYLOAD_BYTES + }; + if request + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > limit as u64) + { + return Err(StatusCode::PAYLOAD_TOO_LARGE); + } + let deadline = Instant::now() + .checked_add(REQUEST_TIMEOUT) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let body = timeout_at(deadline, Limited::new(request.into_body(), limit).collect()) + .await + .map_err(|_| StatusCode::REQUEST_TIMEOUT)? + .map_err(|error| { + if error.is::() { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + } + })?; + if body.trailers().is_some() { + return Err(StatusCode::BAD_REQUEST); + } + let payload = Payload::try_from(body.to_bytes()).map_err(|_| StatusCode::PAYLOAD_TOO_LARGE)?; + match route { + Route::Next => { + let deadline = Instant::now() + .checked_add(idle_lease) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let invocation = session.next(deadline).await.map_err(status)?; + match invocation_response(&invocation) { + Ok(response) => Ok(response), + Err(error) => { + session.close(); + Err(error) + } + } + } + Route::Complete { attempt, outcome } => { + let completed = session + .complete(attempt, invocation_id, outcome, payload, deadline) + .await; + if matches!(completed, Err(SessionError::UnknownAttempt)) && invocation_id.is_some() { + return Ok(invalid_invocation_id()); + } + completed.map_err(status)?; + Ok(response(StatusCode::ACCEPTED, Bytes::new(), false)) + } + Route::InitError => { + session + .initialization_error(payload, deadline) + .await + .map_err(status)?; + Ok(response(StatusCode::ACCEPTED, Bytes::new(), false)) + } + } +} + +fn invocation_response(invocation: &Invocation) -> Result>, StatusCode> { + let (attempt, deadline, arn, trace, payload) = invocation.parts(); + let deadline = deadline_unix_ms(deadline).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Response::builder() + .status(StatusCode::OK) + .header("Lambda-Runtime-Aws-Request-Id", attempt.to_string()) + .header(INVOCATION_ID, attempt.to_string()) + .header("Lambda-Runtime-Deadline-Ms", deadline.get().to_string()) + .header("Lambda-Runtime-Invoked-Function-Arn", arn.as_ref()) + .header("Lambda-Runtime-Trace-Id", trace.as_ref()) + .header(CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(payload.clone()))) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) +} + +fn status(error: SessionError) -> StatusCode { + match error { + SessionError::Busy => StatusCode::CONFLICT, + SessionError::UnknownAttempt | SessionError::InvalidMetadata => StatusCode::BAD_REQUEST, + SessionError::Deadline => StatusCode::REQUEST_TIMEOUT, + _ => StatusCode::SERVICE_UNAVAILABLE, + } +} + +fn response(status: StatusCode, body: Bytes, close: bool) -> Response> { + let mut response = Response::new(Full::new(body)); + *response.status_mut() = status; + if close { + response + .headers_mut() + .insert(CONNECTION, HeaderValue::from_static("close")); + } + response +} + +fn error_response(status: StatusCode) -> Response> { + response(status, Bytes::new(), true) +} + +fn invalid_invocation_id() -> Response> { + response(StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"errorType":"InvalidInvocationId","errorMessage":"Invocation ID does not match active invocation"}"#), true) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/http/tests.rs b/crates/rustack-lambda-runtime/src/http/tests.rs new file mode 100644 index 00000000..608c7267 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/http/tests.rs @@ -0,0 +1,502 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use rstest::rstest; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, +}; + +use super::*; +use crate::{metadata::local_trace_id, session::RuntimeResult}; + +#[derive(Debug)] +struct Reply { + status: u16, + headers: Vec<(String, String)>, + body: Vec, +} + +#[derive(Debug, Clone)] +struct LogWriter(Arc>>); + +impl io::Write for LogWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self.0.lock().unwrap(); + if output.len().saturating_add(bytes.len()) > 2 * 1024 * 1024 { + return Err(io::Error::other("log capture capacity exceeded")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Reply { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str())) + } +} + +async fn raw(addr: SocketAddr, request: Vec) -> Reply { + timeout(Duration::from_secs(3), async { + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(&request).await.unwrap(); + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).await.unwrap(); + let position = bytes + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .unwrap(); + let headers = String::from_utf8(bytes.get(..position).unwrap().to_vec()).unwrap(); + let mut lines = headers.lines(); + let status = lines + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .parse() + .unwrap(); + let headers = lines + .map(|line| { + let (key, value) = line.split_once(':').unwrap(); + (key.into(), value.trim().into()) + }) + .collect(); + Reply { + status, + headers, + body: bytes.get(position + 4..).unwrap().to_vec(), + } + }) + .await + .unwrap() +} + +async fn request(addr: SocketAddr, method: &str, path: &str, headers: &str, body: &[u8]) -> Reply { + let mut bytes = format!( + "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\nContent-Length: \ + {}\r\n{headers}\r\n", + body.len() + ) + .into_bytes(); + bytes.extend_from_slice(body); + raw(addr, bytes).await +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(10), + "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"secret-event\0\xff")).unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn test_should_round_trip_binary_invocations_and_supply_complete_runtime_metadata() { + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + let response = request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + assert_eq!(response.status, 200); + assert_eq!(response.body, call.parts().4.as_ref()); + assert_eq!( + response.header("Lambda-Runtime-Aws-Request-Id").unwrap(), + ticket.attempt().to_string() + ); + assert_eq!( + response.header(INVOCATION_ID).unwrap(), + ticket.attempt().to_string() + ); + assert_eq!( + response.header("Lambda-Runtime-Invoked-Function-Arn"), + Some("arn:aws:lambda:us-east-1:000000000000:function:echo:prod") + ); + assert!( + response + .header("Lambda-Runtime-Trace-Id") + .unwrap() + .ends_with("Sampled=0") + ); + assert!( + response + .header("Lambda-Runtime-Deadline-Ms") + .unwrap() + .parse::() + .unwrap() + > 0 + ); + let path = format!( + "/2018-06-01/runtime/invocation/{}/response", + ticket.attempt() + ); + let result = request( + api.addr(), + "POST", + &path, + &format!("Lambda-Runtime-Invocation-Id: {}\r\n", ticket.attempt()), + b"secret-result\0\xff", + ) + .await; + assert_eq!(result.status, 202); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::Success(payload) if payload.as_ref() == b"secret-result\0\xff") + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_preserve_current_attempt_after_unknown_duplicate_or_wrong_header_completions() +{ + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + let path = format!("/2018-06-01/runtime/invocation/{}/error", ticket.attempt()); + let mismatch = request( + api.addr(), + "POST", + &path, + &format!("Lambda-Runtime-Invocation-Id: {}\r\n", AttemptId::default()), + b"{}", + ) + .await; + assert_eq!(mismatch.status, 400); + assert!( + String::from_utf8(mismatch.body) + .unwrap() + .contains("InvalidInvocationId") + ); + let duplicate_headers = request( + api.addr(), + "POST", + &path, + &format!( + "Lambda-Runtime-Invocation-Id: {}\r\nLambda-Runtime-Invocation-Id: {}\r\n", + ticket.attempt(), + ticket.attempt() + ), + b"{}", + ) + .await; + assert_eq!(duplicate_headers.status, 400); + let unknown = request( + api.addr(), + "POST", + &format!( + "/2018-06-01/runtime/invocation/{}/response", + AttemptId::default() + ), + "", + b"{}", + ) + .await; + assert_eq!(unknown.status, 400); + assert_eq!( + request(api.addr(), "POST", &path, "", b"function-error") + .await + .status, + 202 + ); + assert!( + matches!(ticket.wait().await.unwrap(), RuntimeResult::FunctionError(payload) if payload.as_ref() == b"function-error") + ); + assert_eq!( + request(api.addr(), "POST", &path, "", b"duplicate") + .await + .status, + 400 + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_acknowledge_initialization_error_before_joining_the_closed_session() { + let mut api = start().await.unwrap(); + let receiver = api.take_initialization_error().unwrap(); + assert!(api.take_initialization_error().is_none()); + assert_eq!( + request( + api.addr(), + "POST", + "/2018-06-01/runtime/init/error", + "", + b"secret-init-error" + ) + .await + .status, + 202 + ); + assert_eq!(receiver.await.unwrap().as_ref(), b"secret-init-error"); + api.shutdown().await.unwrap(); +} + +#[rstest] +#[case("GET", "/2018-06-01/runtime/invocation/next?secret-query=1", 400)] +#[case("GET", "http://localhost/2018-06-01/runtime/invocation/next", 400)] +#[case("POST", "/2018-06-01/runtime/invocation/next", 405)] +#[case("GET", "/2018-06-01/runtime/init/error", 405)] +#[case("POST", "/2018-06-01/runtime/invocation/not-an-id/response", 400)] +#[case( + "POST", + "/2018-06-01/runtime/invocation/00112233-4455-6677-8899-aabbccddeeff/extra/response", + 404 +)] +#[tokio::test] +async fn test_should_reject_invalid_routes_before_bootstrap_readiness( + #[case] method: &str, + #[case] path: &str, + #[case] expected: u16, +) { + let mut api = start().await.unwrap(); + assert_eq!( + request(api.addr(), method, path, "", b"").await.status, + expected + ); + assert_eq!( + api.session().bootstrap_ready(Instant::now()).await, + Err(SessionError::Deadline) + ); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_bound_headers_get_bodies_and_declared_post_bodies_before_session_commands() { + let mut api = start().await.unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"x" + ) + .await + .status, + 413 + ); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + &format!("X-Secret: {}\r\n", "s".repeat(257)), + b"" + ) + .await + .status, + 431 + ); + let request = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: \ + close\r\nContent-Length: {}\r\n\r\n", + MAX_PAYLOAD_BYTES + 1 + ) + .into_bytes(); + assert_eq!(raw(api.addr(), request).await.status, 413); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_enforce_streamed_chunked_body_limit_without_accepting_init_error() { + let mut api = start().await.unwrap(); + let mut notification = api.take_initialization_error().unwrap(); + let stream = TcpStream::connect(api.addr()).await.unwrap(); + let (mut reader, mut writer) = stream.into_split(); + let send = tokio::spawn(async move { + writer.write_all(b"POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n").await.unwrap(); + let header = format!("{:x}\r\n", MAX_PAYLOAD_BYTES + 1); + writer.write_all(header.as_bytes()).await.unwrap(); + let _ = writer.write_all(&vec![0xff; MAX_PAYLOAD_BYTES + 1]).await; + let _ = writer.write_all(b"\r\n0\r\n\r\n").await; + }); + let mut reply = Vec::new(); + timeout(Duration::from_secs(3), reader.read_to_end(&mut reply)) + .await + .unwrap() + .unwrap(); + send.await.unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 413")); + assert!(notification.try_recv().is_err()); + api.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_keep_shutdown_owned_when_its_waiter_is_dropped_and_close_the_listener() { + let mut api = start().await.unwrap(); + let addr = api.addr(); + let mut waiting = Box::pin(api.shutdown()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(api.server.task.is_some()); + api.shutdown().await.unwrap(); + api.shutdown().await.unwrap(); + assert!(TcpStream::connect(addr).await.is_err()); + assert!(matches!( + start_at("0.0.0.0:0".parse().unwrap(), Duration::from_secs(60)).await, + Err(ApiError::NonLoopback) + )); +} + +#[tokio::test] +async fn test_should_retain_failed_supervision_on_repeated_shutdown() { + let mut api = start().await.unwrap(); + api.server.task.as_ref().unwrap().abort(); + assert!(matches!( + api.shutdown().await, + Err(ApiError::SupervisorFailed) + )); + assert!(matches!( + api.shutdown().await, + Err(ApiError::SupervisorFailed) + )); +} + +#[tokio::test] +async fn test_should_omit_payloads_headers_and_targets_from_trace_level_runtime_diagnostics() { + let logs = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = Arc::clone(&logs); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::TRACE) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + .with_writer(move || LogWriter(Arc::clone(&captured))) + .finish(); + // This current-thread runtime keeps its tasks under one thread-local + // dispatcher, without changing process-wide logging or environment settings. + let _subscriber = tracing::subscriber::set_default(subscriber); + let mut api = start().await.unwrap(); + let call = invocation(); + let ticket = api + .session() + .submit(call.clone(), call.parts().1) + .await + .unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "X-Secret: secret-header\r\n", + b"" + ) + .await + .status, + 200 + ); + let path = format!( + "/2018-06-01/runtime/invocation/{}/response", + ticket.attempt() + ); + assert_eq!( + request(api.addr(), "POST", &path, "", b"secret-result") + .await + .status, + 202 + ); + ticket.wait().await.unwrap(); + assert_eq!( + request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next?secret-target", + "", + b"" + ) + .await + .status, + 400 + ); + let mut stream = TcpStream::connect(api.addr()).await.unwrap(); + stream.write_all(b"POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX-Secret: secret-fragmented-trailer").await.unwrap(); + // Force another decoder poll with a partially buffered trailer. + tokio::time::sleep(Duration::from_millis(30)).await; + stream.write_all(b"\r\n\r\n").await.unwrap(); + let mut reply = Vec::new(); + timeout(Duration::from_secs(3), stream.read_to_end(&mut reply)) + .await + .unwrap() + .unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 400")); + api.shutdown().await.unwrap(); + let output = String::from_utf8(logs.lock().unwrap().clone()).unwrap(); + assert!( + !output.is_empty(), + "trace capture must observe runtime spans" + ); + for secret in [ + "secret-event", + "secret-header", + "secret-result", + "secret-target", + "secret-fragmented-trailer", + ] { + assert!( + !output.contains(secret), + "runtime diagnostics leaked {secret}" + ); + } +} + +#[tokio::test] +async fn test_should_reject_all_trailers_before_initialization_or_invocation_mutation() { + let mut api = start().await.unwrap(); + let mut notification = api.take_initialization_error().unwrap(); + for trailer in [ + "X-Secret: secret-trailer".to_owned(), + format!("X-Long: {}", "s".repeat(300)), + format!("Lambda-Runtime-Invocation-Id: {}", AttemptId::default()), + ] { + let bytes = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: \ + close\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n{trailer}\r\n\r\n" + ) + .into_bytes(); + assert_eq!(raw(api.addr(), bytes).await.status, 400); + assert!(notification.try_recv().is_err()); + assert!(!api.session().is_closed()); + } + api.shutdown().await.unwrap(); +} diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index 356f9629..df94bf82 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -6,5 +6,8 @@ #![forbid(unsafe_code)] +mod diagnostics; +pub mod http; +pub mod metadata; pub mod protocol; pub mod session; diff --git a/crates/rustack-lambda-runtime/src/metadata.rs b/crates/rustack-lambda-runtime/src/metadata.rs new file mode 100644 index 00000000..1ee6d34b --- /dev/null +++ b/crates/rustack-lambda-runtime/src/metadata.rs @@ -0,0 +1,134 @@ +//! Checked wall/monotonic deadline conversion and local tracing metadata. + +use std::{ + num::NonZeroU64, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use thiserror::Error; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::protocol::TraceId; + +/// Clock/metadata failures contain no invocation contents. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum MetadataError { + /// Supplied deadline has already elapsed. + #[error("runtime deadline expired")] + Expired, + /// Clock/range/conversion cannot represent the metadata. + #[error("invalid runtime clock metadata")] + Invalid, +} + +/// Convert a future monotonic deadline into checked Unix milliseconds. +/// +/// # Errors +/// Rejects expired deadlines, a pre-epoch wall clock or arithmetic overflow. +/// +/// ``` +/// let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); +/// assert!(rustack_lambda_runtime::metadata::deadline_unix_ms(deadline).is_ok()); +/// ``` +pub fn deadline_unix_ms(deadline: Instant) -> Result { + to_wall(deadline, Instant::now(), SystemTime::now()) +} + +fn to_wall(deadline: Instant, now: Instant, wall: SystemTime) -> Result { + let remaining = deadline + .checked_duration_since(now) + .filter(|remaining| !remaining.is_zero()) + .ok_or(MetadataError::Expired)?; + let epoch = wall + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)?; + let total = epoch.checked_add(remaining).ok_or(MetadataError::Invalid)?; + NonZeroU64::new(u64::try_from(total.as_millis()).map_err(|_| MetadataError::Invalid)?) + .ok_or(MetadataError::Invalid) +} + +/// Translate an incoming wall deadline into a bounded monotonic deadline. +/// +/// # Errors +/// Rejects expired deadlines, invalid clocks and more than one hour remaining. +/// Lambda's≤900s handler plus bounded startup fits within this local ceiling. +/// +/// ``` +/// # use rustack_lambda_runtime::metadata::{deadline_unix_ms, deadline_from_unix_ms}; +/// let value = deadline_unix_ms(tokio::time::Instant::now() + std::time::Duration::from_secs(5))?; +/// assert!(deadline_from_unix_ms(value).is_ok()); +/// # Ok::<(), rustack_lambda_runtime::metadata::MetadataError>(()) +/// ``` +pub fn deadline_from_unix_ms(value: NonZeroU64) -> Result { + from_wall(value, Instant::now(), SystemTime::now()) +} + +fn from_wall(value: NonZeroU64, now: Instant, wall: SystemTime) -> Result { + let epoch = wall + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)?; + let remaining = Duration::from_millis(value.get()) + .checked_sub(epoch) + .filter(|remaining| !remaining.is_zero()) + .ok_or(MetadataError::Expired)?; + if remaining > Duration::from_hours(1) { + return Err(MetadataError::Invalid); + } + now.checked_add(remaining).ok_or(MetadataError::Invalid) +} + +/// Generate an X-Ray-shaped local trace, without claiming upstream sampling. +/// +/// # Errors +/// Returns Invalid if the wall clock/validated header cannot be represented. +/// +/// ``` +/// assert!(rustack_lambda_runtime::metadata::local_trace_id()?.as_ref().ends_with("Sampled=0")); +/// # Ok::<(), rustack_lambda_runtime::metadata::MetadataError>(()) +/// ``` +pub fn local_trace_id() -> Result { + let epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| MetadataError::Invalid)? + .as_secs() + % (1_u64 << 32); + let root = Uuid::new_v4().simple().to_string(); + let parent = Uuid::new_v4().simple().to_string(); + let root = root.get(..24).ok_or(MetadataError::Invalid)?; + let parent = parent.get(..16).ok_or(MetadataError::Invalid)?; + format!("Root=1-{epoch:08x}-{root};Parent={parent};Sampled=0") + .parse() + .map_err(|_| MetadataError::Invalid) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_convert_checked_fractional_deadlines_and_reject_invalid_clocks() { + let now = Instant::now(); + let wall = UNIX_EPOCH + Duration::from_millis(1000); + let deadline = now + Duration::from_micros(2500); + assert_eq!(to_wall(deadline, now, wall).unwrap().get(), 1002); + assert_eq!( + from_wall(NonZeroU64::new(1002).unwrap(), now, wall).unwrap(), + now + Duration::from_millis(2) + ); + assert_eq!(to_wall(now, now, wall), Err(MetadataError::Expired)); + assert_eq!( + to_wall(deadline, now, UNIX_EPOCH - Duration::from_secs(1)), + Err(MetadataError::Invalid) + ); + assert_eq!( + from_wall(NonZeroU64::new(1000).unwrap(), now, wall), + Err(MetadataError::Expired) + ); + assert_eq!( + from_wall(NonZeroU64::new(u64::MAX).unwrap(), now, wall), + Err(MetadataError::Invalid) + ); + } +} diff --git a/crates/rustack-lambda-runtime/src/session.rs b/crates/rustack-lambda-runtime/src/session.rs index b646609e..26173639 100644 --- a/crates/rustack-lambda-runtime/src/session.rs +++ b/crates/rustack-lambda-runtime/src/session.rs @@ -490,6 +490,29 @@ impl SessionHandle { pub fn close(&self) { self.stop.send_replace(true); } + + /// Wait until the actor closes its command receiver. + /// + /// ```no_run + /// # async fn closed(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// handle.closed().await; + /// # } + /// ``` + pub async fn closed(&self) { + self.inbox.closed().await; + } + + /// Whether admission is closed or its owner has requested stop. + /// + /// ``` + /// # fn inspect(handle: &rustack_lambda_runtime::session::SessionHandle) { + /// let closed = handle.is_closed(); + /// # } + /// ``` + #[must_use] + pub fn is_closed(&self) -> bool { + *self.stop.borrow() || self.inbox.is_closed() + } } fn check_deadline(deadline: Instant) -> Result<(), SessionError> { diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index c621f39c..7a1b2d75 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -211,6 +211,18 @@ Offering an event through a oneshot channel is not delivery proof: retain the qu The actor prioritizes owner stop, active attempt cancellation/deadline and idle lease expiry over mailbox traffic. Activity refreshes the idle lease only at valid session commands; a running attempt uses its own deadline. Each command envelope carries its absolute RPC deadline, checked again before state mutation. Shutdown signals the actor and awaits its stored join handle by mutable borrow, so a cancelled shutdown waiter cannot lose the task. Cache failed joins so repeated shutdown never reports false success. Owner Drop signals stop and leaves the actor to finish its in-process channel cleanup; adapters still own/await their HTTP tasks and external execution resources. The parent Tokio runtime must remain alive until normal service shutdown has joined these owners. Paused-clock and contention tests pin single admission/delivery/settlement, cancelled poll replacement, stale-completion isolation, old-ticket cancellation, init/readiness, idle/attempt deadlines, overload and dropped shutdown/admission waiters. +### 3.3 Shared HTTP adapter and Native startup + +The runtime library's HTTP adapter binds only an explicitly supplied loopback SocketAddr (Native127.0.0.1:0, relay127.0.0.1:9001). RuntimeApiHandle exposes its address/session port, take-once initialization notification and cancel-safe joined shutdown. One owned server task retains RuntimeSession and a JoinSet of at most four HTTP connections. Stop/session closure ends admission, signals graceful HTTP shutdown under a five-second deadline and joins every connection, then joins the session. This lets an accepted initialization-error response reach the RIC before closure; timed-out connection futures close their sockets and fail settlement. Drop signals the retained supervisor rather than losing ownership. Failed supervision is cached and prevents a later successful shutdown claim. + +HTTP parsing limits are compile-time constants:16 KiB header buffer/aggregate header bytes,64 headers,256 bytes per value and256 bytes per request target. Reject absolute-form targets, query strings, duplicate Invocation-Id headers and nonprintable header values. Routes match the entire method/path and canonical attempt UUID; extra path segments do not reach the session. GET bodies must be empty. POST bodies are raw bounded bytes≤6 MiB, collected with a five-second absolute deadline shared with command processing. Reject all HTTP trailers before session mutation; streaming responses are outside this profile. Content-Length over the limit rejects before reading the body. Header parsing has its own five-second deadline. GET /next waits within the configured session idle lease instead of a five-second business timeout. Responses to rejected requests close the connection; normal RIC keepalive remains supported. + +Wrap the HTTP supervisor and every spawned connection future with an independent task-bound subscriber filter that suppresses Hyper/hyper_util DEBUG/TRACE regardless of the application's subscriber/RUST_LOG. Preserve allowed application span IDs/context and events through delegation. Pinned Hyper1.11.1 logs its partial chunk-trailer buffer at TRACE before application validation, so rejecting trailers alone cannot establish secrecy. Capture a fragmented secret trailer under TRACE as well as ordinary header/body/target values. This shared adapter filter is separate from the mandatory Bollard filter at the future Docker boundary. + +GET responses include Request-Id, Invocation-Id (the same synthetic attempt UUID), checked epoch-millisecond Deadline-Ms, requested Function-Arn and a syntactically valid local Trace-Id. This local profile generates a fresh request/attempt UUID for every Invoke rather than implementing event-source event identity across redelivery. If Invocation-Id is present on completion it must match the path and delivered attempt; omission stays compatible with older RICs. Mismatch uses400/InvalidInvocationId without echoing values, Busy maps409, unknown attempts400, expired requests408 and closed sessions503. Successful completion/init error returns202. Infrastructure traces contain bounded classes/counts only, not targets, headers or payloads. This follows the [current AWS Runtime API contract](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html). + +Native starts the API/session before spawning the bootstrap, then waits under one initialization deadline for first valid GET /next, initialization error, process exit or caller cancellation. Spawned children remain owned and are terminated/reaped on every failed/cancelled startup branch; API/session tasks are joined before clean settlement. The handler budget is captured after successful first-poll readiness when dispatching the event, additionally bounded by the caller's absolute deadline. Initialization time cannot consume/extend an unbounded fresh handler/Task budget. Runtime API errors still require retiring the process before propagating failure. Keep the executor module's runtime_api reexports, replacing the old pending-map/queue implementation with the shared adapter rather than maintaining a second session engine. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index e6cb5163..aa95014f 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -141,7 +141,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | -| 1.2 | In progress | Cancellation/revision foundation committed as93bd1ec; bounded protocol committed as4da7df3. Shared RuntimeSession actor now implements bounded commands, first-poll readiness, initialization failure, per-attempt cancellation/deadlines, receipt-confirmed delivery, stale completion rejection and cancel-safe joined shutdown. W17–W19 races fixed/rechecked. Workspace:1,822 passing tests; runtime:55 tests (including512-case properties),23 doctests and strict production-boundary Clippy pass. make test-lambda-runtime passes. Native HTTP migration, relay executable/socket client and startup probes remain required; no Docker runtime acceptance yet. | +| 1.2 | In progress | Bounded protocol4da7df3 and shared actor2c50c2d are committed. Native now uses the shared loopback HTTP/session adapter and waits first /next before handler dispatch. HTTP validation, checked clock metadata, mandatory Hyper log filtering, joined connection/process retirement and W20–W22 review fixes are verified. Workspace:1,841 tests; runtime:71 tests/34 doctests and strict boundary Clippy; Lambda core:212 tests/26 doctests;11 real Native fixture regressions and native SDK target pass. Relay executable/socket client and Docker startup probes remain required; no Docker runtime acceptance yet. | | 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: migrate Native HTTP to the shared session and build the relay/socket client and startup probes. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: build the relay executable/socket client and Docker startup probes on the shared HTTP/session implementation. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 0bcc0ae7..ad3f7cdc 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -77,6 +77,20 @@ The independent read-only review found three cancellation/order races. All were The actor owns an eight-command mailbox, one queued/offered/delivered attempt and one poll; per-attempt tickets prevent old cancellation from targeting new work. RPC envelopes carry their own deadline, readiness changes once, initialization notifies once, shutdown retains its join handle across cancelled waiters and caches failed settlement. Sixteen actor tests and the39 codec tests pass;23 runtime doctests and strict production-boundary Clippy pass. Workspace build/fmt/Clippy/rustdoc and1,822 tests pass (392 preexisting ignored). This review covers the shared actor only: the Native HTTP adapter still uses its existing session implementation until migration; the relay/socket client, Docker executor and ASL engine are not implemented by this slice. B01/B03 remain required. +## P1 task1.2 shared HTTP / Native migration review + +The independent read-only review found three boundary/lifecycle defects. All were fixed and the final recheck found no remaining concrete in-slice finding. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W20 | P1; `crates/rustack-lambda-runtime/src/http.rs:216`, pinned Hyper1.11.1 `proto/h1/decode.rs:149` | Hyper TRACE includes partial trailer bytes before application validation. Every HTTP supervisor/connection future now uses an independent subscriber denying Hyper/hyper_util DEBUG/TRACE while delegating allowed application context/events | TRACE capture covers ordinary target/header/body and a deliberately fragmented secret trailer; no secret reaches diagnostics | +| W21 | P2; `crates/rustack-lambda-runtime/src/http.rs:456` | Collected trailers bypassed per-header validation. All trailers now reject before session mutation | Short, oversized and Invocation-Id trailers return400 without initialization notification or session closure | +| W22 | P2; `crates/rustack-lambda-core/src/executor/instance.rs:309` | Revision mismatch cleared old instances through Drop, losing API join ownership. Acquisition now extracts stale instances and awaits bounded parallel retirement outside map guards; empty buckets are removed | Real old API holds an incomplete body while code changes; replacement waits its408 deadline/join and old child reap before invoking new code | + +The shared adapter replaces the old pending-map implementation while preserving executor module reexports. Native cold start now races first-poll readiness, init error, process exit and caller cancellation under one init deadline, retiring API/process owners on failure. The Native executor's one cleanup task atomically captures reaper/pool/child ownership, survives cancelled/concurrent shutdown waiters and preserves cleanup failures. Eleven real fixture tests cover slow/no polling, init failure, startup cancellation, version isolation, stale connections, busy cancellation and joined shutdown. The existing native SDK target passes six Native/DryRun scenarios; its separately gated Squib selection remains excluded as VM execution evidence. + +Workspace build/fmt/Clippy/rustdoc and1,841 tests pass (392 preexisting ignored). Shared runtime:71 tests,34 doctests and strict production-boundary Clippy; Lambda core:212 tests and26 doctests. Three old core HTTP tests moved into the shared adapter's stronger wire suite. Cargo audit still finds the same16 B01 advisories; no new dependency version/package landed, and deny bans/licenses/sources pass. B03 remains open for old core boundaries. The relay executable/client, Docker executor and ASL engine remain required before milestone completion. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index f4d51a70..ce2ef8f3 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -116,6 +116,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [shared session ownership and state diagram](./rustack-workflows-20-lambda-runtime-design.md#32-runtime-session-ownership). - Date: 2026-10-25. +## D16 — Enforce HTTP dependency secrecy and retain adapter retirement + +- Alternatives: redact only application Debug; trust RUST_LOG defaults; reject trailers after decoding without filtering the dependency; drop invalidated warm API handles without awaiting retirement. +- Decision: task-bound Hyper/hyper_util verbosity filtering independent of the application's subscriber, no supported HTTP trailers, and explicit joined API/process retirement on revision invalidation and shutdown. Cache supervision failures and retain the complete Native cleanup future across dropped/concurrent waiters. +- Why: pinned Hyper1.11.1 logs buffered trailer contents before application validation. Dropping a handle only signals asynchronous cleanup and cannot establish that an old listener's connections have finished. First-poll readiness, joined retirement and checked metadata let Native and the future Docker relay share one actual session engine. +- Pinned by: [shared HTTP adapter and Native startup](./rustack-workflows-20-lambda-runtime-design.md#33-shared-http-adapter-and-native-startup), [review W20–W22](./rustack-workflows-93-review.md#p1-task12-shared-http--native-migration-review). +- Date:2026-10-25. The separate mandatory Bollard filter W01 remains required when Docker is implemented. + ## Glossary | Term | Meaning in this specification | diff --git a/tests/lambda-fixtures/echo-bootstrap/src/main.rs b/tests/lambda-fixtures/echo-bootstrap/src/main.rs index 3ea78713..854d2fe6 100644 --- a/tests/lambda-fixtures/echo-bootstrap/src/main.rs +++ b/tests/lambda-fixtures/echo-bootstrap/src/main.rs @@ -41,6 +41,20 @@ fn main() { return; } } + if let Some(delay) = std::env::var("INIT_DELAY_MILLIS") + .ok() + .and_then(|value| value.parse::().ok()) + { + std::thread::sleep(Duration::from_millis(delay)); + } + if std::env::var("INIT_FAIL").is_ok_and(|value| value == "1") { + let _ = agent + .post(&format!("http://{api}/2018-06-01/runtime/init/error")) + .send_string( + r#"{"errorMessage":"secret-initialization-value","errorType":"TestInitError"}"#, + ); + return; + } loop { let next = agent .get(&format!("http://{api}/2018-06-01/runtime/invocation/next")) @@ -84,6 +98,7 @@ fn main() { "codeMarker": code_marker, "configMarker": config_marker, "pid": std::process::id(), + "runtimeApi": api, }); let _ = agent .post(&format!( diff --git a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs index 09393f34..b791d20e 100644 --- a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs +++ b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs @@ -75,6 +75,248 @@ async fn invoke(provider: &RustackLambda, version: Option<&str>) -> serde_json:: serde_json::from_slice(&response.payload).unwrap() } +#[tokio::test] +async fn test_should_wait_for_first_poll_before_starting_the_native_handler_budget() { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + init_timeout: Duration::from_secs(3), + ..Default::default() + }, + ); + let mut input = create("slow-init"); + input.timeout = Some(1); + input.environment = Some(Environment { + variables: Some(HashMap::from([("INIT_DELAY_MILLIS".into(), "1200".into())])), + }); + provider.create_function(input).await.unwrap(); + let started = tokio::time::Instant::now(); + assert_eq!(invoke(&provider, None).await["codeMarker"], "slow-init"); + assert!(started.elapsed() >= Duration::from_millis(1200)); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_join_stale_revision_api_connections_before_invoking_replacement_code() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + provider.create_function(create("old")).await.unwrap(); + let old = invoke(&provider, None).await; + let address = old["runtimeApi"].as_str().unwrap(); + let mut held = tokio::net::TcpStream::connect(address).await.unwrap(); + held.write_all(b"POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nContent-Length: 1\r\n\r\n").await.unwrap(); + // Keep the old API's body collector active while the revision is replaced. + tokio::time::sleep(Duration::from_millis(50)).await; + provider + .update_function_code( + "revision-test", + UpdateFunctionCodeInput { + zip_file: Some(package("new")), + ..Default::default() + }, + ) + .await + .unwrap(); + let started = tokio::time::Instant::now(); + assert_eq!(invoke(&provider, None).await["codeMarker"], "new"); + assert!( + started.elapsed() >= Duration::from_secs(4), + "replacement must await the old API connection deadline and join" + ); + let mut reply = Vec::new(); + tokio::time::timeout(Duration::from_secs(1), held.read_to_end(&mut reply)) + .await + .unwrap() + .unwrap(); + assert!(reply.starts_with(b"HTTP/1.1 408")); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(old["pid"].as_u64().unwrap().to_string()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!(!status.success()); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_time_out_and_reap_a_bootstrap_that_never_polls_the_runtime_api() { + let root = tempfile::tempdir().unwrap(); + let marker = root.path().join("started"); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Native, + init_timeout: Duration::from_secs(2), + ..Default::default() + }, + ); + let mut input = create("never-ready"); + input.timeout = Some(120); + input.environment = Some(Environment { + variables: Some(HashMap::from([ + ("INIT_DELAY_MILLIS".into(), "60000".into()), + ("STARTED_FILE".into(), marker.to_string_lossy().into_owned()), + ])), + }); + provider.create_function(input).await.unwrap(); + let started = tokio::time::Instant::now(); + assert!( + provider + .invoke("revision-test", None, b"{}", InvokeKind::RequestResponse) + .await + .is_err() + ); + assert!(started.elapsed() < Duration::from_secs(4)); + let pid = tokio::fs::read_to_string(marker).await.unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(pid.trim()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!( + !status.success(), + "initialization timeout must join child cleanup" + ); + provider.shutdown().await; +} + +#[tokio::test] +async fn test_should_report_native_initialization_error_without_echoing_its_payload() { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + let mut input = create("init-error"); + input.environment = Some(Environment { + variables: Some(HashMap::from([("INIT_FAIL".into(), "1".into())])), + }); + provider.create_function(input).await.unwrap(); + let error = provider + .invoke("revision-test", None, b"{}", InvokeKind::RequestResponse) + .await + .unwrap_err(); + assert!(!format!("{error} {error:?}").contains("secret-initialization-value")); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_cancel_and_reap_a_bootstrap_while_waiting_for_first_poll() { + let root = tempfile::tempdir().unwrap(); + let marker = root.path().join("started"); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Native, + init_timeout: Duration::from_secs(30), + ..Default::default() + }, + ); + let mut input = create("cancel-init"); + input.environment = Some(Environment { + variables: Some(HashMap::from([ + ("INIT_DELAY_MILLIS".into(), "60000".into()), + ("STARTED_FILE".into(), marker.to_string_lossy().into_owned()), + ])), + }); + provider.create_function(input).await.unwrap(); + let handle = provider + .start_invocation("revision-test", None, b"{}", None) + .await + .unwrap(); + let pid = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match tokio::fs::read_to_string(&marker).await { + Ok(pid) if !pid.is_empty() => break pid, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => {} + Err(error) => panic!("Read initialization marker: {error}"), + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + handle + .cancel_and_wait(tokio::time::Instant::now() + Duration::from_secs(5)) + .await + .unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(pid.trim()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!( + !status.success(), + "cancelled startup must reap before settlement" + ); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_join_shared_api_and_child_cleanup_after_cancelled_and_concurrent_native_shutdown_waiters() + { + use std::{ + future::Future, + task::{Context, Waker}, + }; + + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + provider.create_function(create("shutdown")).await.unwrap(); + let response = invoke(&provider, None).await; + let executor = provider.executor().clone(); + let mut waiting = Box::pin(executor.shutdown()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + let (first, second) = tokio::join!(executor.shutdown(), executor.shutdown()); + first.unwrap(); + second.unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(response["pid"].as_u64().unwrap().to_string()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!(!status.success()); + provider.shutdown().await; +} + #[tokio::test] async fn test_should_preserve_published_code_and_refresh_warm_revisions() { let root = tempfile::tempdir().unwrap(); From 6779f93debee51602a1d056b01065c672f6f1c19 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 18:33:50 -0700 Subject: [PATCH 008/106] phase 1: supervise Linux Lambda relay and qualify real image execution Add the nonblocking-pipe relay executable, bounded supervised control IO, flush-confirmed messages and cancel-safe borrowed invocation results. Retain API and worker cleanup through Close, cancellation and transport failure, preserving completed IO errors during priority shutdown. Build pinned arm64/amd64 relay images through Make. Four explicit Docker qualification tests pass on arm64 VZ and amd64/QEMU with unchanged official Python RIC computation, warm reuse, failure/cancellation and pipe/process cleanup evidence. Independent review findings W23-W26 are fixed/rechecked. Workspace build/test/fmt/Clippy/rustdoc pass (1853 tests); runtime strict boundary and actual Linux entry-point Clippy pass. Existing B01 advisory and B03 core-boundary gates remain open. Host control client, Docker executor/provider integration and the ASL engine remain required by P1-P5. --- Cargo.lock | 2 + Makefile | 17 + assets/lambda/runtime/Dockerfile | 5 + crates/rustack-lambda-runtime/Cargo.toml | 4 +- crates/rustack-lambda-runtime/src/http.rs | 2 +- .../rustack-lambda-runtime/src/http/tests.rs | 14 +- crates/rustack-lambda-runtime/src/lib.rs | 1 + crates/rustack-lambda-runtime/src/main.rs | 39 ++ crates/rustack-lambda-runtime/src/process.rs | 68 ++ crates/rustack-lambda-runtime/src/relay.rs | 525 +++++++++++++++ .../rustack-lambda-runtime/src/relay/tests.rs | 432 ++++++++++++ crates/rustack-lambda-runtime/src/session.rs | 53 +- .../src/session/tests.rs | 29 + crates/rustack-lambda-runtime/tests/docker.rs | 618 ++++++++++++++++++ ...tack-workflows-20-lambda-runtime-design.md | 14 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 17 + specs/rustack-workflows-99-decisions.md | 8 + 19 files changed, 1838 insertions(+), 16 deletions(-) create mode 100644 assets/lambda/runtime/Dockerfile create mode 100644 crates/rustack-lambda-runtime/src/main.rs create mode 100644 crates/rustack-lambda-runtime/src/process.rs create mode 100644 crates/rustack-lambda-runtime/src/relay.rs create mode 100644 crates/rustack-lambda-runtime/src/relay/tests.rs create mode 100644 crates/rustack-lambda-runtime/tests/docker.rs diff --git a/Cargo.lock b/Cargo.lock index 80fcf0b4..3f904e77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3878,6 +3878,7 @@ dependencies = [ name = "rustack-lambda-runtime" version = "0.9.0" dependencies = [ + "anyhow", "bytes", "ciborium", "http 1.5.0", @@ -3887,6 +3888,7 @@ dependencies = [ "proptest", "rstest", "serde", + "serde_json", "thiserror", "tokio", "tracing", diff --git a/Makefile b/Makefile index 7e413ff2..c755b7c9 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,23 @@ WORKFLOWS_SPIKE_DIR ?= /tmp/rustack-spikes WORKFLOWS_DOCKER_SOCKET ?= /var/run/docker.sock WORKFLOWS_ARCH ?= $(if $(filter arm64 aarch64,$(shell uname -m)),arm64,amd64) WORKFLOWS_IMAGE_BUILDER ?= docker buildx +LAMBDA_RUNTIME_DIR ?= /tmp/rustack-lambda-runtime +LAMBDA_RUNTIME_IMAGE ?= rustack/lambda-runtime:integration-$(WORKFLOWS_ARCH) + +lambda-runtime-build: + @case "$(WORKFLOWS_ARCH)" in arm64|amd64) ;; *) echo "WORKFLOWS_ARCH must be arm64 or amd64" >&2; exit 1;; esac + @cargo lambda build -p rustack-lambda-runtime --bin rustack-lambda-runtime --target "$(if $(filter arm64,$(WORKFLOWS_ARCH)),aarch64,x86_64)-unknown-linux-gnu" --release --lambda-dir "$(LAMBDA_RUNTIME_DIR)/$(WORKFLOWS_ARCH)" + +lambda-runtime-image: lambda-runtime-build + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file assets/lambda/runtime/Dockerfile --tag "$(LAMBDA_RUNTIME_IMAGE)" "$(LAMBDA_RUNTIME_DIR)/$(WORKFLOWS_ARCH)/rustack-lambda-runtime" + +lambda-runtime-fixture-image: + @DOCKER_HOST="unix://$(WORKFLOWS_DOCKER_SOCKET)" $(WORKFLOWS_IMAGE_BUILDER) build --load --platform "linux/$(WORKFLOWS_ARCH)" --file tools/workflows-spikes/Handler.Dockerfile --tag "rustack-p0-handler:$(WORKFLOWS_ARCH)" tools/workflows-spikes + +test-lambda-runtime-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-runtime --test docker -- --ignored --test-threads=1 + +.PHONY: lambda-runtime-build lambda-runtime-image lambda-runtime-fixture-image test-lambda-runtime-docker workflows-model-check: @python3 -c 'import hashlib,json,pathlib; p=pathlib.Path("codegen/smithy-model"); m=json.loads((p/"stepfunctions.provenance.json").read_text()); raw=(p/"stepfunctions.json").read_bytes(); assert len(raw)==m["bytes"] and hashlib.sha256(raw).hexdigest()==m["sha256"], "SFN source hash mismatch"; s=json.loads(raw)["shapes"][m["service"]]; assert s["version"]=="2016-11-23" and "aws.protocols#awsJson1_0" in s["traits"] and s["traits"]["aws.auth#sigv4"]["name"]=="states"; print("Pinned SFN model verified:",len(s["operations"]),"operations")' diff --git a/assets/lambda/runtime/Dockerfile b/assets/lambda/runtime/Dockerfile new file mode 100644 index 00000000..4fa08b2a --- /dev/null +++ b/assets/lambda/runtime/Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/lambda/provided:al2023@sha256:612c98f126c2a95e77a016d56fed50a83055e11d636132ee6ebdd461f01b9617 +COPY bootstrap /rustack-lambda-runtime +USER 1000:1000 +ENTRYPOINT ["/rustack-lambda-runtime"] +CMD [] diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml index 286c1547..d0ec070e 100644 --- a/crates/rustack-lambda-runtime/Cargo.toml +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -23,10 +23,12 @@ typed-builder = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-core = { workspace = true } +tracing-subscriber = { workspace = true } uuid = { workspace = true } [dev-dependencies] -tracing-subscriber = { workspace = true } +anyhow = { workspace = true } proptest = { workspace = true } rstest = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-runtime/src/http.rs b/crates/rustack-lambda-runtime/src/http.rs index c0ccb3b3..d26f38de 100644 --- a/crates/rustack-lambda-runtime/src/http.rs +++ b/crates/rustack-lambda-runtime/src/http.rs @@ -535,4 +535,4 @@ fn invalid_invocation_id() -> Response> { } #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/rustack-lambda-runtime/src/http/tests.rs b/crates/rustack-lambda-runtime/src/http/tests.rs index 608c7267..65da0227 100644 --- a/crates/rustack-lambda-runtime/src/http/tests.rs +++ b/crates/rustack-lambda-runtime/src/http/tests.rs @@ -13,10 +13,10 @@ use super::*; use crate::{metadata::local_trace_id, session::RuntimeResult}; #[derive(Debug)] -struct Reply { - status: u16, +pub(crate) struct Reply { + pub(crate) status: u16, headers: Vec<(String, String)>, - body: Vec, + pub(crate) body: Vec, } #[derive(Debug, Clone)] @@ -80,7 +80,13 @@ async fn raw(addr: SocketAddr, request: Vec) -> Reply { .unwrap() } -async fn request(addr: SocketAddr, method: &str, path: &str, headers: &str, body: &[u8]) -> Reply { +pub(crate) async fn request( + addr: SocketAddr, + method: &str, + path: &str, + headers: &str, + body: &[u8], +) -> Reply { let mut bytes = format!( "{method} {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\nContent-Length: \ {}\r\n{headers}\r\n", diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index df94bf82..8b88072d 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -10,4 +10,5 @@ mod diagnostics; pub mod http; pub mod metadata; pub mod protocol; +pub mod relay; pub mod session; diff --git a/crates/rustack-lambda-runtime/src/main.rs b/crates/rustack-lambda-runtime/src/main.rs new file mode 100644 index 00000000..3c30eccb --- /dev/null +++ b/crates/rustack-lambda-runtime/src/main.rs @@ -0,0 +1,39 @@ +//! Linux-only Lambda relay process. Stdout is reserved for binary control frames. + +#![forbid(unsafe_code)] + +#[cfg(not(target_os = "linux"))] +use std::future::{Ready, ready}; +use std::process::ExitCode; + +#[cfg(not(target_os = "linux"))] +use rustack_lambda_runtime::relay::RelayError; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> ExitCode { + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_target(false) + .with_writer(std::io::stderr) + .finish(); + if tracing::subscriber::set_global_default(subscriber).is_err() { + return ExitCode::FAILURE; + } + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + tracing::error!(%error, "Lambda runtime relay exited"); + ExitCode::FAILURE + } + } +} + +#[cfg(not(target_os = "linux"))] +fn run() -> Ready> { + ready(Err(RelayError::UnsupportedPlatform)) +} + +#[cfg(target_os = "linux")] +mod process; +#[cfg(target_os = "linux")] +use process::run; diff --git a/crates/rustack-lambda-runtime/src/process.rs b/crates/rustack-lambda-runtime/src/process.rs new file mode 100644 index 00000000..1016a305 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/process.rs @@ -0,0 +1,68 @@ +//! Linux process wiring with nonblocking owned pipe descriptors. + +use std::{ + env, + io::{stdin, stdout}, + os::fd::AsFd, + time::Duration, +}; + +use rustack_lambda_runtime::{ + http::start_at, + protocol::{Architecture, EnvironmentId}, + relay::{RelayConfig, RelayError, start}, +}; +use tokio::net::unix::pipe::{Receiver, Sender}; + +fn seconds(key: &str, default: u64, maximum: u64) -> Result { + let value = match env::var(key) { + Ok(value) => { + if value.len() > 5 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(RelayError::Configuration); + } + let parsed = value + .parse::() + .map_err(|_| RelayError::Configuration)?; + if parsed.to_string() != value { + return Err(RelayError::Configuration); + } + parsed + } + Err(env::VarError::NotPresent) => default, + Err(_) => return Err(RelayError::Configuration), + }; + if value == 0 || value > maximum { + return Err(RelayError::Configuration); + } + Ok(Duration::from_secs(value)) +} + +/// Run the private Linux relay process. +pub(crate) async fn run() -> Result<(), RelayError> { + let environment = + env::var("RUSTACK_LAMBDA_ENVIRONMENT_ID").map_err(|_| RelayError::Configuration)?; + if environment.len() != 36 { + return Err(RelayError::Configuration); + } + let environment: EnvironmentId = environment.parse().map_err(|_| RelayError::Configuration)?; + let architecture = match env::consts::ARCH { + "x86_64" => Architecture::X86_64, + "aarch64" => Architecture::Arm64, + _ => return Err(RelayError::UnsupportedPlatform), + }; + let idle = seconds("RUSTACK_LAMBDA_RELAY_IDLE_SECONDS", 600, 86_400)?; + let io = seconds("RUSTACK_LAMBDA_RELAY_IO_SECONDS", 5, 60)?; + let retirement = seconds("RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS", 10, 60)?; + let config = RelayConfig::new(environment, architecture, idle, io, retirement)?; + let input = Receiver::from_owned_fd(stdin().as_fd().try_clone_to_owned()?)?; + let output = Sender::from_owned_fd(stdout().as_fd().try_clone_to_owned()?)?; + let api = start_at( + "127.0.0.1:9001" + .parse() + .map_err(|_| RelayError::Configuration)?, + idle, + ) + .await?; + let mut relay = start(api, config, input, output)?; + relay.wait().await +} diff --git a/crates/rustack-lambda-runtime/src/relay.rs b/crates/rustack-lambda-runtime/src/relay.rs new file mode 100644 index 00000000..49f81bf8 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/relay.rs @@ -0,0 +1,525 @@ +//! Supervised control transport for an environment-local Lambda Runtime API. +//! +//! This owner invalidates the API session; the host environment supervisor +//! remains responsible for stopping/removing the separate function container. + +use std::{future::pending, io, sync::Arc, time::Duration}; + +use thiserror::Error; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite}, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, sleep_until, timeout_at}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; + +use crate::{ + http::{ApiError, RuntimeApiHandle}, + metadata::deadline_from_unix_ms, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + ProtocolError, read_frame, write_frame, + }, + session::{Invocation, InvocationTicket, RuntimeResult, SessionError}, +}; + +const MAILBOX_CAPACITY: usize = 2; +const CONTROL_SILENCE: Duration = Duration::from_hours(1); + +/// Relay errors contain classes and redacted protocol/IO sources. +#[derive(Debug, Clone, Error)] +#[non_exhaustive] +pub enum RelayError { + /// The process entry point needs Linux and supported CPU architecture. + #[error("Lambda relay requires Linux x86_64 or arm64")] + UnsupportedPlatform, + /// Invalid operator-owned worker configuration. + #[error("invalid Lambda relay configuration")] + Configuration, + /// Unexpected EOF before a normal Close exchange. + #[error("Lambda relay control EOF")] + Eof, + /// Protocol validation/transport failure. + #[error("Lambda relay protocol failed")] + Protocol(#[source] Arc), + /// IO while reading the frame's first byte failed. + #[error("Lambda relay control IO failed")] + Io(#[source] Arc), + /// A frame belongs to another environment. + #[error("Lambda relay environment mismatch")] + Scope, + /// Message direction, identity or session phase is invalid. + #[error("unexpected Lambda relay control message")] + UnexpectedMessage, + /// An IO/retirement lease expired. + #[error("Lambda relay deadline exceeded")] + Deadline, + /// Output could not be confirmed flushed. + #[error("Lambda relay output failed")] + Output, + /// An owned worker terminated unexpectedly. + #[error("Lambda relay worker failed")] + Worker, + /// Shared session failed. + #[error(transparent)] + Session(#[from] SessionError), + /// The API could not be cleaned up. + #[error(transparent)] + Api(#[from] ApiError), +} + +impl From for RelayError { + fn from(error: ProtocolError) -> Self { + Self::Protocol(Arc::new(error)) + } +} +impl From for RelayError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} + +/// Validated environment identity and forwarded worker leases. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct RelayConfig { + environment: EnvironmentId, + architecture: Architecture, + idle_lease: Duration, + io_budget: Duration, + retirement_budget: Duration, +} + +impl RelayConfig { + /// Validate worker leases without reading/mutating process environment. + /// + /// # Errors + /// Rejects zero leases, idle over24h or IO/retirement over60s. + /// + /// ``` + /// # use rustack_lambda_runtime::{protocol::{EnvironmentId, Architecture}, relay::RelayConfig}; + /// # use std::time::Duration; + /// let config = RelayConfig::new(EnvironmentId::default(), Architecture::Arm64, + /// Duration::from_secs(600), Duration::from_secs(5), Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_runtime::relay::RelayError>(()) + /// ``` + pub fn new( + environment: EnvironmentId, + architecture: Architecture, + idle_lease: Duration, + io_budget: Duration, + retirement_budget: Duration, + ) -> Result { + if idle_lease.is_zero() + || idle_lease > Duration::from_hours(24) + || io_budget.is_zero() + || io_budget > Duration::from_secs(60) + || retirement_budget.is_zero() + || retirement_budget > Duration::from_secs(60) + { + return Err(RelayError::Configuration); + } + Ok(Self { + environment, + architecture, + idle_lease, + io_budget, + retirement_budget, + }) + } +} + +/// Owner of the relay supervisor; Drop signals stop without abandoning cleanup. +#[derive(Debug)] +#[non_exhaustive] +pub struct RelayHandle { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +impl RelayHandle { + /// Await natural Close/EOF/failure without requesting shutdown. + /// + /// # Errors + /// Returns cached relay/protocol/supervision failures. + /// + /// ```no_run + /// # async fn wait(relay: &mut rustack_lambda_runtime::relay::RelayHandle) { + /// let outcome = relay.wait().await; + /// # } + /// ``` + pub async fn wait(&mut self) -> Result<(), RelayError> { + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| RelayError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settlement = Some(result.clone()); + result + } + + /// Signal priority stop and join the owned API/control workers. + /// + /// # Errors + /// Returns protocol, API cleanup or supervision failure. + /// + /// ```no_run + /// # async fn stop(relay: &mut rustack_lambda_runtime::relay::RelayHandle) { + /// let clean = relay.shutdown().await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), RelayError> { + self.stop.send_replace(true); + self.wait().await + } +} + +impl Drop for RelayHandle { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +/// Transfer the API and cancel-safe owned streams into one relay supervisor. +/// +/// IO must be asynchronously cancellable; the Linux binary supplies nonblocking +/// Unix pipes. A blocking stdin adapter cannot satisfy the shutdown contract. +/// +/// # Errors +/// Returns unavailable supervision when no Tokio runtime is entered. +/// +/// ```no_run +/// # async fn example(api: rustack_lambda_runtime::http::RuntimeApiHandle, config: rustack_lambda_runtime::relay::RelayConfig) -> Result<(), rustack_lambda_runtime::relay::RelayError> { +/// let (reader, _) = tokio::io::duplex(1024); +/// let (writer, _) = tokio::io::duplex(1024); +/// let mut relay = rustack_lambda_runtime::relay::start(api, config, reader, writer)?; +/// relay.shutdown().await?; +/// # Ok(()) } +/// ``` +pub fn start( + mut api: RuntimeApiHandle, + config: RelayConfig, + reader: R, + writer: W, +) -> Result +where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, +{ + let runtime = Handle::try_current().map_err(|_| RelayError::Worker)?; + let (stop, stopped) = watch::channel(false); + let (input, commands) = mpsc::channel(MAILBOX_CAPACITY); + let (output, frames) = mpsc::channel(MAILBOX_CAPACITY); + let initialization_error = api.take_initialization_error(); + let mut workers = JoinSet::new(); + workers.spawn(read_input(reader, input, config).with_current_subscriber()); + workers.spawn(write_output(writer, frames).with_current_subscriber()); + let actor = RelayActor::builder() + .api(api) + .config(config) + .input(commands) + .output(output) + .workers(workers) + .stop(stopped) + .initialization_error(initialization_error) + .build(); + let task = runtime.spawn(actor.run().with_current_subscriber()); + Ok(RelayHandle { + stop, + task: Some(task), + settlement: None, + }) +} + +#[derive(Debug)] +struct Outbound { + frame: Frame, + deadline: Instant, + written: oneshot::Sender, +} + +#[derive(Debug)] +enum WorkerExit { + InputClose, + OutputClosed, +} + +async fn read_input( + mut reader: R, + input: mpsc::Sender, + config: RelayConfig, +) -> Result { + loop { + let deadline = after(config.idle_lease.max(CONTROL_SILENCE))?; + let mut first = [0_u8]; + let count = timeout_at(deadline, reader.read(&mut first)) + .await + .map_err(|_| RelayError::Deadline)??; + if count == 0 { + return Err(RelayError::Eof); + } + let mut prefixed = first.as_slice().chain(&mut reader); + let frame = read_frame(&mut prefixed, after(config.io_budget)?) + .await? + .ok_or(RelayError::Eof)?; + let close = matches!(frame.message(), Message::Close); + timeout_at(after(config.io_budget)?, input.send(frame)) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Worker)?; + if close { + return Ok(WorkerExit::InputClose); + } + } +} + +async fn write_output( + mut writer: W, + mut output: mpsc::Receiver, +) -> Result { + while let Some(frame) = output.recv().await { + let result = write_frame(&mut writer, &frame.frame, frame.deadline).await; + let _ = frame.written.send(result.is_ok()); + result?; + } + Ok(WorkerExit::OutputClosed) +} + +fn after(duration: Duration) -> Result { + Instant::now() + .checked_add(duration) + .ok_or(RelayError::Configuration) +} + +#[derive(Debug, Default)] +enum Phase { + #[default] + Initializing, + Idle, + Busy(InvocationTicket), + Retiring(Instant), +} + +impl Phase { + async fn result(&mut self) -> Result { + match self { + Self::Busy(ticket) => ticket.result().await, + _ => pending().await, + } + } +} + +#[derive(Debug, TypedBuilder)] +struct RelayActor { + api: RuntimeApiHandle, + config: RelayConfig, + input: mpsc::Receiver, + output: mpsc::Sender, + workers: JoinSet>, + stop: watch::Receiver, + initialization_error: Option>, + #[builder(default)] + phase: Phase, + #[builder(default)] + bootstrap_announced: bool, + #[builder(default)] + last_attempt: Option, +} + +impl RelayActor { + async fn run(mut self) -> Result<(), RelayError> { + let outcome = self.control().await; + self.api.session().close(); + let api = self.api.shutdown().await.map_err(RelayError::from); + self.workers.abort_all(); + let mut workers = Ok(()); + while let Some(result) = self.workers.join_next().await { + let result = match result { + Ok(Ok(WorkerExit::InputClose)) => Ok(()), + Ok(Err(error)) => Err(error), + Err(error) if error.is_cancelled() => Ok(()), + Ok(Ok(WorkerExit::OutputClosed)) | Err(_) => Err(RelayError::Worker), + }; + if workers.is_ok() { + workers = result; + } + } + outcome.and(api).and(workers) + } + + async fn emit(&self, message: Message) -> Result<(), RelayError> { + let deadline = match self.phase { + Phase::Retiring(until) => after(self.config.io_budget)?.min(until), + _ => after(self.config.io_budget)?, + }; + let (written, receipt) = oneshot::channel(); + timeout_at( + deadline, + self.output.send(Outbound { + frame: Frame::new(self.config.environment, message), + deadline, + written, + }), + ) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Output)?; + match timeout_at(deadline, receipt).await { + Ok(Ok(true)) => Ok(()), + _ => Err(RelayError::Output), + } + } + + async fn control(&mut self) -> Result<(), RelayError> { + self.emit(Message::Hello { + architecture: self.config.architecture, + }) + .await?; + self.emit(Message::Ready).await?; + let session = self.api.session().clone(); + loop { + let retirement = match self.phase { + Phase::Retiring(deadline) => Some(deadline), + _ => None, + }; + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => return Ok(()), + () = retirement_deadline(retirement) => return Err(RelayError::Deadline), + worker = self.workers.join_next(), if !self.workers.is_empty() => { + match worker { Some(Ok(Ok(WorkerExit::InputClose))) => {}, Some(Ok(Err(error))) => return Err(error), _ => return Err(RelayError::Worker) } + } + frame = self.input.recv() => { + let frame = frame.ok_or(RelayError::Eof)?; + if frame.environment() != self.config.environment { return Err(RelayError::Scope); } + if self.command(frame.into_message()).await? { return Ok(()); } + } + error = initialization_error(&mut self.initialization_error), if self.initialization_error.is_some() && retirement.is_none() => { + self.initialization_error = None; + if let Some(payload) = error { + self.emit(Message::InitError { payload }).await?; + self.retire().await?; + } + } + ready = session.bootstrap_ready(after(CONTROL_SILENCE)?), if !self.bootstrap_announced && !matches!(self.phase, Phase::Retiring(_)) => { + ready?; + self.bootstrap_announced = true; + self.phase = Phase::Idle; + self.emit(Message::BootstrapReady).await?; + } + result = self.phase.result() => { self.completed(result?).await?; } + () = session.closed(), if !matches!(self.phase, Phase::Retiring(_)) => return Err(RelayError::Session(SessionError::Unavailable)), + } + } + } + + async fn command(&mut self, message: Message) -> Result { + match message { + Message::Invoke { + attempt, + deadline_unix_ms, + function_arn, + trace_id, + payload, + } if matches!(self.phase, Phase::Idle) && self.last_attempt != Some(attempt) => { + let deadline = + deadline_from_unix_ms(deadline_unix_ms).map_err(|_| RelayError::Deadline)?; + let invocation = + Invocation::new(attempt, deadline, function_arn, trace_id, payload)?; + let ticket = self + .api + .session() + .submit(invocation, after(self.config.io_budget)?.min(deadline)) + .await?; + self.last_attempt = Some(attempt); + self.phase = Phase::Busy(ticket); + } + Message::Cancel { attempt } if self.last_attempt == Some(attempt) => { + self.retire().await?; + self.emit(Message::Ack(Acknowledgement::Cancel(attempt))) + .await?; + } + Message::Close => { + self.api.session().close(); + self.api.shutdown().await?; + self.emit(Message::Ack(Acknowledgement::Close)).await?; + return Ok(true); + } + _ => return Err(RelayError::UnexpectedMessage), + } + Ok(false) + } + + async fn completed(&mut self, result: RuntimeResult) -> Result<(), RelayError> { + let attempt = self.last_attempt.ok_or(RelayError::UnexpectedMessage)?; + self.phase = Phase::Idle; + match result { + RuntimeResult::Success(payload) => { + self.emit(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload, + }) + .await?; + } + RuntimeResult::FunctionError(payload) => { + self.emit(Message::InvocationResult { + attempt, + outcome: Outcome::FunctionError, + payload, + }) + .await?; + self.retire().await?; + } + RuntimeResult::InitError(payload) => { + self.initialization_error = None; + self.emit(Message::InitError { payload }).await?; + self.retire().await?; + } + } + Ok(()) + } + + async fn retire(&mut self) -> Result<(), RelayError> { + if !matches!(self.phase, Phase::Retiring(_)) { + if let Phase::Busy(ticket) = &self.phase { + ticket.cancel(); + } + self.api.session().close(); + timeout_at(after(self.config.io_budget)?, self.api.session().closed()) + .await + .map_err(|_| RelayError::Deadline)?; + self.phase = Phase::Retiring(after(self.config.retirement_budget)?); + } + Ok(()) + } +} + +async fn initialization_error( + receiver: &mut Option>, +) -> Option { + match receiver { + Some(receiver) => receiver.await.ok(), + None => pending().await, + } +} + +async fn retirement_deadline(deadline: Option) { + match deadline { + Some(deadline) => sleep_until(deadline).await, + None => pending().await, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/relay/tests.rs b/crates/rustack-lambda-runtime/src/relay/tests.rs new file mode 100644 index 00000000..adc86628 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/relay/tests.rs @@ -0,0 +1,432 @@ +use std::{ + future::Future, + num::NonZeroU64, + task::{Context, Waker}, +}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncWriteExt, DuplexStream, duplex}, + net::TcpStream, +}; + +use super::*; +use crate::{ + http::tests::{Reply, request}, + metadata::{deadline_unix_ms, local_trace_id}, + protocol::{FunctionArn, Payload}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +struct Harness { + owner: RelayHandle, + environment: EnvironmentId, + input: DuplexStream, + output: DuplexStream, + api: std::net::SocketAddr, +} + +impl Harness { + async fn new() -> Self { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let environment = EnvironmentId::default(); + let config = RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(2), + ) + .unwrap(); + let (input, reader) = duplex(128); + let (writer, output) = duplex(128); + let owner = start(api, config, reader, writer).unwrap(); + let mut harness = Self { + owner, + environment, + input, + output, + api: address, + }; + assert!(matches!( + harness.next().await, + Message::Hello { + architecture: Architecture::Arm64 + } + )); + assert_eq!(harness.next().await, Message::Ready); + harness + } + + async fn next(&mut self) -> Message { + let frame = read_frame(&mut self.output, deadline()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.environment(), self.environment); + frame.into_message() + } + + async fn send(&mut self, message: Message) { + write_frame( + &mut self.input, + &Frame::new(self.environment, message), + deadline(), + ) + .await + .unwrap(); + } + + async fn close(mut self) { + self.send(Message::Close).await; + assert_eq!(self.next().await, Message::Ack(Acknowledgement::Close)); + timeout_at(deadline(), self.owner.wait()) + .await + .unwrap() + .unwrap(); + assert!( + read_frame(&mut self.output, deadline()) + .await + .unwrap() + .is_none() + ); + assert!(TcpStream::connect(self.api).await.is_err()); + } + + async fn bootstrap(&mut self) -> JoinHandle { + let api = self.api; + let poll = tokio::spawn(async move { + request(api, "GET", "/2018-06-01/runtime/invocation/next", "", b"").await + }); + assert_eq!(self.next().await, Message::BootstrapReady); + poll + } +} + +fn invoke() -> (AttemptId, Message) { + let attempt = AttemptId::default(); + ( + attempt, + Message::Invoke { + attempt, + deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(10)).unwrap(), + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" + .parse::() + .unwrap(), + trace_id: local_trace_id().unwrap(), + payload: Payload::try_from(Bytes::from_static(b"secret-event")).unwrap(), + }, + ) +} + +#[tokio::test] +async fn test_should_handshake_forward_http_results_and_reuse_one_session() { + let mut harness = Harness::new().await; + let first = harness.bootstrap().await; + for (index, poll) in [(0, Some(first)), (1, None)] { + let poll = poll.unwrap_or_else(|| { + let api = harness.api; + tokio::spawn(async move { + request(api, "GET", "/2018-06-01/runtime/invocation/next", "", b"").await + }) + }); + let (attempt, message) = invoke(); + harness.send(message).await; + let event = poll.await.unwrap(); + assert_eq!(event.status, 200); + assert_eq!(event.body, b"secret-event"); + let computed = format!("{{\"computed\":{},\"iteration\":{index}}}", 6 * 7); + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/response"), + "", + computed.as_bytes() + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InvocationResult { attempt: id, outcome: Outcome::Success, payload } if id == attempt && payload.as_ref() == computed.as_bytes()) + ); + } + harness.close().await; +} + +#[tokio::test] +async fn test_should_ack_cancel_while_busy_and_after_completion_without_replaying_a_result() { + for completed in [false, true] { + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, message) = invoke(); + harness.send(message).await; + assert_eq!(poll.await.unwrap().status, 200); + if completed { + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/response"), + "", + b"done" + ) + .await + .status, + 202 + ); + assert!(matches!( + harness.next().await, + Message::InvocationResult { + outcome: Outcome::Success, + .. + } + )); + } + harness.send(Message::Cancel { attempt }).await; + assert_eq!( + harness.next().await, + Message::Ack(Acknowledgement::Cancel(attempt)) + ); + harness.send(Message::Cancel { attempt }).await; + assert_eq!( + harness.next().await, + Message::Ack(Acknowledgement::Cancel(attempt)) + ); + harness.close().await; + } +} + +#[tokio::test] +async fn test_should_distinguish_function_and_initialization_errors_and_allow_close_ack() { + let mut harness = Harness::new().await; + assert_eq!( + request( + harness.api, + "POST", + "/2018-06-01/runtime/init/error", + "", + b"secret-init-error" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InitError { payload } if payload.as_ref() == b"secret-init-error") + ); + harness.close().await; + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, message) = invoke(); + harness.send(message).await; + poll.await.unwrap(); + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/error"), + "", + b"secret-function-error" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.next().await, Message::InvocationResult { outcome: Outcome::FunctionError, payload, .. } if payload.as_ref() == b"secret-function-error") + ); + harness.close().await; +} + +#[tokio::test] +async fn test_should_reject_foreign_scope_wrong_direction_and_invoke_before_bootstrap() { + for variant in 0..3 { + let mut harness = Harness::new().await; + match variant { + 0 => write_frame( + &mut harness.input, + &Frame::new(EnvironmentId::default(), Message::Close), + deadline(), + ) + .await + .unwrap(), + 1 => harness.send(Message::Ready).await, + _ => harness.send(invoke().1).await, + } + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + assert!(harness.owner.wait().await.is_err()); + assert!(TcpStream::connect(harness.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_retain_shutdown_after_dropping_its_waiter_and_interrupt_idle_input() { + let mut harness = Harness::new().await; + let mut waiting = Box::pin(harness.owner.shutdown()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(harness.owner.task.is_some()); + timeout_at(deadline(), harness.owner.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(TcpStream::connect(harness.api).await.is_err()); +} + +#[tokio::test] +async fn test_should_cleanup_api_after_eof_and_truncated_control_frames() { + for truncated in [false, true] { + let mut harness = Harness::new().await; + if truncated { + harness.input.write_all(&[0, 0]).await.unwrap(); + } + harness.input.shutdown().await.unwrap(); + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + assert!(TcpStream::connect(harness.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_reject_bad_clock_and_overlapping_invocations_without_success() { + for overlapping in [false, true] { + let mut harness = Harness::new().await; + let mut poll = Some(harness.bootstrap().await); + let (_, mut message) = invoke(); + if overlapping { + harness.send(message).await; + poll.take().unwrap().await.unwrap(); + message = invoke().1; + } else if let Message::Invoke { + deadline_unix_ms, .. + } = &mut message + { + *deadline_unix_ms = NonZeroU64::new(1).unwrap(); + } + harness.send(message).await; + if let Some(poll) = poll { + let _ = poll.await; + } + assert!( + timeout_at(deadline(), harness.owner.wait()) + .await + .unwrap() + .is_err() + ); + } +} + +#[tokio::test] +async fn test_should_close_a_probe_before_any_function_has_polled() { + Harness::new().await.close().await; +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_partial_frame_and_blocked_output_io_and_join_the_api() { + let mut harness = Harness::new().await; + harness.input.write_all(&[0]).await.unwrap(); + assert!( + matches!(harness.owner.wait().await, Err(RelayError::Protocol(error)) if matches!(&*error, ProtocolError::Deadline)) + ); + assert!(TcpStream::connect(harness.api).await.is_err()); + + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let config = RelayConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(1), + ) + .unwrap(); + let (_host_input, reader) = duplex(1); + let (writer, _host_output) = duplex(1); + let mut owner = start(api, config, reader, writer).unwrap(); + assert!(matches!(owner.wait().await, Err(RelayError::Output))); + assert!(TcpStream::connect(address).await.is_err()); +} + +#[tokio::test] +async fn test_should_preserve_a_completed_io_failure_when_priority_stop_wins() { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let config = RelayConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(1), + ) + .unwrap(); + let (_input, commands) = mpsc::channel(MAILBOX_CAPACITY); + let (output, mut frames) = mpsc::channel::(MAILBOX_CAPACITY); + let written = tokio::spawn(async move { + for _ in 0..2 { + frames.recv().await.unwrap().written.send(true).unwrap(); + } + }); + let mut workers = JoinSet::new(); + let failed = workers.spawn(async { Err(RelayError::Eof) }); + while !failed.is_finished() { + tokio::task::yield_now().await; + } + let (_stop, stopped) = watch::channel(true); + let actor = RelayActor::builder() + .api(api) + .config(config) + .input(commands) + .output(output) + .workers(workers) + .stop(stopped) + .initialization_error(None) + .build(); + assert!(matches!(actor.run().await, Err(RelayError::Eof))); + written.await.unwrap(); + assert!(TcpStream::connect(address).await.is_err()); +} + +#[test] +fn test_should_bound_relay_config() { + let environment = EnvironmentId::default(); + assert!( + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::ZERO, + Duration::from_secs(1), + Duration::from_secs(1) + ) + .is_err() + ); + assert!( + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(61), + Duration::from_secs(1) + ) + .is_err() + ); +} diff --git a/crates/rustack-lambda-runtime/src/session.rs b/crates/rustack-lambda-runtime/src/session.rs index 26173639..1d253e70 100644 --- a/crates/rustack-lambda-runtime/src/session.rs +++ b/crates/rustack-lambda-runtime/src/session.rs @@ -181,19 +181,56 @@ impl InvocationTicket { /// # } /// ``` pub async fn wait(mut self) -> Result { - check_deadline(self.deadline)?; - if *self.cancellation.borrow() { + self.result().await + } + + /// Await while retaining ticket ownership across a cancelled select branch. + /// + /// # Errors + /// Returns cancellation, expiry or session failure. Repeated calls after a + /// terminal result return Unavailable, without polling the consumed channel. + /// + /// ```no_run + /// # async fn inspect(ticket: &mut rustack_lambda_runtime::session::InvocationTicket) { + /// let result = ticket.result().await; + /// # } + /// ``` + pub async fn result(&mut self) -> Result { + if !self.attached { + return Err(SessionError::Unavailable); + } + if let Err(error) = check_deadline(self.deadline) { + self.cancel(); + self.attached = false; + return Err(error); + } + let cancelled = *self.cancellation.borrow(); + if cancelled { + self.attached = false; return Err(SessionError::Cancelled); } - let result = timeout_at(self.deadline, &mut self.response) - .await - .map_err(|_| SessionError::Deadline)? - .map_err(|_| SessionError::Unavailable)?; - check_deadline(self.deadline)?; + let result = timeout_at(self.deadline, &mut self.response).await; + // The response channel has either settled or the attempt has expired. + // Keep a completed receiver from ever being polled a second time. + self.attached = false; + let result = match result { + Ok(Ok(result)) => result, + Ok(Err(_)) => { + self.cancel(); + return Err(SessionError::Unavailable); + } + Err(_) => { + self.cancel(); + return Err(SessionError::Deadline); + } + }; + if let Err(error) = check_deadline(self.deadline) { + self.cancel(); + return Err(error); + } if *self.cancellation.borrow() { return Err(SessionError::Cancelled); } - self.attached = false; result } } diff --git a/crates/rustack-lambda-runtime/src/session/tests.rs b/crates/rustack-lambda-runtime/src/session/tests.rs index 169285a2..1a9551a4 100644 --- a/crates/rustack-lambda-runtime/src/session/tests.rs +++ b/crates/rustack-lambda-runtime/src/session/tests.rs @@ -540,6 +540,35 @@ fn transition_actor() -> SessionActor { .build() } +#[tokio::test] +async fn test_should_resume_a_borrowed_ticket_result_after_select_cancellation_and_reject_repolling() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut ticket = handle.submit(invocation(), deadline()).await.unwrap(); + let mut result = Box::pin(ticket.result()); + assert!(poll_once(&mut result).is_pending()); + drop(result); + assert!(!*ticket.cancellation.borrow()); + handle.next(deadline()).await.unwrap(); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + assert_eq!( + ticket.result().await.unwrap(), + RuntimeResult::Success(payload()) + ); + assert_eq!(ticket.result().await, Err(SessionError::Unavailable)); + session.shutdown().await.unwrap(); +} + #[tokio::test(start_paused = true)] async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_delivery_transition() { diff --git a/crates/rustack-lambda-runtime/tests/docker.rs b/crates/rustack-lambda-runtime/tests/docker.rs new file mode 100644 index 00000000..e61c6a42 --- /dev/null +++ b/crates/rustack-lambda-runtime/tests/docker.rs @@ -0,0 +1,618 @@ +//! Explicit, local-only qualification of the Linux relay and an official RIC. +//! +//! The CLI is a test transport. Production attach, deployment and pool behavior +//! are deliberately tested by the Docker executor's separate acceptance suite. + +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{env, os::unix::fs::FileTypeExt, process::Stdio, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use rustack_lambda_runtime::{ + metadata::{deadline_unix_ms, local_trace_id}, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + read_frame, write_frame, + }, +}; +use serde_json::{Value, json}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + process::{Child, ChildStdin, ChildStdout, Command}, + time::{Instant, sleep_until, timeout, timeout_at}, +}; + +const OPERATION: Duration = Duration::from_secs(45); +const MAX_OUTPUT: usize = 64 * 1024; + +#[derive(Debug)] +struct Docker { + host: String, + architecture: Architecture, + relay_image: String, + function_image: String, +} + +impl Docker { + async fn configured() -> Result { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET") + .context("explicit local Docker socket is required")?; + ensure!( + socket.starts_with('/') && socket.len() <= 4096, + "Docker socket must be a bounded absolute Unix path" + ); + ensure!( + tokio::fs::metadata(&socket).await?.file_type().is_socket(), + "configured Docker socket is unavailable" + ); + let architecture = env::var("RUSTACK_RUNTIME_ARCH").context("architecture is required")?; + let (expected, architecture) = match architecture.as_str() { + "arm64" => ("aarch64", Architecture::Arm64), + "amd64" => ("x86_64", Architecture::X86_64), + _ => bail!("architecture must be arm64 or amd64"), + }; + let relay_image = env::var("RUSTACK_RUNTIME_IMAGE").context("relay image is required")?; + ensure!( + !relay_image.is_empty() + && relay_image.len() <= 1024 + && relay_image.bytes().all(|byte| byte.is_ascii_graphic()), + "relay image reference must be bounded printable ASCII" + ); + let function_image = format!( + "rustack-p0-handler:{}", + if architecture == Architecture::Arm64 { + "arm64" + } else { + "amd64" + } + ); + let docker = Self { + host: format!("unix://{socket}"), + architecture, + relay_image, + function_image, + }; + let info = docker + .checked(&["info", "--format", "{{.OSType}} {{.Architecture}}"]) + .await?; + ensure!( + info.trim() == format!("linux {expected}"), + "Docker daemon platform does not match this qualification lane" + ); + for image in [&docker.relay_image, &docker.function_image] { + docker.checked(&["image", "inspect", image]).await?; + } + Ok(docker) + } + + fn command(&self) -> Command { + let mut command = Command::new("docker"); + command + .arg("--host") + .arg(&self.host) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + command + } + + async fn output(&self, arguments: &[&str]) -> Result<(bool, String)> { + self.output_until(arguments, Instant::now() + OPERATION) + .await + } + + async fn output_until(&self, arguments: &[&str], deadline: Instant) -> Result<(bool, String)> { + ensure!(Instant::now() < deadline, "Docker CLI deadline expired"); + let mut child = self.command().args(arguments).spawn()?; + let stdout = child.stdout.take().context("CLI stdout pipe")?; + let stderr = child.stderr.take().context("CLI stderr pipe")?; + let outcome = timeout_at(deadline, async { + let (status, stdout, stderr) = + tokio::join!(child.wait(), bounded(stdout), bounded(stderr)); + Ok::<_, anyhow::Error>((status?.success(), stdout?, stderr?)) + }) + .await; + if let Ok(result) = outcome { + let (success, stdout, _) = result?; + Ok((success, String::from_utf8(stdout)?)) + } else { + child.kill().await.context("kill expired Docker CLI")?; + child.wait().await.context("reap expired Docker CLI")?; + bail!("Docker CLI operation deadline exceeded") + } + } + + async fn checked(&self, arguments: &[&str]) -> Result { + self.checked_until(arguments, Instant::now() + OPERATION) + .await + } + + async fn checked_until(&self, arguments: &[&str], deadline: Instant) -> Result { + let (success, output) = self.output_until(arguments, deadline).await?; + ensure!(success, "Docker CLI operation failed"); + Ok(output) + } +} + +async fn bounded(reader: impl AsyncRead + Unpin) -> Result> { + let mut output = Vec::new(); + reader + .take(u64::try_from(MAX_OUTPUT)? + 1) + .read_to_end(&mut output) + .await?; + ensure!(output.len() <= MAX_OUTPUT, "Docker CLI output exceeded cap"); + Ok(output) +} + +#[derive(Debug)] +struct Names { + relay: String, + function: String, + label: String, +} + +#[derive(Debug)] +struct Control { + child: Child, + input: Option, + output: ChildStdout, +} + +#[derive(Debug)] +struct Environment { + docker: Docker, + identity: EnvironmentId, + names: Names, + control: Option, +} + +impl Environment { + async fn new() -> Result { + let identity = EnvironmentId::default(); + Ok(Self { + docker: Docker::configured().await?, + identity, + names: Names { + relay: format!("rustack-runtime-relay-{identity}"), + function: format!("rustack-runtime-function-{identity}"), + label: format!("rustack.runtime.test={identity}"), + }, + control: None, + }) + } + + async fn start(&mut self, idle_seconds: &str) -> Result<()> { + let mut command = self.docker.command(); + command + .args([ + "run", + "--pull=never", + "--rm", + "--interactive", + "--name", + &self.names.relay, + "--label", + &self.names.label, + "--network", + "none", + "--user", + "1000:1000", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + "64m", + "--cpus", + "0.1", + "--pids-limit", + "64", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=16777216", + "--env", + &format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={}", self.identity), + "--env", + &format!("RUSTACK_LAMBDA_RELAY_IDLE_SECONDS={idle_seconds}"), + &self.docker.relay_image, + ]) + .stdin(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = command.spawn()?; + let input = child.stdin.take().context("relay input pipe")?; + let output = child.stdout.take().context("relay output pipe")?; + self.control = Some(Control { + child, + input: Some(input), + output, + }); + ensure!( + self.next().await? + == Message::Hello { + architecture: self.docker.architecture + }, + "relay Hello/platform mismatch" + ); + ensure!(self.next().await? == Message::Ready, "missing relay Ready"); + let config = self + .docker + .checked(&["inspect", "--format", "{{json .Config}}", &self.names.relay]) + .await?; + let config: Value = serde_json::from_str(&config)?; + ensure!( + config["StdinOnce"] == true && config["Tty"] == false, + "qualification requires non-TTY stdin with EOF delivery" + ); + Ok(()) + } + + async fn next(&mut self) -> Result { + let control = self.control.as_mut().context("relay has not started")?; + let frame = read_frame(&mut control.output, Instant::now() + OPERATION) + .await? + .context("relay closed control output")?; + ensure!(frame.environment() == self.identity, "wrong environment"); + Ok(frame.into_message()) + } + + async fn send(&mut self, message: Message) -> Result<()> { + let input = self + .control + .as_mut() + .and_then(|control| control.input.as_mut()) + .context("relay input is closed")?; + write_frame( + input, + &Frame::new(self.identity, message), + Instant::now() + OPERATION, + ) + .await?; + Ok(()) + } + + async fn function(&mut self) -> Result<()> { + self.docker + .checked(&[ + "create", + "--pull=never", + "--name", + &self.names.function, + "--label", + &self.names.label, + "--network", + &format!("container:{}", self.names.relay), + "--user", + "1000:1000", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges=true", + "--memory", + "192m", + "--cpus", + "1", + "--pids-limit", + "256", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=536870912", + "--env", + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001", + "--env", + "AWS_LAMBDA_FUNCTION_NAME=spike", + "--env", + "AWS_LAMBDA_FUNCTION_VERSION=$LATEST", + "--env", + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192", + "--env", + "AWS_REGION=us-east-1", + "--env", + "AWS_DEFAULT_REGION=us-east-1", + &self.docker.function_image, + ]) + .await?; + self.docker + .checked(&["start", &self.names.function]) + .await?; + ensure!( + self.next().await? == Message::BootstrapReady, + "missing first RIC poll" + ); + Ok(()) + } + + async fn invoke(&mut self, event: Value) -> Result { + let attempt = AttemptId::default(); + self.send(Message::Invoke { + attempt, + deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(60))?, + function_arn: "arn:aws:lambda:us-east-1:000000000000:function:spike:prod".parse()?, + trace_id: local_trace_id()?, + payload: Payload::try_from(Bytes::from(serde_json::to_vec(&event)?))?, + }) + .await?; + Ok(attempt) + } + + async fn result(&mut self, attempt: AttemptId, expected: Outcome) -> Result { + match self.next().await? { + Message::InvocationResult { + attempt: actual, + outcome, + payload, + } if actual == attempt && outcome == expected => { + serde_json::from_slice(payload.as_ref()).context("RIC JSON result") + } + _ => bail!("unexpected relay result identity/outcome/variant"), + } + } + + async fn wait(&mut self, success: bool) -> Result<()> { + let control = self.control.as_mut().context("relay has not started")?; + let status = timeout(OPERATION, control.child.wait()) + .await + .context("relay failed to exit with input ownership retained")??; + ensure!(status.success() == success, "unexpected relay exit status"); + Ok(()) + } + + async fn close(&mut self) -> Result<()> { + self.send(Message::Close).await?; + ensure!( + self.next().await? == Message::Ack(Acknowledgement::Close), + "missing flushed Close acknowledgement" + ); + self.wait(true).await + } + + async fn remove_function(&self) -> Result<()> { + self.docker + .checked(&["rm", "--force", &self.names.function]) + .await?; + Ok(()) + } + + async fn cleanup(&mut self) -> Result<()> { + // Names are reserved before create/run, even if the response is lost. + let removal = self + .docker + .output(&["rm", "--force", &self.names.function, &self.names.relay]) + .await; + let reaped = if let Some(control) = &mut self.control { + control.input.take(); + let killed = control.child.start_kill().context("kill relay CLI"); + let waited = if let Ok(result) = timeout(OPERATION, control.child.wait()).await { + result.map(|_| ()).context("reap relay CLI") + } else { + // An expired waiter must not discard process ownership. + let result = control + .child + .wait() + .await + .context("retained relay CLI reap"); + result.and_then(|_| bail!("relay CLI cleanup deadline")) + }; + killed.and(waited) + } else { + Ok(()) + }; + let reconciled = async { + let remaining = self + .docker + .checked(&[ + "ps", + "--all", + "--quiet", + "--filter", + &format!("label={}", self.names.label), + ]) + .await?; + ensure!(remaining.trim().is_empty(), "owned containers remain"); + Ok(()) + } + .await; + removal.map(|_| ()).and(reaped).and(reconciled) + } + + async fn finish(&mut self, outcome: Result<()>) -> Result<()> { + let cleanup = self.cleanup().await; + match (outcome, cleanup) { + (Err(error), Err(cleanup)) => { + Err(error.context(format!("cleanup failed: {cleanup:#}"))) + } + (Err(error), Ok(())) => Err(error), + (Ok(()), cleanup) => cleanup, + } + } +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_exit_on_close_eof_truncation_and_io_timeout_with_owned_pipes() -> Result<()> { + for mode in ["close", "eof", "truncated", "partial", "idle"] { + let mut environment = Environment::new().await?; + let result = async { + // Allow time for inspect during setup; idle expiry must still exit + // while the parent retains the open, otherwise silent stdin pipe. + environment + .start(if mode == "idle" { "5" } else { "600" }) + .await?; + if mode == "close" { + environment.close().await?; + } else { + let control = environment.control.as_mut().context("relay control")?; + if mode == "truncated" || mode == "partial" { + control + .input + .as_mut() + .context("input")? + .write_all(&[0]) + .await?; + } + if mode == "eof" || mode == "truncated" { + control.input.take(); + } + environment.wait(false).await?; + } + Ok(()) + } + .await; + environment + .finish(result) + .await + .with_context(|| format!("pipe case {mode}"))?; + } + Ok(()) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_reconcile_failed_setup_and_reap_an_expired_cli_helper() -> Result<()> { + for mode in ["setup", "scope", "cli"] { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + match mode { + "setup" => { + environment.docker.function_image = + format!("rustack-runtime-missing:{}", environment.identity); + ensure!( + environment.function().await.is_err(), + "missing image setup unexpectedly succeeded" + ); + } + "scope" => { + let input = environment + .control + .as_mut() + .and_then(|control| control.input.as_mut()) + .context("relay input")?; + write_frame( + input, + &Frame::new(EnvironmentId::default(), Message::Close), + Instant::now() + OPERATION, + ) + .await?; + environment.wait(false).await?; + } + _ => { + // docker wait owns a different CLI child and cannot finish + // while the relay is running. Its own deadline must reap + // that child without cancelling or stopping the relay. + let expired = environment + .docker + .checked_until( + &["wait", &environment.names.relay], + Instant::now() + Duration::from_millis(200), + ) + .await; + ensure!(expired.is_err(), "CLI helper did not reach its deadline"); + environment.close().await?; + } + } + Ok(()) + } + .await; + environment + .finish(result) + .await + .with_context(|| format!("failure case {mode}"))?; + } + Ok(()) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_compute_reuse_and_report_official_ric_handler_errors() -> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + environment.function().await?; + for (value, computed, counter) in [(35, 42, 1), (0, 7, 2)] { + let attempt = environment.invoke(json!({ "value": value })).await?; + let result = environment.result(attempt, Outcome::Success).await?; + ensure!( + result["computed"] == computed && result["counter"] == counter, + "incorrect computation or warm counter" + ); + ensure!( + result["requestId"] == attempt.to_string(), + "incorrect RIC attempt metadata" + ); + ensure!( + result["uid"] == 1000 && result["rootReadOnly"] == true, + "function user/rootfs limits differ" + ); + ensure!( + result["memoryMax"] == "201326592" && result["pidsMax"] == "256", + "function cgroup limits differ" + ); + } + let attempt = environment.invoke(json!({ "fail": true })).await?; + let error = environment.result(attempt, Outcome::FunctionError).await?; + ensure!( + error["errorType"] == "ValueError", + "handler error was not preserved" + ); + environment.remove_function().await?; + environment.close().await + } + .await; + environment.finish(result).await +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_cancel_a_running_official_ric_and_remove_both_containers() -> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start("600").await?; + environment.function().await?; + let attempt = environment.invoke(json!({ "sleep": true })).await?; + let marker = format!("P0_BUSY_START={attempt}"); + let busy_deadline = Instant::now() + OPERATION; + loop { + // Forward the absolute deadline into the CLI owner. Cancelling an + // outer timeout here would discard its in-flight kill/reap path. + let logs = environment + .docker + .checked_until(&["logs", &environment.names.function], busy_deadline) + .await?; + if logs.contains(&marker) { + break; + } + ensure!( + Instant::now() < busy_deadline, + "handler never entered its busy section" + ); + sleep_until((Instant::now() + Duration::from_millis(50)).min(busy_deadline)).await; + } + for _ in 0..2 { + environment.send(Message::Cancel { attempt }).await?; + ensure!( + environment.next().await? == Message::Ack(Acknowledgement::Cancel(attempt)), + "missing matching cancel acknowledgement" + ); + } + let running = environment + .docker + .checked(&[ + "inspect", + "--format", + "{{.State.Running}}", + &environment.names.function, + ]) + .await?; + ensure!( + running.trim() == "true", + "Cancel Ack must not claim external process termination" + ); + environment.remove_function().await?; + environment.close().await + } + .await; + environment.finish(result).await +} diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 7a1b2d75..b289002e 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -223,6 +223,20 @@ GET responses include Request-Id, Invocation-Id (the same synthetic attempt UUID Native starts the API/session before spawning the bootstrap, then waits under one initialization deadline for first valid GET /next, initialization error, process exit or caller cancellation. Spawned children remain owned and are terminated/reaped on every failed/cancelled startup branch; API/session tasks are joined before clean settlement. The handler budget is captured after successful first-poll readiness when dispatching the event, additionally bounded by the caller's absolute deadline. Initialization time cannot consume/extend an unbounded fresh handler/Task budget. Runtime API errors still require retiring the process before propagating failure. Keep the executor module's runtime_api reexports, replacing the old pending-map/queue implementation with the shared adapter rather than maintaining a second session engine. +### 3.4 Relay control supervisor and process entry point + +RelayHandle owns one supervisor; Drop signals priority stop and leaves that supervisor owning API/control cleanup, while shutdown joins/caches its result without moving the join handle across awaits. The supervisor owns RuntimeApiHandle, dedicated control reader/writer tasks and bounded two-message input/output mailboxes. A queued output includes a write receipt; Hello, Ready, results and acknowledgements count as emitted only after the frame is flushed. Unexpected IO/task termination invalidates the environment without reconnect/replay. Every exit closes/joins the API and aborts/joins the in-process IO workers; external function termination remains the host environment owner's responsibility. + +Relay state is Initializing→Idle→Busy(ticket)→Idle; cancellation or initialization/function failure enters Retiring. Retiring accepts Close and idempotent Cancel for the last attempt until its bounded cleanup lease expires; acknowledging another matching Cancel must not extend that lease. Unexpected session closure or attempt expiry invalidates the control connection and exits under joined cleanup; the host must retire the external environment on that transport failure. Emit Hello then Ready after listener creation, and BootstrapReady once on the first valid GET /next. Invoke requires Idle and a matching environment UUID; reject wrong-direction messages, overlapping calls, wrong-scope frames and mismatched cancellation IDs. Validate/convert deadlineUnixMs before session admission. Keep attempt ticket ownership while selecting result versus control input; a cancel-safe borrowed ticket result method must not drop/replay a pending result or permit polling an already consumed oneshot. Completion and Close/Cancel races settle once; Cancel invalidates/closes the session before its Ack, never claims external user-code exit. Close joins the API and flushes Ack before IO teardown. A reader that queued Close may finish without making the supervisor discard that queued Close. + +Separate first-byte waiting from partial-frame IO: silence is bounded by the larger of the session idle lease and one hour (the validated wall-deadline ceiling), while receipt of the first byte starts the short control IO budget for the remainder of the frame. Session/retirement deadlines still stop an idle/busy relay sooner. Do not cancel/restart a partially read frame when adjusting leases. Output writes use the short absolute IO budget. Defaults are idle600s, IO5s and retirement10s; validate the same local ranges as their owner budgets. + +The `rustack-lambda-runtime` executable is Linux-only and serves127.0.0.1:9001. Required private child environment is RUSTACK_LAMBDA_ENVIRONMENT_ID; optional RUSTACK_LAMBDA_RELAY_IDLE_SECONDS / IO_SECONDS / RETIRE_SECONDS forward the owner's validated configuration without adding a new user configuration loader. Reject invalid/noncanonical values without echoing them. Advertise the binary's compiled x86_64/arm64 architecture. Stdout contains control frames only; diagnostic errors use bounded static tracing classes on stderr. + +Use safe OwnedFd clones of stdin/stdout with Tokio Unix pipe Receiver/Sender::from_owned_fd inside the entered IO runtime. The pinned Tokio1.53.1 constructors validate pipe/access mode and enable nonblocking IO. Do not use tokio::io::stdin or a blocking read task: cancelling its future does not stop the blocking read and can keep runtime shutdown waiting after a lease expires. Reject nonpipe descriptors/unsupported platforms explicitly. Unit tests use owned cancel-safe duplex streams; process tests exercise pipe EOF/Close and actual Linux builds precede Docker qualification. + +The optional local `test-lambda-runtime-docker` target builds the relay image and explicitly runs the resource-gated process suite against the selected local Unix socket. Missing Docker/images fail this explicit target; ordinary workspace tests do not pretend to provide Docker evidence. The test-only Docker CLI supplies non-TTY binary stdin/stdout; the production executor still requires the separately supervised Docker attach client. Preallocate unique container names and a run label before allocation so setup failures retain reconciliation ownership. Always remove both owned names, reap the CLI process and verify no run-labeled containers remain, even when the assertions fail. Exercise Hello/Ready/Close before a function exists, control EOF/truncation, and an unchanged official RIC image's computed result, warm counter, handler error and busy cancellation. These relay checks do not establish SDK/provider, deployment/pool or full M0 acceptance. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 7462281c..df60f88c 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -71,6 +71,8 @@ For Rust-relevant implementation work run cargo build, cargo test, cargo +nightl The shared runtime contract suite is available as `make test-lambda-runtime`; it runs the library tests/doctests and strict production boundary Clippy without lint ignores. Protocol property tests use [proptest1.12](https://docs.rs/proptest/1.12.0/proptest/) with512 cases per property, exercising bounded malformed bytes, mutations of valid envelopes and arbitrary binary payloads. Only its std feature is enabled; subprocess/fork machinery is unnecessary for these deterministic bounded parser checks. Runtime-session/relay tests extend this same target as their consumers land; a passing codec suite alone does not satisfy Docker acceptance. +`make test-lambda-runtime-docker` builds the selected architecture's relay and pinned official Python RIC fixture images, then explicitly executes every resource-gated test in the shared runtime's Docker suite. It requires WORKFLOWS_DOCKER_SOCKET and a matching WORKFLOWS_ARCH; missing prerequisites fail rather than skip. The CLI is only the test transport. Assert computed42/7, warm counter1/2, request metadata, cgroup/user/rootfs restrictions, distinct FunctionError, running-handler cancellation, pre-function Close, EOF/truncated/partial frames, idle expiry, failed setup and expired helper reaping. Every case reconciles its preallocated owned names/run label and joins its CLI process. These are relay qualification checks, not provider/deployment/pool acceptance for L01–L09. Native x86, Docker Desktop and rootless qualification still require their M4 lanes. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index aa95014f..08795cca 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -141,7 +141,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | -| 1.2 | In progress | Bounded protocol4da7df3 and shared actor2c50c2d are committed. Native now uses the shared loopback HTTP/session adapter and waits first /next before handler dispatch. HTTP validation, checked clock metadata, mandatory Hyper log filtering, joined connection/process retirement and W20–W22 review fixes are verified. Workspace:1,841 tests; runtime:71 tests/34 doctests and strict boundary Clippy; Lambda core:212 tests/26 doctests;11 real Native fixture regressions and native SDK target pass. Relay executable/socket client and Docker startup probes remain required; no Docker runtime acceptance yet. | +| 1.2 | In progress | Bounded protocol4da7df3, shared actor2c50c2d and Native migration95282c4 are committed. The Linux relay now owns bounded control IO, flush receipts and joined API/worker retirement. Workspace:1,853 tests; runtime:83 tests/39 doctests and strict boundary Clippy; Linux binary cross-target Clippy passes. Four explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including actual official RIC computation/warm reuse/error/cancellation, pipe/lease exits and failed setup/helper cleanup; both run labels reconcile to zero containers. W23–W26 are fixed/rechecked. Host control client remains required; no production Docker executor/provider acceptance yet. | | 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: build the relay executable/socket client and Docker startup probes on the shared HTTP/session implementation. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: build the host control client, then consume the qualified relay/HTTP/session contracts from the task1.3 Docker environment owner. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index ad3f7cdc..e68defde 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -91,6 +91,23 @@ The shared adapter replaces the old pending-map implementation while preserving Workspace build/fmt/Clippy/rustdoc and1,841 tests pass (392 preexisting ignored). Shared runtime:71 tests,34 doctests and strict production-boundary Clippy; Lambda core:212 tests and26 doctests. Three old core HTTP tests moved into the shared adapter's stronger wire suite. Cargo audit still finds the same16 B01 advisories; no new dependency version/package landed, and deny bans/licenses/sources pass. B03 remains open for old core boundaries. The relay executable/client, Docker executor and ASL engine remain required before milestone completion. +## P1 task1.2 Linux relay slice review + +The independent read-only review found one documentation mismatch and two test-harness ownership gaps. A self-review also found a completed-worker error masking race; its fix was independently rechecked. The final review found no remaining concrete in-slice issue. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W23 | P3; `specs/rustack-workflows-20-lambda-runtime-design.md` §3.4 | Retiring was described as Close-only while duplicate matching Cancel was deliberately idempotent. The contract now documents that exception without refreshing the lease, and distinguishes fatal session/attempt expiry from the bounded retirement handshake | Repeated busy/post-completion Cancel tests plus explicit real RIC cancellation qualification | +| W24 | P2; self-review, `crates/rustack-lambda-runtime/src/relay.rs`, RelayActor::run | Priority stop could win while a completed IO error remained in JoinSet; teardown ignored that error. Drain now preserves completed IO failure/unexpected writer exit and ignores only joined abort cancellation or the expected queued-Close reader exit | Deterministic pre-completed EOF worker with priority stop requires EOF failure and a closed API listener | +| W25 | P2; `crates/rustack-lambda-runtime/tests/docker.rs`, busy-marker loop | An outer timeout could cancel a CLI helper before explicit kill/reap. Forward one absolute deadline into the helper owner instead | Real docker wait helper expires and is reaped while the relay remains usable for Close/Ack | +| W26 | P2; same file, Environment::cleanup | A CLI reaping timeout returned before independent label reconciliation. Record cleanup errors, retain process ownership through reaping and still perform the label check before combining results | Every successful/failed setup, scope, pipe and handler case verifies no run-labeled container remains; final independent review confirms error aggregation order | + +The relay binary uses safe nonblocking owned Linux pipe descriptors, task-bound tracing, bounded two-frame mailboxes and confirmed flush receipts. It separates relay Ready from first-RIC-poll BootstrapReady, retains borrowed attempt tickets across select cancellation, rejects invalid scope/order/direction, distinguishes runtime error payloads, and joins the shared API and control workers on every exit. Close and cancellation do not claim external function termination. Pinned provided.al2023 relay images are built by Make targets for both supported Linux architectures. + +Workspace build/fmt/Clippy/rustdoc and1,853 tests pass. The ordinary suite has396 ignored tests:392 preexisting resource tests plus four explicitly gated Docker relay tests. Runtime:83 unit tests,39 doctests and strict production-boundary Clippy pass; the actual Linux entry point also passes cross-target Clippy. `make test-lambda-runtime-docker` explicitly ran all four tests with zero ignored on each arm64 VZ and amd64/QEMU daemon (eight executed test cases overall), proving official Python RIC computed42/7, warm counter1/2, request metadata, nonroot/read-only/cgroup limits, handler error, busy cancellation, Close/EOF/truncated/partial/idle exits, failed allocation and CLI helper timeout cleanup. Both daemons have zero remaining test-labeled containers. Native x86, Desktop and rootless qualification remain M4. + +No dependency version/package was added; existing anyhow/serde_json are test dependencies and tracing-subscriber is also used by the relay entry point. Audit/deny report the same16 B01 advisories; bans/licenses/sources pass. B03 remains open. The production host control client, Docker deployment/pool/provider integration and ASL engine remain required; this is not full P1/M0 acceptance. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index ce2ef8f3..d527a94e 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -124,6 +124,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [shared HTTP adapter and Native startup](./rustack-workflows-20-lambda-runtime-design.md#33-shared-http-adapter-and-native-startup), [review W20–W22](./rustack-workflows-93-review.md#p1-task12-shared-http--native-migration-review). - Date:2026-10-25. The separate mandatory Bollard filter W01 remains required when Docker is implemented. +## D17 — Retain relay IO ownership through bounded control settlement + +- Alternatives: read stdin through Tokio's blocking adapter; read control frames inside a cancel-and-restart select branch; treat channel enqueue as successful emission; discard completed worker errors when priority stop wins. +- Decision: Linux nonblocking owned pipe descriptors, dedicated supervised reader/writer tasks, bounded two-message mailboxes and explicit flush receipts. Retain attempt tickets across cancelled result polls, join the API on every exit, and preserve completed IO failures while draining aborted workers. Matching duplicate Cancel is idempotent and never refreshes retirement. +- Why: a blocking stdin read can hold process shutdown open; restarting a partial read corrupts framing; an enqueued Ack does not prove it reached the host. Cancellation only invalidates the session, so the separate host owner must still terminate/remove the function. Real relay/RIC process tests complement the in-memory contract tests without claiming deployment/pool or SDK acceptance. +- Pinned by: [relay control supervisor and process entry point](./rustack-workflows-20-lambda-runtime-design.md#34-relay-control-supervisor-and-process-entry-point). +- Date:2026-10-25. + ## Glossary | Term | Meaning in this specification | From 6d3123812371bfc3d6a8163876870331e6af2616 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 18:48:14 -0700 Subject: [PATCH 009/106] phase 1: own ordered host Lambda control sessions and shared stream workers Add ControlClient with causal handshake/result/cancel/close validation, cancel-safe receives and fail-closed interrupted sends. Extract retained bounded stream supervision for both client and relay, preserving terminal frames, flush receipts and completed IO failures through joined retirement. Independent review findings W27-W28 and self-found W29 are fixed and rechecked. Workspace build/test/fmt/Clippy/rustdoc pass with 1867 tests; runtime 97 tests/47 doctests and strict boundary checks pass, as does the Linux binary cross-target check. Five explicit Docker tests pass on each arm64 VZ and amd64/QEMU daemon, including the production client driving real official RIC warm computation, handler failure and busy cancellation. This completes the shared runtime contracts for the P1 Docker consumer; startup probes, deployment/environment/pool integration and B01/B03 gates remain required. No production Docker executor or full P1/M0 claim. --- crates/rustack-lambda-runtime/src/client.rs | 466 ++++++++++++++ .../src/client/tests.rs | 592 ++++++++++++++++++ crates/rustack-lambda-runtime/src/lib.rs | 2 + crates/rustack-lambda-runtime/src/relay.rs | 137 +--- .../rustack-lambda-runtime/src/relay/tests.rs | 33 +- .../rustack-lambda-runtime/src/transport.rs | 267 ++++++++ crates/rustack-lambda-runtime/tests/docker.rs | 194 +++++- ...tack-workflows-20-lambda-runtime-design.md | 32 + .../rustack-workflows-70-verification-plan.md | 2 +- specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 16 + specs/rustack-workflows-99-decisions.md | 8 + 12 files changed, 1594 insertions(+), 159 deletions(-) create mode 100644 crates/rustack-lambda-runtime/src/client.rs create mode 100644 crates/rustack-lambda-runtime/src/client/tests.rs create mode 100644 crates/rustack-lambda-runtime/src/transport.rs diff --git a/crates/rustack-lambda-runtime/src/client.rs b/crates/rustack-lambda-runtime/src/client.rs new file mode 100644 index 00000000..6e2327b2 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/client.rs @@ -0,0 +1,466 @@ +//! Ordered host-side relay protocol; external resources remain host-owned. +//! +//! Partial framing lives in retained stream workers. Selecting and cancelling +//! `next()` never cancels those workers or replays a business invocation. + +use std::time::Duration; + +use tokio::{ + io::{AsyncRead, AsyncWrite}, + time::{Instant, timeout_at}, +}; + +use crate::{ + metadata::deadline_unix_ms, + protocol::{ + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + }, + relay::RelayError, + session::{Invocation, RuntimeResult}, + transport::{InputEnd, Settings, Transport, after}, +}; + +/// Validated host identity, expected architecture and control IO budget. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ClientConfig { + environment: EnvironmentId, + architecture: Architecture, + io_budget: Duration, +} + +impl ClientConfig { + /// Validate a nonzero control budget of at most sixty seconds. + /// + /// # Errors + /// Returns Configuration when the budget is outside its allowed range. + /// + /// ``` + /// # use rustack_lambda_runtime::{client::ClientConfig, protocol::{Architecture, EnvironmentId}}; + /// let config = ClientConfig::new(EnvironmentId::default(), Architecture::Arm64, + /// std::time::Duration::from_secs(5))?; + /// # Ok::<(), rustack_lambda_runtime::relay::RelayError>(()) + /// ``` + pub fn new( + environment: EnvironmentId, + architecture: Architecture, + io_budget: Duration, + ) -> Result { + if io_budget.is_zero() || io_budget > Duration::from_secs(60) { + return Err(RelayError::Configuration); + } + Ok(Self { + environment, + architecture, + io_budget, + }) + } +} + +/// One validated relay event, with payload diagnostics redacted. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ClientEvent { + /// Hello/architecture and Ready were received in order. + RelayReady, + /// The function issued its first valid Runtime API poll. + BootstrapReady, + /// One matching attempt completed; `FunctionError` retires this session. + InvocationResult { + /// Correlated invocation identity. + attempt: AttemptId, + /// Raw success or function error, without an implicit retry. + result: RuntimeResult, + }, + /// Initialization failed; this environment cannot be reused. + InitError(Payload), + /// The relay invalidated the attempt; external code may still be running. + Cancelled(AttemptId), + /// Close was acknowledged after the peer joined its Runtime API. + Closed, +} + +#[derive(Debug)] +enum Phase { + AwaitHello, + AwaitReady, + Initializing, + Idle, + Busy(AttemptId), + Cancelling { + attempt: AttemptId, + outcome_seen: bool, + }, + Retiring, + Closed, + Failed(RelayError), +} + +#[derive(Debug)] +struct State { + phase: Phase, + closing: bool, + writing: bool, + delivered: bool, +} + +/// One host control session with retained IO and fail-closed send semantics. +/// +/// Drop signals its stream supervisor. Normal environment shutdown must call +/// shutdown and separately terminate/remove every external resource. +#[derive(Debug)] +#[non_exhaustive] +pub struct ControlClient { + config: ClientConfig, + transport: Transport, + state: State, + last_attempt: Option, + settlement: Option>, +} + +impl ControlClient { + /// Own asynchronously cancellable streams and start their retained workers. + /// + /// # Errors + /// Returns Worker if no Tokio runtime is entered. + /// + /// ```no_run + /// # async fn example(config: rustack_lambda_runtime::client::ClientConfig) -> Result<(), rustack_lambda_runtime::relay::RelayError> { + /// let (reader, _) = tokio::io::duplex(1024); + /// let (writer, _) = tokio::io::duplex(1024); + /// let mut client = rustack_lambda_runtime::client::ControlClient::start(config, reader, writer)?; + /// let first = client.next(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # Ok(()) } + /// ``` + pub fn start(config: ClientConfig, reader: R, writer: W) -> Result + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + Ok(Self { + config, + transport: Transport::start( + reader, + writer, + Settings { + silence: Duration::from_hours(1), + io_budget: config.io_budget, + input_end: InputEnd::CloseAcknowledgement, + }, + )?, + state: State { + phase: Phase::AwaitHello, + closing: false, + writing: false, + delivered: false, + }, + last_attempt: None, + settlement: None, + }) + } + + /// Whether a healthy, bootstrapped session can accept a new attempt now. + /// + /// ```no_run + /// # fn inspect(client: &rustack_lambda_runtime::client::ControlClient) { + /// let may_invoke = client.is_idle(); + /// # } + /// ``` + #[must_use] + pub fn is_idle(&self) -> bool { + matches!(self.state.phase, Phase::Idle) + && !self.state.closing + && !self.state.writing + && !self.transport.finished() + } + + /// Receive one validated event; cancelling this future preserves framing. + /// + /// # Errors + /// Returns a cached protocol/IO failure or invalidates the session at the + /// absolute deadline. It rejects calls after an interrupted send. + /// + /// ```no_run + /// # async fn receive(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let event = client.next(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn next(&mut self, deadline: Instant) -> Result { + self.check()?; + loop { + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + let received = timeout_at(deadline, self.transport.recv()).await; + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + let frame = match received { + Ok(Ok(frame)) => frame, + Ok(Err(error)) => return Err(self.fail(error)), + Err(_) => return Err(self.fail(RelayError::Deadline)), + }; + if frame.environment() != self.config.environment { + return Err(self.fail(RelayError::Scope)); + } + match self.accept(frame.into_message()) { + Ok(Some(event)) => return Ok(event), + Ok(None) => {} + Err(error) => return Err(self.fail(error)), + } + } + } + + /// Flush one attempt after first-poll readiness; never reconnect or replay. + /// + /// # Errors + /// Returns invalid order/identity, expired metadata or transport failure. + /// Dropping this send future poisons later business operations. + /// + /// ```no_run + /// # async fn invoke(client: &mut rustack_lambda_runtime::client::ControlClient, invocation: rustack_lambda_runtime::session::Invocation) { + /// let sent = client.invoke(invocation, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn invoke( + &mut self, + invocation: Invocation, + deadline: Instant, + ) -> Result<(), RelayError> { + self.check()?; + let (attempt, call_deadline, function_arn, trace_id, payload) = invocation.parts(); + if self.state.closing + || !matches!(self.state.phase, Phase::Idle) + || self.last_attempt == Some(attempt) + { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + let deadline_unix_ms = match deadline_unix_ms(call_deadline) { + Ok(deadline) + if call_deadline.saturating_duration_since(Instant::now()) + <= Duration::from_hours(1) => + { + deadline + } + _ => return Err(self.fail(RelayError::Deadline)), + }; + self.last_attempt = Some(attempt); + self.state.phase = Phase::Busy(attempt); + self.send( + Message::Invoke { + attempt, + deadline_unix_ms, + function_arn: function_arn.clone(), + trace_id: trace_id.clone(), + payload: payload.clone(), + }, + deadline.min(call_deadline), + ) + .await + } + + /// Invalidate the active or just-completed last attempt; await its event next. + /// + /// # Errors + /// Rejects another identity, overlapping cancellation or a closing session. + /// IO failure invalidates the connection without asserting process exit. + /// + /// ```no_run + /// # async fn cancel(client: &mut rustack_lambda_runtime::client::ControlClient, id: rustack_lambda_runtime::protocol::AttemptId) { + /// let sent = client.cancel(id, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn cancel( + &mut self, + attempt: AttemptId, + deadline: Instant, + ) -> Result<(), RelayError> { + self.check()?; + if self.state.closing || self.last_attempt != Some(attempt) { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + let outcome_seen = match self.state.phase { + Phase::Busy(active) if active == attempt => false, + Phase::Idle | Phase::Retiring => true, + _ => return Err(self.fail(RelayError::UnexpectedMessage)), + }; + self.state.phase = Phase::Cancelling { + attempt, + outcome_seen, + }; + self.send(Message::Cancel { attempt }, deadline).await + } + + /// Flush Close while retaining validation of preceding peer events. + /// + /// # Errors + /// Requires ordered Hello/Ready and a healthy, uninterrupted transport. + /// + /// ```no_run + /// # async fn close(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let sent = client.begin_close(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn begin_close(&mut self, deadline: Instant) -> Result<(), RelayError> { + self.check()?; + if matches!(self.state.phase, Phase::AwaitHello | Phase::AwaitReady) { + return Err(self.fail(RelayError::UnexpectedMessage)); + } + if !self.state.closing { + self.state.closing = true; + self.send(Message::Close, deadline).await?; + } + Ok(()) + } + + /// Complete Close/Ack and join IO, retaining ownership across dropped waiters. + /// + /// # Errors + /// Caches protocol, deadline, uncertain-write and worker failures. A failed + /// handshake still stops/joins stream workers; external cleanup is separate. + /// + /// ```no_run + /// # async fn shutdown(client: &mut rustack_lambda_runtime::client::ControlClient) { + /// let clean = client.shutdown(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await.is_ok(); + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), RelayError> { + if let Some(result) = &self.settlement { + return result.clone(); + } + let outcome = async { + if matches!(self.state.phase, Phase::Closed) { + return Ok(()); + } + self.check()?; + while matches!(self.state.phase, Phase::AwaitHello | Phase::AwaitReady) { + self.next(deadline).await?; + } + self.begin_close(deadline).await?; + while !matches!(self.state.phase, Phase::Closed) { + self.next(deadline).await?; + } + Ok(()) + } + .await; + let io = self.transport.shutdown().await; + let outcome = outcome.and(io); + self.settlement = Some(outcome.clone()); + outcome + } + + fn check(&mut self) -> Result<(), RelayError> { + if let Phase::Failed(error) = &self.state.phase { + return Err(error.clone()); + } + if self.state.writing { + return Err(self.fail(RelayError::Output)); + } + if matches!(self.state.phase, Phase::Closed) { + return Err(RelayError::UnexpectedMessage); + } + Ok(()) + } + + fn fail(&mut self, error: RelayError) -> RelayError { + self.state.phase = Phase::Failed(error.clone()); + self.transport.stop(); + error + } + + async fn send(&mut self, message: Message, deadline: Instant) -> Result<(), RelayError> { + let deadline = deadline.min(after(self.config.io_budget)?); + if Instant::now() >= deadline { + return Err(self.fail(RelayError::Deadline)); + } + self.state.writing = true; + let result = self + .transport + .send(Frame::new(self.config.environment, message), deadline) + .await; + match result { + Ok(()) => { + self.state.writing = false; + Ok(()) + } + Err(error) => Err(self.fail(error)), + } + } + + fn accept(&mut self, message: Message) -> Result, RelayError> { + match message { + Message::Hello { architecture } + if matches!(self.state.phase, Phase::AwaitHello) + && architecture == self.config.architecture => + { + self.state.phase = Phase::AwaitReady; + Ok(None) + } + Message::Ready if matches!(self.state.phase, Phase::AwaitReady) => { + self.state.phase = Phase::Initializing; + Ok(Some(ClientEvent::RelayReady)) + } + Message::BootstrapReady if matches!(self.state.phase, Phase::Initializing) => { + self.state.phase = Phase::Idle; + Ok(Some(ClientEvent::BootstrapReady)) + } + Message::InvocationResult { + attempt, + outcome, + payload, + } => match &mut self.state.phase { + Phase::Busy(active) if *active == attempt => { + self.state.delivered = true; + let result = match outcome { + Outcome::Success => { + self.state.phase = Phase::Idle; + RuntimeResult::Success(payload) + } + Outcome::FunctionError => { + self.state.phase = Phase::Retiring; + RuntimeResult::FunctionError(payload) + } + }; + Ok(Some(ClientEvent::InvocationResult { attempt, result })) + } + Phase::Cancelling { + attempt: active, + outcome_seen, + } if *active == attempt && !*outcome_seen => { + self.state.delivered = true; + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + }, + Message::InitError { payload } if !self.state.delivered => { + match &mut self.state.phase { + Phase::Initializing | Phase::Idle | Phase::Busy(_) => { + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::InitError(payload))) + } + Phase::Cancelling { outcome_seen, .. } if !*outcome_seen => { + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + } + } + Message::Ack(Acknowledgement::Cancel(attempt)) if matches!(self.state.phase, Phase::Cancelling { attempt: active, .. } if active == attempt) => + { + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::Cancelled(attempt))) + } + Message::Ack(Acknowledgement::Close) + if self.state.closing && !matches!(self.state.phase, Phase::Cancelling { .. }) => + { + self.state.phase = Phase::Closed; + Ok(Some(ClientEvent::Closed)) + } + _ => Err(RelayError::UnexpectedMessage), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/client/tests.rs b/crates/rustack-lambda-runtime/src/client/tests.rs new file mode 100644 index 00000000..37b89f90 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/client/tests.rs @@ -0,0 +1,592 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncWriteExt, DuplexStream, duplex}, + net::TcpStream, + task::JoinHandle, +}; + +use super::*; +use crate::{ + http::tests::{Reply, request}, + metadata::local_trace_id, + protocol::{encode, read_frame, write_frame}, + relay::{RelayConfig, RelayHandle, start}, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +fn invocation() -> Invocation { + Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(30), + "arn:aws:lambda:us-east-1:000000000000:function:test:prod" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"event-secret")).unwrap(), + ) + .unwrap() +} + +struct Peer { + client: ControlClient, + environment: EnvironmentId, + input: DuplexStream, + output: DuplexStream, +} + +impl Peer { + fn new(output_size: usize) -> Self { + Self::with_limits(16, output_size, Duration::from_secs(1)) + } + + fn with_limits(input_size: usize, output_size: usize, budget: Duration) -> Self { + let environment = EnvironmentId::default(); + let config = ClientConfig::new(environment, Architecture::Arm64, budget).unwrap(); + let (reader, output) = duplex(input_size); + let (input, writer) = duplex(output_size); + let client = ControlClient::start(config, reader, writer).unwrap(); + Self { + client, + environment, + input, + output, + } + } + + async fn send(&mut self, message: Message) { + write_frame( + &mut self.output, + &Frame::new(self.environment, message), + deadline(), + ) + .await + .unwrap(); + } + + async fn read(&mut self) -> Message { + let frame = read_frame(&mut self.input, deadline()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.environment(), self.environment); + frame.into_message() + } + + async fn ready(&mut self, bootstrap: bool) { + self.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + self.send(Message::Ready).await; + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + if bootstrap { + self.send(Message::BootstrapReady).await; + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + } + } + + async fn close(mut self) { + self.client.begin_close(deadline()).await.unwrap(); + assert_eq!(self.read().await, Message::Close); + self.send(Message::Ack(Acknowledgement::Close)).await; + self.client.shutdown(deadline()).await.unwrap(); + self.client.shutdown(deadline()).await.unwrap(); + assert!(!self.client.is_idle()); + } + + async fn failed(mut self) { + assert!(self.client.shutdown(deadline()).await.is_err()); + assert!(self.client.shutdown(deadline()).await.is_err()); + assert!(!self.client.is_idle()); + } +} + +struct Harness { + client: ControlClient, + relay: RelayHandle, + api: std::net::SocketAddr, +} + +impl Harness { + async fn new() -> Self { + let api = crate::http::start().await.unwrap(); + let address = api.addr(); + let environment = EnvironmentId::default(); + let relay_config = RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(2), + ) + .unwrap(); + let config = + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(); + let (writer, relay_reader) = duplex(24); + let (relay_writer, reader) = duplex(24); + let relay = start(api, relay_config, relay_reader, relay_writer).unwrap(); + let mut client = ControlClient::start(config, reader, writer).unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + assert!(!client.is_idle()); + Self { + client, + relay, + api: address, + } + } + + async fn bootstrap(&mut self) -> JoinHandle { + let address = self.api; + let poll = tokio::spawn(async move { + request( + address, + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await + }); + assert_eq!( + self.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + assert!(self.client.is_idle()); + poll + } + + async fn close(mut self) { + self.client.shutdown(deadline()).await.unwrap(); + self.relay.wait().await.unwrap(); + assert!(TcpStream::connect(self.api).await.is_err()); + } +} + +#[tokio::test] +async fn test_should_pair_client_with_real_relay_http_and_reuse_matching_attempts() { + let mut harness = Harness::new().await; + let mut poll = Some(harness.bootstrap().await); + for index in 0..3 { + let poll = poll.take().unwrap_or_else(|| { + let address = harness.api; + tokio::spawn(async move { + request( + address, + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await + }) + }); + let call = invocation(); + let attempt = call.parts().0; + harness.client.invoke(call, deadline()).await.unwrap(); + let event = poll.await.unwrap(); + assert_eq!(event.status, 200); + assert_eq!(event.body, b"event-secret"); + let suffix = if index == 2 { "error" } else { "response" }; + assert_eq!( + request( + harness.api, + "POST", + &format!("/2018-06-01/runtime/invocation/{attempt}/{suffix}"), + "", + b"computed-secret" + ) + .await + .status, + 202 + ); + let expected = Payload::try_from(Bytes::from_static(b"computed-secret")).unwrap(); + let result = if index == 2 { + RuntimeResult::FunctionError(expected) + } else { + RuntimeResult::Success(expected) + }; + assert_eq!( + harness.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { attempt, result } + ); + assert_eq!(harness.client.is_idle(), index != 2); + } + harness.close().await; +} + +#[tokio::test] +async fn test_should_forward_init_failure_before_bootstrap_and_close_both_owners() { + let mut harness = Harness::new().await; + assert_eq!( + request( + harness.api, + "POST", + "/2018-06-01/runtime/init/error", + "", + b"init-secret" + ) + .await + .status, + 202 + ); + assert!( + matches!(harness.client.next(deadline()).await.unwrap(), ClientEvent::InitError(payload) if payload.as_ref() == b"init-secret") + ); + assert!(!harness.client.is_idle()); + harness.close().await; +} + +#[tokio::test] +async fn test_should_reject_foreign_scope_architecture_and_out_of_order_handshakes() { + for mode in 0..6 { + let mut peer = Peer::new(8192); + match mode { + 0 => write_frame( + &mut peer.output, + &Frame::new(EnvironmentId::default(), Message::Ready), + deadline(), + ) + .await + .unwrap(), + 1 => { + peer.send(Message::Hello { + architecture: Architecture::X86_64, + }) + .await; + } + 2 => peer.send(Message::Ready).await, + 3 => peer.send(Message::BootstrapReady).await, + 4 => peer.send(Message::Close).await, + _ => { + peer.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + peer.send(Message::Hello { + architecture: Architecture::Arm64, + }) + .await; + } + } + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_reject_foreign_and_duplicate_results_and_cancel_ack_ids() { + for mode in 0..3 { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + assert!(matches!(peer.read().await, Message::Invoke { attempt: id, .. } if id == attempt)); + let outcome = Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::from_static(b"result")).unwrap(), + }; + match mode { + 0 => { + peer.send(Message::InvocationResult { + attempt: AttemptId::default(), + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + } + 1 => { + peer.send(outcome.clone()).await; + assert!(matches!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { .. } + )); + peer.send(outcome).await; + } + _ => { + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(AttemptId::default()))) + .await; + } + } + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_discard_only_one_late_result_during_cancellation_and_require_matching_ack() { + for completed in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::from_static(b"late")).unwrap(), + }) + .await; + if completed { + assert!(matches!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationResult { .. } + )); + } + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + assert!(!peer.client.is_idle()); + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + peer.close().await; + } +} + +#[tokio::test] +async fn test_should_validate_pending_events_before_close_and_reject_early_close_ack() { + for mode in 0..4 { + let mut peer = Peer::new(8192); + peer.ready(mode != 0).await; + let call = invocation(); + let attempt = call.parts().0; + if mode != 0 { + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + } + if mode >= 2 { + peer.client.cancel(attempt, deadline()).await.unwrap(); + peer.read().await; + } + peer.client.begin_close(deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Close); + let preceding = match mode { + 0 => Message::BootstrapReady, + 1 => Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }, + _ => Message::Ack(Acknowledgement::Cancel(attempt)), + }; + if mode == 3 { + peer.send(Message::Ack(Acknowledgement::Close)).await; + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } else { + peer.send(preceding).await; + peer.send(Message::Ack(Acknowledgement::Close)).await; + peer.client.shutdown(deadline()).await.unwrap(); + } + } +} + +#[tokio::test] +async fn test_should_keep_partial_framing_when_a_next_waiter_is_cancelled() { + let mut peer = Peer::new(8192); + peer.ready(false).await; + let bytes = encode(&Frame::new(peer.environment, Message::BootstrapReady)).unwrap(); + peer.output + .write_all(bytes.get(..2).unwrap()) + .await + .unwrap(); + let mut waiting = Box::pin(peer.client.next(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + peer.output + .write_all(bytes.get(2..).unwrap()) + .await + .unwrap(); + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + peer.close().await; +} + +#[tokio::test] +async fn test_should_fail_closed_after_a_cancelled_send_without_losing_io_ownership() { + let mut peer = Peer::new(1); + peer.ready(true).await; + let mut sending = Box::pin(peer.client.invoke(invocation(), deadline())); + assert!( + sending + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(sending); + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::Output) + )); + peer.failed().await; +} + +#[tokio::test] +async fn test_should_resume_close_after_its_waiter_is_dropped() { + let mut peer = Peer::new(8192); + peer.ready(false).await; + peer.client.begin_close(deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Close); + let mut waiting = Box::pin(peer.client.shutdown(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + peer.send(Message::Ack(Acknowledgement::Close)).await; + peer.client.shutdown(deadline()).await.unwrap(); + peer.client.shutdown(deadline()).await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_partial_frames_and_cache_receive_deadline_failures() { + for partial in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(false).await; + if partial { + peer.output.write_all(&[0]).await.unwrap(); + } + assert!(peer.client.next(deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test] +async fn test_should_reject_local_invocation_before_bootstrap_and_reused_attempt_identity() { + for bootstrap in [false, true] { + let mut peer = Peer::new(8192); + peer.ready(bootstrap).await; + let call = invocation(); + if bootstrap { + let attempt = call.parts().0; + peer.client.invoke(call.clone(), deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + peer.client.next(deadline()).await.unwrap(); + } + assert!(peer.client.invoke(call, deadline()).await.is_err()); + peer.failed().await; + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_a_queued_frame_when_a_pending_waiter_resumes_after_its_deadline() { + let mut peer = Peer::with_limits(1, 8192, Duration::from_secs(60)); + peer.ready(false).await; + let expires = Instant::now() + Duration::from_secs(1); + let mut waiting = Box::pin(peer.client.next(expires)); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + write_frame( + &mut peer.output, + &Frame::new(peer.environment, Message::BootstrapReady), + deadline(), + ) + .await + .unwrap(); + // Capacity one forces the reader to finish the preceding frame before this + // byte fits. Its next partial-frame budget is sixty seconds, beyond the + // caller's one-second deadline, so a valid queued frame wins an inner poll. + peer.output.write_all(&[0]).await.unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!(waiting.await, Err(RelayError::Deadline))); + peer.failed().await; +} + +#[tokio::test] +async fn test_should_reject_initialization_errors_after_a_result_proves_delivery() { + for mode in 0..4 { + let mut peer = Peer::new(8192); + peer.ready(true).await; + let call = invocation(); + let mut attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.send(Message::InvocationResult { + attempt, + outcome: Outcome::Success, + payload: Payload::try_from(Bytes::new()).unwrap(), + }) + .await; + peer.client.next(deadline()).await.unwrap(); + if mode == 1 || mode == 3 { + let next = invocation(); + attempt = next.parts().0; + peer.client.invoke(next, deadline()).await.unwrap(); + peer.read().await; + } + if mode >= 2 { + peer.client.cancel(attempt, deadline()).await.unwrap(); + peer.read().await; + } + peer.send(Message::InitError { + payload: Payload::try_from(Bytes::from_static(b"late-init-secret")).unwrap(), + }) + .await; + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + peer.failed().await; + } +} + +#[test] +fn test_should_validate_client_io_budget() { + for budget in [Duration::ZERO, Duration::from_secs(61)] { + assert!(ClientConfig::new(EnvironmentId::default(), Architecture::Arm64, budget).is_err()); + } +} diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index 8b88072d..d89bb144 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -6,9 +6,11 @@ #![forbid(unsafe_code)] +pub mod client; mod diagnostics; pub mod http; pub mod metadata; pub mod protocol; pub mod relay; pub mod session; +mod transport; diff --git a/crates/rustack-lambda-runtime/src/relay.rs b/crates/rustack-lambda-runtime/src/relay.rs index 49f81bf8..6761e42f 100644 --- a/crates/rustack-lambda-runtime/src/relay.rs +++ b/crates/rustack-lambda-runtime/src/relay.rs @@ -7,10 +7,10 @@ use std::{future::pending, io, sync::Arc, time::Duration}; use thiserror::Error; use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWrite}, + io::{AsyncRead, AsyncWrite}, runtime::Handle, - sync::{mpsc, oneshot, watch}, - task::{JoinHandle, JoinSet}, + sync::{oneshot, watch}, + task::JoinHandle, time::{Instant, sleep_until, timeout_at}, }; use tracing::instrument::WithSubscriber; @@ -21,12 +21,12 @@ use crate::{ metadata::deadline_from_unix_ms, protocol::{ Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, - ProtocolError, read_frame, write_frame, + ProtocolError, }, session::{Invocation, InvocationTicket, RuntimeResult, SessionError}, + transport::{InputEnd, Settings, Transport, after}, }; -const MAILBOX_CAPACITY: usize = 2; const CONTROL_SILENCE: Duration = Duration::from_hours(1); /// Relay errors contain classes and redacted protocol/IO sources. @@ -218,18 +218,20 @@ where { let runtime = Handle::try_current().map_err(|_| RelayError::Worker)?; let (stop, stopped) = watch::channel(false); - let (input, commands) = mpsc::channel(MAILBOX_CAPACITY); - let (output, frames) = mpsc::channel(MAILBOX_CAPACITY); + let transport = Transport::start( + reader, + writer, + Settings { + silence: config.idle_lease.max(CONTROL_SILENCE), + io_budget: config.io_budget, + input_end: InputEnd::Close, + }, + )?; let initialization_error = api.take_initialization_error(); - let mut workers = JoinSet::new(); - workers.spawn(read_input(reader, input, config).with_current_subscriber()); - workers.spawn(write_output(writer, frames).with_current_subscriber()); let actor = RelayActor::builder() .api(api) .config(config) - .input(commands) - .output(output) - .workers(workers) + .transport(transport) .stop(stopped) .initialization_error(initialization_error) .build(); @@ -241,66 +243,6 @@ where }) } -#[derive(Debug)] -struct Outbound { - frame: Frame, - deadline: Instant, - written: oneshot::Sender, -} - -#[derive(Debug)] -enum WorkerExit { - InputClose, - OutputClosed, -} - -async fn read_input( - mut reader: R, - input: mpsc::Sender, - config: RelayConfig, -) -> Result { - loop { - let deadline = after(config.idle_lease.max(CONTROL_SILENCE))?; - let mut first = [0_u8]; - let count = timeout_at(deadline, reader.read(&mut first)) - .await - .map_err(|_| RelayError::Deadline)??; - if count == 0 { - return Err(RelayError::Eof); - } - let mut prefixed = first.as_slice().chain(&mut reader); - let frame = read_frame(&mut prefixed, after(config.io_budget)?) - .await? - .ok_or(RelayError::Eof)?; - let close = matches!(frame.message(), Message::Close); - timeout_at(after(config.io_budget)?, input.send(frame)) - .await - .map_err(|_| RelayError::Deadline)? - .map_err(|_| RelayError::Worker)?; - if close { - return Ok(WorkerExit::InputClose); - } - } -} - -async fn write_output( - mut writer: W, - mut output: mpsc::Receiver, -) -> Result { - while let Some(frame) = output.recv().await { - let result = write_frame(&mut writer, &frame.frame, frame.deadline).await; - let _ = frame.written.send(result.is_ok()); - result?; - } - Ok(WorkerExit::OutputClosed) -} - -fn after(duration: Duration) -> Result { - Instant::now() - .checked_add(duration) - .ok_or(RelayError::Configuration) -} - #[derive(Debug, Default)] enum Phase { #[default] @@ -323,9 +265,7 @@ impl Phase { struct RelayActor { api: RuntimeApiHandle, config: RelayConfig, - input: mpsc::Receiver, - output: mpsc::Sender, - workers: JoinSet>, + transport: Transport, stop: watch::Receiver, initialization_error: Option>, #[builder(default)] @@ -341,20 +281,8 @@ impl RelayActor { let outcome = self.control().await; self.api.session().close(); let api = self.api.shutdown().await.map_err(RelayError::from); - self.workers.abort_all(); - let mut workers = Ok(()); - while let Some(result) = self.workers.join_next().await { - let result = match result { - Ok(Ok(WorkerExit::InputClose)) => Ok(()), - Ok(Err(error)) => Err(error), - Err(error) if error.is_cancelled() => Ok(()), - Ok(Ok(WorkerExit::OutputClosed)) | Err(_) => Err(RelayError::Worker), - }; - if workers.is_ok() { - workers = result; - } - } - outcome.and(api).and(workers) + let transport = self.transport.shutdown().await; + outcome.and(api).and(transport) } async fn emit(&self, message: Message) -> Result<(), RelayError> { @@ -362,25 +290,15 @@ impl RelayActor { Phase::Retiring(until) => after(self.config.io_budget)?.min(until), _ => after(self.config.io_budget)?, }; - let (written, receipt) = oneshot::channel(); - timeout_at( - deadline, - self.output.send(Outbound { - frame: Frame::new(self.config.environment, message), - deadline, - written, - }), - ) - .await - .map_err(|_| RelayError::Deadline)? - .map_err(|_| RelayError::Output)?; - match timeout_at(deadline, receipt).await { - Ok(Ok(true)) => Ok(()), - _ => Err(RelayError::Output), - } + self.transport + .send(Frame::new(self.config.environment, message), deadline) + .await } async fn control(&mut self) -> Result<(), RelayError> { + if *self.stop.borrow() { + return Ok(()); + } self.emit(Message::Hello { architecture: self.config.architecture, }) @@ -396,11 +314,8 @@ impl RelayActor { biased; () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => return Ok(()), () = retirement_deadline(retirement) => return Err(RelayError::Deadline), - worker = self.workers.join_next(), if !self.workers.is_empty() => { - match worker { Some(Ok(Ok(WorkerExit::InputClose))) => {}, Some(Ok(Err(error))) => return Err(error), _ => return Err(RelayError::Worker) } - } - frame = self.input.recv() => { - let frame = frame.ok_or(RelayError::Eof)?; + frame = self.transport.recv() => { + let frame = frame?; if frame.environment() != self.config.environment { return Err(RelayError::Scope); } if self.command(frame.into_message()).await? { return Ok(()); } } diff --git a/crates/rustack-lambda-runtime/src/relay/tests.rs b/crates/rustack-lambda-runtime/src/relay/tests.rs index adc86628..9f1fae5d 100644 --- a/crates/rustack-lambda-runtime/src/relay/tests.rs +++ b/crates/rustack-lambda-runtime/src/relay/tests.rs @@ -14,7 +14,7 @@ use super::*; use crate::{ http::tests::{Reply, request}, metadata::{deadline_unix_ms, local_trace_id}, - protocol::{FunctionArn, Payload}, + protocol::{FunctionArn, Payload, read_frame, write_frame}, }; fn deadline() -> Instant { @@ -379,30 +379,29 @@ async fn test_should_preserve_a_completed_io_failure_when_priority_stop_wins() { Duration::from_secs(1), ) .unwrap(); - let (_input, commands) = mpsc::channel(MAILBOX_CAPACITY); - let (output, mut frames) = mpsc::channel::(MAILBOX_CAPACITY); - let written = tokio::spawn(async move { - for _ in 0..2 { - frames.recv().await.unwrap().written.send(true).unwrap(); - } - }); - let mut workers = JoinSet::new(); - let failed = workers.spawn(async { Err(RelayError::Eof) }); - while !failed.is_finished() { - tokio::task::yield_now().await; - } + let (input, reader) = duplex(16); + let (writer, _output) = duplex(16); + drop(input); + let mut transport = Transport::start( + reader, + writer, + Settings { + silence: CONTROL_SILENCE, + io_budget: config.io_budget, + input_end: InputEnd::Close, + }, + ) + .unwrap(); + assert!(matches!(transport.recv().await, Err(RelayError::Eof))); let (_stop, stopped) = watch::channel(true); let actor = RelayActor::builder() .api(api) .config(config) - .input(commands) - .output(output) - .workers(workers) + .transport(transport) .stop(stopped) .initialization_error(None) .build(); assert!(matches!(actor.run().await, Err(RelayError::Eof))); - written.await.unwrap(); assert!(TcpStream::connect(address).await.is_err()); } diff --git a/crates/rustack-lambda-runtime/src/transport.rs b/crates/rustack-lambda-runtime/src/transport.rs new file mode 100644 index 00000000..21c87d47 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/transport.rs @@ -0,0 +1,267 @@ +//! Retained, bounded stream workers shared by both ends of relay control. + +use std::time::Duration; + +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite}, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use crate::{ + protocol::{Acknowledgement, Frame, Message, read_frame, write_frame}, + relay::RelayError, +}; + +const MAILBOX_CAPACITY: usize = 2; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum InputEnd { + Close, + CloseAcknowledgement, +} + +impl InputEnd { + fn matches(self, message: &Message) -> bool { + matches!( + (self, message), + (Self::Close, Message::Close) + | ( + Self::CloseAcknowledgement, + Message::Ack(Acknowledgement::Close) + ) + ) + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Settings { + pub(crate) silence: Duration, + pub(crate) io_budget: Duration, + pub(crate) input_end: InputEnd, +} + +#[derive(Debug)] +struct Outbound { + frame: Frame, + deadline: Instant, + written: oneshot::Sender, +} + +#[derive(Debug)] +enum WorkerExit { + InputClose, + OutputClosed, +} + +#[derive(Debug)] +struct OwnerHandle { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} + +#[derive(Debug)] +pub(crate) struct Transport { + input: mpsc::Receiver, + output: mpsc::Sender, + status: watch::Receiver>>, + owner: OwnerHandle, +} + +impl Transport { + pub(crate) fn start(reader: R, writer: W, settings: Settings) -> Result + where + R: AsyncRead + Unpin + Send + 'static, + W: AsyncWrite + Unpin + Send + 'static, + { + let runtime = Handle::try_current().map_err(|_| RelayError::Worker)?; + let (stop, stopped) = watch::channel(false); + let (input, frames) = mpsc::channel(MAILBOX_CAPACITY); + let (output, queued) = mpsc::channel(MAILBOX_CAPACITY); + let (status, observed) = watch::channel(None); + let mut workers = JoinSet::new(); + workers.spawn(read_input(reader, input, settings).with_current_subscriber()); + workers.spawn(write_output(writer, queued).with_current_subscriber()); + let task = runtime.spawn( + Owner { + workers, + stop: stopped, + status, + } + .run() + .with_current_subscriber(), + ); + Ok(Self { + input: frames, + output, + status: observed, + owner: OwnerHandle { + stop, + task: Some(task), + settlement: None, + }, + }) + } + + pub(crate) fn finished(&self) -> bool { + *self.owner.stop.borrow() + || self.status.borrow().is_some() + || self.status.has_changed().is_err() + } + + pub(crate) fn stop(&self) { + self.owner.stop.send_replace(true); + } + + pub(crate) async fn recv(&mut self) -> Result { + loop { + if let Some(result) = self.status.borrow().as_ref() { + return Err(result.clone().err().unwrap_or(RelayError::Eof)); + } + tokio::select! { + biased; + changed = self.status.changed() => { + changed.map_err(|_| RelayError::Worker)?; + } + frame = self.input.recv() => match frame { + Some(frame) => return Ok(frame), + // The reader drops its sender before the supervisor joins + // it. Await that settlement so EOF cannot mask its actual + // protocol/deadline failure. + None => { self.status.changed().await.map_err(|_| RelayError::Worker)?; } + }, + } + } + } + + pub(crate) async fn send(&self, frame: Frame, deadline: Instant) -> Result<(), RelayError> { + if self.finished() { + return Err(RelayError::Output); + } + let (written, receipt) = oneshot::channel(); + timeout_at( + deadline, + self.output.send(Outbound { + frame, + deadline, + written, + }), + ) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Output)?; + match timeout_at(deadline, receipt).await { + Ok(Ok(true)) if Instant::now() < deadline => Ok(()), + Ok(Ok(true)) => Err(RelayError::Deadline), + _ => Err(RelayError::Output), + } + } + + pub(crate) async fn shutdown(&mut self) -> Result<(), RelayError> { + self.stop(); + if let Some(result) = &self.owner.settlement { + return result.clone(); + } + let result = match self.owner.task.as_mut() { + Some(task) => task + .await + .map_err(|_| RelayError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.owner.task.take(); + self.owner.settlement = Some(result.clone()); + result + } +} + +impl Drop for Transport { + fn drop(&mut self) { + self.stop(); + } +} + +#[derive(Debug)] +struct Owner { + workers: JoinSet>, + stop: watch::Receiver, + status: watch::Sender>>, +} + +impl Owner { + async fn run(mut self) -> Result<(), RelayError> { + let mut outcome = loop { + tokio::select! { + biased; + () = async { let _ = self.stop.wait_for(|stop| *stop).await; } => break Ok(()), + worker = self.workers.join_next() => match worker { + Some(Ok(Ok(WorkerExit::InputClose))) => {}, + Some(Ok(Err(error))) => break Err(error), + _ => break Err(RelayError::Worker), + } + } + }; + self.workers.abort_all(); + while let Some(result) = self.workers.join_next().await { + let result = match result { + Ok(Ok(WorkerExit::InputClose)) => Ok(()), + Ok(Err(error)) => Err(error), + Err(error) if error.is_cancelled() => Ok(()), + Ok(Ok(WorkerExit::OutputClosed)) | Err(_) => Err(RelayError::Worker), + }; + outcome = outcome.and(result); + } + self.status.send_replace(Some(outcome.clone())); + outcome + } +} + +async fn read_input( + mut reader: R, + input: mpsc::Sender, + settings: Settings, +) -> Result { + loop { + let mut first = [0_u8]; + let count = timeout_at(after(settings.silence)?, reader.read(&mut first)) + .await + .map_err(|_| RelayError::Deadline)??; + if count == 0 { + return Err(RelayError::Eof); + } + let mut prefixed = first.as_slice().chain(&mut reader); + let frame = read_frame(&mut prefixed, after(settings.io_budget)?) + .await? + .ok_or(RelayError::Eof)?; + let close = settings.input_end.matches(frame.message()); + timeout_at(after(settings.io_budget)?, input.send(frame)) + .await + .map_err(|_| RelayError::Deadline)? + .map_err(|_| RelayError::Worker)?; + if close { + return Ok(WorkerExit::InputClose); + } + } +} + +async fn write_output( + mut writer: W, + mut output: mpsc::Receiver, +) -> Result { + while let Some(frame) = output.recv().await { + let result = write_frame(&mut writer, &frame.frame, frame.deadline).await; + let _ = frame.written.send(result.is_ok()); + result?; + } + Ok(WorkerExit::OutputClosed) +} + +pub(crate) fn after(duration: Duration) -> Result { + Instant::now() + .checked_add(duration) + .ok_or(RelayError::Configuration) +} diff --git a/crates/rustack-lambda-runtime/tests/docker.rs b/crates/rustack-lambda-runtime/tests/docker.rs index e61c6a42..69486fc1 100644 --- a/crates/rustack-lambda-runtime/tests/docker.rs +++ b/crates/rustack-lambda-runtime/tests/docker.rs @@ -11,11 +11,13 @@ use std::{env, os::unix::fs::FileTypeExt, process::Stdio, time::Duration}; use anyhow::{Context, Result, bail, ensure}; use bytes::Bytes; use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, metadata::{deadline_unix_ms, local_trace_id}, protocol::{ Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, read_frame, write_frame, }, + session::{Invocation, RuntimeResult}, }; use serde_json::{Value, json}; use tokio::{ @@ -158,7 +160,7 @@ struct Names { struct Control { child: Child, input: Option, - output: ChildStdout, + output: Option, } #[derive(Debug)] @@ -184,7 +186,7 @@ impl Environment { }) } - async fn start(&mut self, idle_seconds: &str) -> Result<()> { + fn spawn(&mut self, idle_seconds: &str) -> Result<()> { let mut command = self.docker.command(); command .args([ @@ -227,8 +229,13 @@ impl Environment { self.control = Some(Control { child, input: Some(input), - output, + output: Some(output), }); + Ok(()) + } + + async fn start(&mut self, idle_seconds: &str) -> Result<()> { + self.spawn(idle_seconds)?; ensure!( self.next().await? == Message::Hello { @@ -251,9 +258,15 @@ impl Environment { async fn next(&mut self) -> Result { let control = self.control.as_mut().context("relay has not started")?; - let frame = read_frame(&mut control.output, Instant::now() + OPERATION) - .await? - .context("relay closed control output")?; + let frame = read_frame( + control + .output + .as_mut() + .context("relay output is owned by client")?, + Instant::now() + OPERATION, + ) + .await? + .context("relay closed control output")?; ensure!(frame.environment() == self.identity, "wrong environment"); Ok(frame.into_message()) } @@ -274,6 +287,15 @@ impl Environment { } async fn function(&mut self) -> Result<()> { + self.create_function().await?; + ensure!( + self.next().await? == Message::BootstrapReady, + "missing first RIC poll" + ); + Ok(()) + } + + async fn create_function(&self) -> Result<()> { self.docker .checked(&[ "create", @@ -317,10 +339,6 @@ impl Environment { self.docker .checked(&["start", &self.names.function]) .await?; - ensure!( - self.next().await? == Message::BootstrapReady, - "missing first RIC poll" - ); Ok(()) } @@ -375,6 +393,27 @@ impl Environment { Ok(()) } + async fn wait_busy(&self, attempt: AttemptId) -> Result<()> { + let marker = format!("P0_BUSY_START={attempt}"); + let busy_deadline = Instant::now() + OPERATION; + loop { + // The helper owns the deadline so it cannot be cancelled before + // its CLI process has been killed and reaped. + let logs = self + .docker + .checked_until(&["logs", &self.names.function], busy_deadline) + .await?; + if logs.contains(&marker) { + return Ok(()); + } + ensure!( + Instant::now() < busy_deadline, + "handler never entered its busy section" + ); + sleep_until((Instant::now() + Duration::from_millis(50)).min(busy_deadline)).await; + } + } + async fn cleanup(&mut self) -> Result<()> { // Names are reserved before create/run, even if the response is lost. let removal = self @@ -572,24 +611,7 @@ async fn test_should_cancel_a_running_official_ric_and_remove_both_containers() environment.start("600").await?; environment.function().await?; let attempt = environment.invoke(json!({ "sleep": true })).await?; - let marker = format!("P0_BUSY_START={attempt}"); - let busy_deadline = Instant::now() + OPERATION; - loop { - // Forward the absolute deadline into the CLI owner. Cancelling an - // outer timeout here would discard its in-flight kill/reap path. - let logs = environment - .docker - .checked_until(&["logs", &environment.names.function], busy_deadline) - .await?; - if logs.contains(&marker) { - break; - } - ensure!( - Instant::now() < busy_deadline, - "handler never entered its busy section" - ); - sleep_until((Instant::now() + Duration::from_millis(50)).min(busy_deadline)).await; - } + environment.wait_busy(attempt).await?; for _ in 0..2 { environment.send(Message::Cancel { attempt }).await?; ensure!( @@ -616,3 +638,119 @@ async fn test_should_cancel_a_running_official_ric_and_remove_both_containers() .await; environment.finish(result).await } + +fn json_invocation(event: &Value) -> Result { + Ok(Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(60), + "arn:aws:lambda:us-east-1:000000000000:function:spike:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?) +} + +#[tokio::test] +#[ignore = "requires the explicit local Docker relay qualification target"] +async fn test_should_drive_real_ric_through_host_client_and_join_its_streams() -> Result<()> { + for cancel in [false, true] { + let mut environment = Environment::new().await?; + let mut owner = None; + let result = async { + environment.spawn("600")?; + let config = ClientConfig::new( + environment.identity, + environment.docker.architecture, + Duration::from_secs(5), + )?; + let control = environment.control.as_mut().context("relay control")?; + owner = Some(ControlClient::start( + config, + control.output.take().context("client reader")?, + control.input.take().context("client writer")?, + )?); + let client = owner.as_mut().context("client owner")?; + ensure!( + client.next(Instant::now() + OPERATION).await? == ClientEvent::RelayReady, + "client relay handshake failed" + ); + environment.create_function().await?; + ensure!( + client.next(Instant::now() + OPERATION).await? == ClientEvent::BootstrapReady, + "client bootstrap handshake failed" + ); + for (value, computed, counter) in [(35, 42, 1), (0, 7, 2)] { + let call = json_invocation(&json!({"value": value}))?; + let expected = call.parts().0; + client.invoke(call, Instant::now() + OPERATION).await?; + match client.next(Instant::now() + OPERATION).await? { + ClientEvent::InvocationResult { + attempt, + result: RuntimeResult::Success(payload), + } if attempt == expected => { + let result: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + result["computed"] == computed + && result["counter"] == counter + && result["requestId"] == expected.to_string(), + "client computation/warm metadata mismatch" + ); + } + _ => bail!("unexpected client computation event"), + } + ensure!( + client.is_idle(), + "client did not restore Idle after success" + ); + } + let event = if cancel { + json!({"sleep": true}) + } else { + json!({"fail": true}) + }; + let call = json_invocation(&event)?; + let expected = call.parts().0; + client.invoke(call, Instant::now() + OPERATION).await?; + if cancel { + environment.wait_busy(expected).await?; + client.cancel(expected, Instant::now() + OPERATION).await?; + ensure!( + client.next(Instant::now() + OPERATION).await? + == ClientEvent::Cancelled(expected), + "client cancellation acknowledgement mismatch" + ); + } else { + match client.next(Instant::now() + OPERATION).await? { + ClientEvent::InvocationResult { + attempt, + result: RuntimeResult::FunctionError(payload), + } if attempt == expected => { + let result: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + result["errorType"] == "ValueError", + "client function error mismatch" + ); + } + _ => bail!("unexpected client handler error event"), + } + } + ensure!(!client.is_idle(), "failed/cancelled client is reusable"); + environment.remove_function().await?; + client.shutdown(Instant::now() + OPERATION).await?; + environment.wait(true).await + } + .await; + // Always join streams, remove/reconcile containers and reap the CLI, + // including failures after streams have moved out of Control. + let client_cleanup = if let Some(client) = &mut owner { + client + .shutdown(Instant::now() + OPERATION) + .await + .map_err(anyhow::Error::from) + } else { + Ok(()) + }; + let cleanup = environment.cleanup().await; + result.and(client_cleanup).and(cleanup)?; + } + Ok(()) +} diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index b289002e..0aa8cd8d 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -237,6 +237,38 @@ Use safe OwnedFd clones of stdin/stdout with Tokio Unix pipe Receiver/Sender::fr The optional local `test-lambda-runtime-docker` target builds the relay image and explicitly runs the resource-gated process suite against the selected local Unix socket. Missing Docker/images fail this explicit target; ordinary workspace tests do not pretend to provide Docker evidence. The test-only Docker CLI supplies non-TTY binary stdin/stdout; the production executor still requires the separately supervised Docker attach client. Preallocate unique container names and a run label before allocation so setup failures retain reconciliation ownership. Always remove both owned names, reap the CLI process and verify no run-labeled containers remain, even when the assertions fail. Exercise Hello/Ready/Close before a function exists, control EOF/truncation, and an unchanged official RIC image's computed result, warm counter, handler error and busy cancellation. These relay checks do not establish SDK/provider, deployment/pool or full M0 acceptance. +### 3.5 Host control client and shared stream ownership + +The runtime crate's client module exposes ClientConfig::new(environment, architecture, io_budget), ControlClient::start(config, reader, writer), next(absolute_deadline), invoke(Invocation, absolute_io_deadline), cancel(attempt, absolute_io_deadline), begin_close(absolute_io_deadline) and shutdown(absolute_cleanup_deadline). Configuration validates a nonzero IO budget≤60s; invocation metadata uses the same checked wall deadline conversion and validated Invocation types as the local session. ClientEvent distinguishes RelayReady, BootstrapReady, InvocationResult(attempt, RuntimeResult), InitError(payload), Cancelled(attempt) and Closed. Payload Debug remains redacted. The client owns protocol state; it never owns or claims termination of external containers. + +Share the relay's control-stream workers with the host through an internal transport owner. It owns bounded two-message mailboxes, partial-frame parsing, flush receipts, stop signal and a retained supervisor. The reader stops after queuing its direction's terminal message (Close on relay input, Ack/close on host input), so normal peer EOF cannot overtake a queued terminal acknowledgement. First-byte silence and partial-frame IO remain separate budgets. Drop signals stop; shutdown joins/caches its supervisor by mutable borrow, preserves completed IO failures and awaits every aborted worker. A cancelled next() waits on a cancel-safe mailbox receive; it never drops the worker's partial parser. Relay API shutdown still belongs to RelayHandle, before transport retirement. + +Host receive state is AwaitHello→AwaitReady→Initializing→Idle→Busy(attempt)→Idle. Validate environment and advertised architecture before accepting Ready; consume Hello internally and return RelayReady only after the ordered pair. BootstrapReady is accepted once after Ready. Invoke requires Idle and a fresh last-attempt identity. Success restores Idle; FunctionError or InitError enters Retiring. Only one result per attempt is legal; any invocation result proves delivery, after which InitError is causally invalid even during later calls/cancellation. Recheck absolute receive/send deadlines after await and before successful state mutation: an already-ready inner future can win Tokio timeout polling after expiry. Reject duplicate/out-of-order/wrong-direction/foreign-scope frames and mismatched result/ack IDs without echoing their contents; invalidate the transport without replay. + +Set a local write-in-progress flag before awaiting a flush receipt, clearing it only after success. If a send future is dropped, that flag makes later business operations fail closed: the invocation may already have reached the relay. The retained writer still owns any partial frame. Cancellation accepts the active or just-completed last attempt, sends Cancel, and waits through next() for its matching acknowledgement. While Cancelling, at most one matching late result/init failure can be discarded before Cancel Ack; it cannot settle the cancelled call. Matching Cancel after completed/error settlement is permitted to cover completion/cancel races. No fresh Invoke is permitted after cancellation. + +begin_close sends Close after the Hello/Ready handshake and records closure separately from receive state. Until Ack/close, continue validating already queued bootstrap/result/init/cancel events in their ordinary causal order. Ack/close is illegal before an outstanding Cancel Ack. shutdown completes the handshake if necessary, begins closure once, consumes permitted prior events until Closed, then stops/joins IO. A protocol/IO/deadline error invalidates the session; shutdown still joins transport ownership and caches failure. Dropped shutdown waiters retain ownership and a later call must not turn an uncertain write or failed join into success. The environment owner stops/removes the function independently and retains its allocation reservation through confirmed cleanup or quarantine. + +```text +┌─ Host environment owner ───────────────────────────────────────────────┐ +│ ControlClient: ordered handshake / active attempt / cancellation │ +│ next() may be selected/cancelled; invoke/cancel/close await flush │ +│ │ │ +│ ┌───────────────▼─ shared transport supervisor ────────────────┐ │ +│ │ bounded frame mailbox2 ◀── retained partial-frame reader │ │ +│ │ bounded write mailbox2 ──▶ retained writer + flush receipts │ │ +│ │ stop / cached join outcome; abort and join both workers │ │ +│ └──────────────────────┬───────────────────────────────────────┘ │ +└──────────────────────────────┼─────────────────────────────────────────┘ + demultiplexed Docker attach streams (no reconnect) +┌──────────────────────────────▼─────────────────────────────────────────┐ +│ Relay transport / RelayHandle ── RuntimeApiHandle ── official RIC │ +│ Close Ack: API joined + frame flushed; external function still owned │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +Tests pair real relay/shared HTTP with the host client over fragmented duplex streams. Pin ordered readiness, actual HTTP result/error forwarding, warm attempts, wrong scope/architecture/order/IDs, duplicate outcomes, cancellation with a late result, Close with queued events, cancelled next/send/shutdown waiters and cached IO failures. Re-run real Linux relay/RIC qualification after extracting shared IO; a production Docker attach adapter is still required before executor acceptance. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index df60f88c..27c3a0b2 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -71,7 +71,7 @@ For Rust-relevant implementation work run cargo build, cargo test, cargo +nightl The shared runtime contract suite is available as `make test-lambda-runtime`; it runs the library tests/doctests and strict production boundary Clippy without lint ignores. Protocol property tests use [proptest1.12](https://docs.rs/proptest/1.12.0/proptest/) with512 cases per property, exercising bounded malformed bytes, mutations of valid envelopes and arbitrary binary payloads. Only its std feature is enabled; subprocess/fork machinery is unnecessary for these deterministic bounded parser checks. Runtime-session/relay tests extend this same target as their consumers land; a passing codec suite alone does not satisfy Docker acceptance. -`make test-lambda-runtime-docker` builds the selected architecture's relay and pinned official Python RIC fixture images, then explicitly executes every resource-gated test in the shared runtime's Docker suite. It requires WORKFLOWS_DOCKER_SOCKET and a matching WORKFLOWS_ARCH; missing prerequisites fail rather than skip. The CLI is only the test transport. Assert computed42/7, warm counter1/2, request metadata, cgroup/user/rootfs restrictions, distinct FunctionError, running-handler cancellation, pre-function Close, EOF/truncated/partial frames, idle expiry, failed setup and expired helper reaping. Every case reconciles its preallocated owned names/run label and joins its CLI process. These are relay qualification checks, not provider/deployment/pool acceptance for L01–L09. Native x86, Docker Desktop and rootless qualification still require their M4 lanes. +`make test-lambda-runtime-docker` builds the selected architecture's relay and pinned official Python RIC fixture images, then explicitly executes every resource-gated test in the shared runtime's Docker suite. It requires WORKFLOWS_DOCKER_SOCKET and a matching WORKFLOWS_ARCH; missing prerequisites fail rather than skip. The CLI is only the test transport. Assert computed42/7, warm counter1/2, request metadata, cgroup/user/rootfs restrictions, distinct FunctionError, running-handler cancellation, pre-function Close, EOF/truncated/partial frames, idle expiry, failed setup and expired helper reaping. The ordered production ControlClient also consumes real container pipes and drives computed warm calls followed by separate function-error/busy-cancellation scenarios; join its stream supervisor independently of container/CLI cleanup on all exits. Every case reconciles its preallocated owned names/run label and joins its CLI process. These are relay/client qualification checks, not provider/deployment/pool acceptance for L01–L09. Native x86, Docker Desktop and rootless qualification still require their M4 lanes. ## 4. Security acceptance diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 08795cca..3e4b828e 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -141,7 +141,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | -| 1.2 | In progress | Bounded protocol4da7df3, shared actor2c50c2d and Native migration95282c4 are committed. The Linux relay now owns bounded control IO, flush receipts and joined API/worker retirement. Workspace:1,853 tests; runtime:83 tests/39 doctests and strict boundary Clippy; Linux binary cross-target Clippy passes. Four explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including actual official RIC computation/warm reuse/error/cancellation, pipe/lease exits and failed setup/helper cleanup; both run labels reconcile to zero containers. W23–W26 are fixed/rechecked. Host control client remains required; no production Docker executor/provider acceptance yet. | +| 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | | 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. The next production work remains P1 task1.2: build the host control client, then consume the qualified relay/HTTP/session contracts from the task1.3 Docker environment owner. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next consume the qualified client/relay/HTTP/session contracts from the P1 task1.3 Docker deployment/environment owner, then close startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index e68defde..12586296 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -108,6 +108,22 @@ Workspace build/fmt/Clippy/rustdoc and1,853 tests pass. The ordinary suite has39 No dependency version/package was added; existing anyhow/serde_json are test dependencies and tracing-subscriber is also used by the relay entry point. Audit/deny report the same16 B01 advisories; bans/licenses/sources pass. B03 remains open. The production host control client, Docker deployment/pool/provider integration and ASL engine remain required; this is not full P1/M0 acceptance. +## P1 task1.2 host client / shared stream ownership review + +The independent read-only review found two protocol-validation defects. Both were fixed and the final recheck found no remaining concrete in-slice finding. Self-review also corrected an error-class race exposed by the existing partial-frame regression after extracting stream ownership. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W27 | P2; `crates/rustack-lambda-runtime/src/client.rs`, ControlClient::next | Tokio timeout can poll an already-ready inner receive before its expired timer. Recheck the absolute clock after await and before frame/state mutation; shared send also rechecks after its flush receipt | Paused-clock pending waiter, capacity-one input and a queued valid frame reject after caller expiry; no false BootstrapReady | +| W28 | P3; same file, InitError handling | InitError was accepted after an earlier result proved delivery. Track proven delivery, including a discarded late cancellation result, and reject later initialization failures | Ordered Idle, later Busy and both cancellation-state cases fail closed after successful delivery | +| W29 | P2; self-review, `crates/rustack-lambda-runtime/src/transport.rs`, Transport::recv | Reader drops its sender before supervisor settlement, so an empty mailbox could report EOF before the actual parser/deadline failure. Wait for retained supervisor classification on channel closure | Existing paused partial-frame regression still requires Protocol/Deadline; completed EOF/priority-stop test preserves EOF and joined API closure | + +The host validates ordered Hello/architecture/Ready and first-poll BootstrapReady, owns one active attempt, rejects foreign/duplicate/out-of-order events and discards at most one matching late outcome while cancelling. A write-in-progress flag prevents reuse after a dropped send future. Closing preserves validation of queued results/bootstrap/cancel acknowledgements; Ack/close cannot precede outstanding Cancel Ack. Cancelled next/shutdown waiters retain parsing and joined IO ownership. The same private transport supervisor now owns both relay and host stream workers, bounded mailboxes, terminal-frame preservation, flush receipts and cached failure settlement; the relay still separately joins its API before transport retirement. + +Workspace build/test/fmt/Clippy/rustdoc pass with1,867 tests and397 ignored (392 preexisting plus five explicit Docker tests). Runtime:97 tests,47 doctests and strict production-boundary Clippy pass; the Linux entry point passes cross-target Clippy. All five resource-gated Docker tests explicitly pass with zero ignored on arm64 VZ and amd64/QEMU. The new case transfers actual CLI pipes into production ControlClient, performs computed42/7 and warm1/2, then tests separate real FunctionError and busy Cancel paths, joins streams, removes containers, reaps the CLI and reconciles its run label. This qualifies client/relay composition, not the future production Docker attach/deployment/pool/provider. + +No dependency changes landed in this slice. The preceding audit remains unchanged B01 evidence; B01/B03 remain mandatory P1 gates. Docker startup/resource probes, production lifecycle integration and the ASL engine remain required by the complete P0–P5 objective. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index d527a94e..ff61051f 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -132,6 +132,14 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [relay control supervisor and process entry point](./rustack-workflows-20-lambda-runtime-design.md#34-relay-control-supervisor-and-process-entry-point). - Date:2026-10-25. +## D18 — Keep host protocol state separate from retained stream parsing + +- Alternatives: let the Docker environment read/write raw frames directly inside cancellation selects; reconnect after attach failure; put container cleanup into the runtime library. +- Decision: an ordered host ControlClient consumes shared supervised stream workers. Mailbox receive is cancel-safe, partial parsing never restarts, and an interrupted send poisons later business operations until joined retirement. Close preserves ordinary validation of preceding events; cancellation discards only one matching late outcome before its acknowledgement. +- Why: Docker cancellation can race a flushed result or a partial read/write. The host must preserve framing and exact attempt identity without interpreting uncertain transport state as permission to replay or reuse. Container termination and allocation reservations remain with the environment owner that can prove them. +- Pinned by: [host control client and shared stream ownership](./rustack-workflows-20-lambda-runtime-design.md#35-host-control-client-and-shared-stream-ownership). +- Date:2026-10-25. + ## Glossary | Term | Meaning in this specification | From 8a5d652a56d0b70a4c4e9f4a5595ae0c5de6447e Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 18:58:31 -0700 Subject: [PATCH 010/106] phase 1: pin bounded Docker transport contracts before consumer integration Vendor Bollard v0.21.1 at its published source pin and study response collection, request/body timeouts, attach framing/EOF and synchronous dependency logging. Reuse generated model/query contracts while owning the narrow bounded Unix HTTP/upgrade transport in runtime spec 3.6/D19. Independent design review W30 clarifies reserved cleanup capacity and supervised stream handoff so warm connections cannot starve management. W31 source citations are corrected. Source pins, relative links and diff checks pass. This is documentation/reference-source preflight, with no production Rust/dependency changes and no new runtime acceptance claim. P1 Docker consumer and its fake-daemon/real image gates remain required, along with B01/B03 and the complete subsequent workflow implementation. --- .gitmodules | 3 + docs/index.md | 1 + ...udy-bollard-docker-transport-boundaries.md | 73 +++++++++++++++++++ specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 20 ++++- specs/rustack-workflows-90-impl-plan.md | 2 +- specs/rustack-workflows-93-review.md | 11 +++ specs/rustack-workflows-99-decisions.md | 10 +++ vendors/bollard | 1 + 9 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 docs/research/study-bollard-docker-transport-boundaries.md create mode 160000 vendors/bollard diff --git a/.gitmodules b/.gitmodules index bcc7d616..e625a759 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "vendors/localstack"] path = vendors/localstack url = https://github.com/localstack/localstack.git +[submodule "vendors/bollard"] + path = vendors/bollard + url = https://github.com/fussybeaver/bollard.git diff --git a/docs/index.md b/docs/index.md index 72402b8f..d3d128da 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,7 @@ ## Research +- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse and the bounded production transport contract. - [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. - [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. - [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. diff --git a/docs/research/study-bollard-docker-transport-boundaries.md b/docs/research/study-bollard-docker-transport-boundaries.md new file mode 100644 index 00000000..3f943cb4 --- /dev/null +++ b/docs/research/study-bollard-docker-transport-boundaries.md @@ -0,0 +1,73 @@ +# Study: Bollard's Docker Response and Attach Boundaries + +Status: source review complete; no new runtime experiment · Date:2026-10-25 · Vendor pin: `vendors/bollard` @ `f9ec79e7546edfb674b1066c44a68815cb52251c` (`v0.21.1`) + +## Question + +Can the production Lambda environment owner use Bollard's ordinary response/attach APIs while enforcing the repository's byte bounds, strict framing and diagnostic secrecy before materialization? + +The [existing relay experiment](./spike-lambda-container-relay.md) proves binary Docker attach and unchanged official RIC execution. It does not establish bounds inside the selected client's decoders. This study examines that smaller boundary before P1 task1.3 consumes it. The crate's [official version documentation](https://docs.rs/crate/bollard/0.21.1) and the published package's `.cargo_vcs_info.json` agree with this vendored release pin. No performance or cloud-parity claim follows from source inspection. + +## Architecture map + +```text +┌─ Bollard typed endpoint ───────────────────────────────────────────────┐ +│ models/query parameters → request builder → selected transport │ +│ │ synchronous request TRACE │ +│ ┌────────────────▼──────────────────────────────────┐ │ +│ │ Hyper response headers / Incoming │ │ +│ │ request timeout ends after response acquisition │ │ +│ └───────┬───────────────────────┬───────────────────┘ │ +│ │ │ │ +│ ordinary JSON ▼ ▼ HTTP upgrade │ +│ collect entire body → Bytes split read/write → FramedRead │ +│ → response DEBUG → Serde 8-byte Docker header │ +│ → await declared payload length │ +│ → LogOutput / Console on EOF │ +└──────────────────────────────────────────────────────────────────────┘ + ▲ ▲ + │ body limits must precede collect │ framing limits must + │ │ precede buffering +┌─────────┴──────────────────────────────────────┴──────────────────────┐ +│ Rustack host owner: independent deadlines, bounded parsing, retained │ +│ connection tasks, no uncertain replay, mandatory dependency filter │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +## Hot path and allocations + +Endpoint methods construct generated Docker requests. For example, attach creates an upgrade POST and calls the private upgrade helper, then wraps its reader in the log decoder: `vendors/bollard/src/container.rs:540` and `:557`. The independent relay protocol lives *inside* these Docker stdout records; validating a CBOR prefix after receiving LogOutput cannot bound earlier Docker decoder buffering. + +Ordinary model decoding collects the complete Incoming body before Serde: `vendors/bollard/src/docker.rs:1736`. String/error decoding does the same at `:1727`. Neither call site adds a body-length limiter. A bounded timeout is useful but cannot by itself bound bytes accumulated before expiry. Image inspection exposes image-supplied configuration, so these responses are not only small daemon-version records. + +Request execution applies a timeout around obtaining the Hyper response at `vendors/bollard/src/docker.rs:1703`. Response body consumption occurs afterward. Rustack therefore needs an absolute deadline around header acquisition **and** body/frame processing, with a clock recheck before publishing success. This finding does not mean Bollard has no timeout at all; its existing header/request timeout is a different boundary. + +The custom transport interface returns exactly `Response`: `vendors/bollard/src/docker.rs:127`. A straightforward `Response>` wrapper cannot satisfy that type. Lower-level counted IO or a body-forwarding transport can enforce limits, but would add a second transport layer to retain the otherwise opaque decoder. There is no ordinary response-limit parameter on the examined endpoint path. + +## Attach decoder and EOF behavior + +`NewlineLogOutputDecoder` owns a two-stage state: waiting for an eight-byte header, or waiting for a stream-type/declared-length payload. Its header processing at `vendors/bollard/src/read.rs:75` converts the u32 length and records it without a local maximum. At `:80`, it waits until BytesMut contains that length, then splits a complete payload. Incomplete records therefore remain buffered across polls. This is linear work/storage in received record size, rather than a Rustack-specific upper bound. + +The decoder supports both multiplexed output and console/TTY fallback. A leading byte outside its multiplexed range can yield Console, and EOF handling at `vendors/bollard/src/read.rs:103` flushes trailing bytes as Console instead of treating every incomplete non-TTY record as an error. That compatibility policy is reasonable for a general Docker client. The relay protocol instead requires strict non-TTY framing: malformed/truncated records must invalidate the environment and must never become control payload. + +Streaming a validated record in small chunks avoids allocating its whole declared payload. Docker record boundaries and CBOR frame boundaries are independent: a control frame can span records, and a record can carry bytes belonging to successive control frames. Rustack's existing ControlClient already preserves CBOR partial reads through retained IO; the outer decoder must preserve those bytes without relaxing its own validation. + +## Diagnostic dispatch + +Request-body TRACE runs while `process_request` constructs its future, before that future is polled: `vendors/bollard/src/docker.rs:1580`. Model response DEBUG occurs after body collection at `:1738`. Truncated log EOF DEBUG includes the trailing buffer at `vendors/bollard/src/read.rs:110`. + +Wrapping only an already-constructed future is insufficient for synchronous stream builders. A dependency filter must cover construction and every poll, including spawned HTTP/stream workers. Errors must also avoid formatting arbitrary daemon messages or source payloads into infrastructure output. These observations refine W01; they do not establish that W01's production consumer is already implemented. + +## Reusable contracts + +The generated `bollard-stubs` package contains Serde models and query shapes independently of the network client. The release pins version `1.53.1-rc.29.3.1` with default features disabled in `vendors/bollard/Cargo.toml:65`. Its model/query sources are `vendors/bollard/codegen/swagger/src/models.rs` and `query_parameters.rs`. Those types are useful without adopting the response collector, attach decoder, socket discovery, SSH or TLS features. + +The upstream API negotiation behavior at `vendors/bollard/src/docker.rs:1537` selects the lower supported version after reading the daemon version. Rustack should preserve the observable version compatibility while validating the bounded version response and rejecting an incompatible minimum. Do not infer the runtime API version only from a README example; the release's actual constant and generated model pin are more precise. + +## Outcome for the specification + +Use the generated model/query contracts, with a small bounded Unix HTTP/upgrade adapter for the exact Docker operations in this feature. Keep container/deployment ownership above that adapter. The component design, including concrete quotas, connection supervision and error behavior, belongs in [runtime §3.6](../../specs/rustack-workflows-20-lambda-runtime-design.md#36-bounded-docker-api-boundary), not in this research memo. + +The earlier production suggestion to use Bollard's ordinary pipe client is superseded by D19; the retained P0 driver remains valid as a feasibility experiment. This is an in-phase transport refinement, not a request to rewrite the Docker API or support another container platform. + +Required implementation evidence remains bounded fake-daemon tests, actual attach/large-payload/truncation checks, TRACE capture and the established arm64/amd64 RIC lanes. No unresolved source assumption authorizes bypassing those tests. A source review is not proof that the forthcoming bounded adapter works. diff --git a/specs/index.md b/specs/index.md index 206a2392..e94058f0 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation foundations are implemented. Docker execution and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Production Docker execution and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| @@ -22,6 +22,8 @@ Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/ Read 00 → 10 → 20 → 30 → 40; apply 60/70 throughout. Read 80 for delivery planning and 90 for the next engineering step. The [supporting survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) records source verification and remaining runtime experiments. +The [pinned Docker transport study](../docs/research/study-bollard-docker-transport-boundaries.md) refines the P1 consumer boundary: reuse generated Docker types with bounded, supervised Unix HTTP/attach IO. It supersedes the initial ordinary Bollard pipe-client recommendation; it does not change the relay architecture or establish executor acceptance. + ```text ┌─ P0: runtime experiment + semantic fixtures + pinned sources ───────┐ └─────────────────────────────┬──────────────────────────────────────┘ diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 0aa8cd8d..e33da214 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -269,6 +269,24 @@ begin_close sends Close after the Hello/Ready handshake and records closure sepa Tests pair real relay/shared HTTP with the host client over fragmented duplex streams. Pin ordered readiness, actual HTTP result/error forwarding, warm attempts, wrong scope/architecture/order/IDs, duplicate outcomes, cancellation with a late result, Close with queued events, cancelled next/send/shutdown waiters and cached IO failures. Re-run real Linux relay/RIC qualification after extracting shared IO; a production Docker attach adapter is still required before executor acceptance. +### 3.6 Bounded Docker API boundary + +The pinned [Bollard transport study](../docs/research/study-bollard-docker-transport-boundaries.md) supersedes the initial ordinary-pipe-client recommendation. Reuse its independently published generated `bollard-stubs` models/query types (`1.53.1-rc.29.3.1`, default features disabled); own the small Unix HTTP/upgrade adapter for the operations this feature needs. Ordinary Bollard response collection and its permissive log decoder do not enforce these local pre-materialization limits. This changes the transport implementation, not the Docker API or runtime protocol. No production shell/CLI transport, TCP/SSH fallback or TLS feature is introduced. + +DockerSocket is a validated operator-owned unix:// absolute path≤4,096 bytes, without controls/NUL. Resolve/check its Unix-socket metadata under an IO deadline; never consult ambient DOCKER_HOST or discover another socket. DockerWire owns bounded business/cleanup mailboxes of32/8 commands and at most eight management jobs: six ordinary slots and two reserved cleanup slots. Kill/wait/remove/reconciliation use the priority cleanup lane; owner stop has its own signal. Ordinary deployment/start work cannot consume the reserved cleanup slots. Cloneable ports submit typed request envelopes with an absolute deadline; Drop of a response waiter never authorizes replay or loses connection-task ownership. The environment/deployment actor retains names/labels and side-effect reconciliation above the wire layer. + +Each management request owns a fresh Hyper HTTP/1 connection to that socket, with no keepalive pool. One retained job owns connect, handshake, request, bounded response consumption and connection-driver termination through join. Use a single absolute request deadline and recheck the clock after awaits. Bound socket connect, HTTP header parsing, upgrade negotiation and body/frame IO; do not leave a detached Hyper driver or count response headers as complete body success. Cancellation/expiry closes that request's IO and joins its driver. Caller cancellation does not implicitly cancel/retry the service-side resource operation; preallocated identities and reconciliation handle uncertain create results. + +Compile-time wire quotas: management request/response bodies≤1 MiB each; response header buffer≤16 KiB,≤64 headers and≤4 KiB per value; request target≤4 KiB; decoded metadata JSON depth≤32,≤4,096 aggregate members/elements and≤16 KiB per scalar string. The larger scalar ceiling deliberately permits Docker inspection/configuration metadata; stricter image argv/environment/identity rules still validate before domain publication. Reject duplicate JSON object keys, HTTP trailers, wrong content framing and oversized Content-Length before collection. Stream chunked bodies through a byte limiter before materialization. Do not forward raw daemon error text, response bytes, registry credentials or URLs into infrastructure diagnostics; expose bounded operation/status/error classes with redacted sources. + +Negotiate API versions from a bounded /version response. Support the local1.44..1.53 profile: select min(daemon ApiVersion,1.53), require that selection≥daemon MinAPIVersion and1.44, and reject malformed/incompatible versions. Validate daemon Linux OS/architecture and required resource capabilities during startup. Use fixed typed endpoint templates and correct path-component/query encoding. Initial operation families are version/info, image inspect/pull, container create/start/inspect/kill/wait/remove/list, attach/logs, scoped network create/inspect/remove and bounded startup-probe exec. This is a deliberately scoped adapter; arbitrary user-selected Docker paths/methods/headers never reach it. + +Attach establishes the versioned HTTP upgrade before relay start. Its owner retains the Hyper driver until successful upgrade transfer, then owns the upgraded stream and supervised read/write pumps through shutdown. Attach, following logs and pull-progress jobs release their management slot only after transferring the stream **and** retained driver supervision into an owned response. A dropped/failed handoff signals and joins that connection's cleanup. Long-lived stream owners do not retain management slots: they are bounded by the existing environment reservations (control plus function-log streams, up to32 environments) or deployment reservations (pull streams). Thus warm environments cannot starve inspect/kill/remove admission. Do not pass streams through the ordinary Bollard LogOutput decoder. Require a non-TTY multiplexed eight-byte Docker output header: stdout1/stderr2, zero reserved bytes and declared payload≤8 MiB. Reject console/stdin/unknown channels, malformed/truncated headers/payload and oversized declarations before allocating. Stream payload in≤16 KiB chunks instead of buffering a whole Docker record. Docker record and CBOR frame boundaries are independent. Route relay stdout only into the bounded ControlClient byte stream, drain relay stderr into bounded diagnostics storage without logging its raw content, and preserve raw stdin writes separately. A read failure invalidates the environment without reconnect/replay. Missing Close Ack or uncertain stream cleanup cannot release an environment reservation as clean. + +Function logs use the same strict outer framing with their own retained owner and bounded stdout/stderr rings, never sharing a control mailbox. Drain/count/drop excess output instead of blocking the handler. Control and log readers distinguish first-byte/idle waiting from partial-record IO; once a record starts it has the short control IO deadline. Stop signals must interrupt silent or flooding streams. Pull progress is bounded JSON-line streaming,≤64 KiB per line and≤4,096 bytes per displayed error, with the configured total pull deadline; discard progress after extracting bounded status/counts and never aggregate the whole layer-progress stream. + +Mandatory task-bound diagnostic filtering covers Hyper/hyper_util and Bollard DEBUG/TRACE during construction and polling, including every spawned IO task. Keep the model-only production path and its filter explicit; the P0/full-client compatibility probe may still use Bollard and must not reintroduce verbose payload logs. This refines W01 rather than waiving it. Fake local Unix daemons must test oversized/chunked bodies, duplicate/recursive metadata, headers/trailers, malformed multiplexed lengths/reserved bytes/channels, truncated EOF, slow partial records, dropped waiters and TRACE-secret capture. Real daemon tests must prove 6 MiB-class control payload fragmentation, normal Close/Ack and no owned allocation left behind. Startup/limit enforcement and actual SDK executor acceptance remain task1.3–1.4 gates. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. @@ -297,7 +315,7 @@ M4 adds Docker execution of provided.al2023 ZIPs using an operator-pinned OS-onl Apply AGENTS.md § Error Handling/Async/Type/Safety: thiserror sources; retain async-trait for the existing dyn Executor and document object safety; concrete internal actors use ordinary async methods. Use Bytes/Arc revisions, validated artifact enums, redacted Debug, checked memory/deadline arithmetic, forbid unsafe, IO deadlines and supervised JoinSet tasks. Serde control envelopes use camelCase; Runtime API uses AWS headers. Validate raw lengths before parsing. Tests use actual RIC images and cancellation/failure matrices. Tracing contains IDs, durations and classes, not payloads. Avoid per-invoke pull/inspect. Public APIs document Errors and examples; operational docs cover both deployment topologies. -Use the surveyed bollard 0.21.1 API as a starting point and recheck before implementation; disable default features and enable pipe only. Reuse ciborium. TLS transport is N/A for this release; future TLS must use rustls + aws-lc-rs rather than enabling a ring-backed SSL default. +Use the rechecked Bollard0.21.1 generated-type/API contracts through model-only bollard-stubs; the bounded transport decision and quotas are authoritative in §3.6/D19. Reuse Hyper/Tokio/Bytes/ciborium. TLS transport is N/A for this release; future TLS must use rustls + aws-lc-rs rather than enabling a ring-backed SSL default. ## 8. Cross-references diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 3e4b828e..3124afac 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,6 +142,6 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3–1.4, P2–P5 | Pending | Remain required; no Docker executor or workflow-engine acceptance is claimed yet. | +| 1.3–1.4, P2–P5 | Pending | P1 transport consumer design is refined by the pinned Bollard boundary study and §3.6/D19: generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Runtime implementation/acceptance remains required; no Docker executor or workflow-engine acceptance is claimed yet. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next consume the qualified client/relay/HTTP/session contracts from the P1 task1.3 Docker deployment/environment owner, then close startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 12586296..559dc387 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -124,6 +124,17 @@ Workspace build/test/fmt/Clippy/rustdoc pass with1,867 tests and397 ignored (392 No dependency changes landed in this slice. The preceding audit remains unchanged B01 evidence; B01/B03 remain mandatory P1 gates. Docker startup/resource probes, production lifecycle integration and the ASL engine remain required by the complete P0–P5 objective. +## P1 task1.3 transport design preflight + +The pinned [source study](../docs/research/study-bollard-docker-transport-boundaries.md) and runtime §3.6/D19 refine the forthcoming Docker consumer before production code. Independent read-only review confirmed the response collection, timeout, attach decoder and diagnostic facts. These are design/source checks, not runtime acceptance. + +| ID | Priority / source | Finding and resolution | Remaining gate | +|---|---|---|---| +| W30 | P2; runtime §3.6 capacity | An eight-connection limit was ambiguous when32 environments each retain control/log streams. Management now uses six business/two reserved cleanup slots and32/8 mailboxes; supervised stream handoff releases management slots and retains independent environment/deployment reservations | Implement saturation/cleanup-lane tests and prove long-lived streams cannot starve kill/remove | +| W31 | P3; source study citations | Several references named enclosing methods instead of exact collection/length/trace expressions. Corrected to the vendored release's verified lines | Source/link/pin checks; no runtime behavior claim | + +The final independent review found no remaining load-bearing design gap. W01 remains a mandatory consumer/filter/TRACE gate under the refined model-only production path. No Rust production/dependency source changed in this preflight; bounded fake-daemon and actual Docker wire/large-payload evidence remain required before the consumer lands. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index ff61051f..41a89598 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -140,6 +140,16 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [host control client and shared stream ownership](./rustack-workflows-20-lambda-runtime-design.md#35-host-control-client-and-shared-stream-ownership). - Date:2026-10-25. +## D19 — Bound Docker transport before generated-model materialization + +- Context: P1 consumer source review of pinned Bollard0.21.1 found whole-body collection, response-body work outside the header timeout, an attach decoder without a Rustack frame cap and console/trailing-EOF fallback. +- Alternatives: ordinary pipe client with post-deserialization checks; a counted custom connector plus a second raw attach transport; a local forwarding proxy; duplicate Docker model definitions. +- Decision: reuse model-only bollard-stubs1.53.1-rc.29.3.1 and implement the narrow, bounded Unix HTTP/upgrade boundary over the workspace Hyper/Tokio stack. Keep generated wire contracts, strict pre-materialization bounds, joined drivers and the mandatory dependency diagnostic filter. +- Why: checking after collection cannot bound earlier allocation, and an outer CBOR decoder cannot validate opaque Docker buffering first. The custom transport's Response type does not directly accept a Limited body wrapper. Owning the small required operation set avoids a proxy/second connector while preserving explicit cleanup and no-replay semantics. +- Supersedes: the initial recommendation to enable Bollard's ordinary pipe client in runtime §7; D2's relay architecture and the P0 feasibility evidence remain unchanged. +- Pinned by: [bounded Docker API boundary](./rustack-workflows-20-lambda-runtime-design.md#36-bounded-docker-api-boundary), [pinned source study](../docs/research/study-bollard-docker-transport-boundaries.md). +- Date:2026-10-25. Production acceptance still requires the specified fake-daemon and real image tests; source inspection alone closes no runtime gate. + ## Glossary | Term | Meaning in this specification | diff --git a/vendors/bollard b/vendors/bollard new file mode 160000 index 00000000..f9ec79e7 --- /dev/null +++ b/vendors/bollard @@ -0,0 +1 @@ +Subproject commit f9ec79e7546edfb674b1066c44a68815cb52251c From 7cadafd25bc6c49e86d75e171c62fc6f88436037 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 19:07:04 -0700 Subject: [PATCH 011/106] phase 1: bound JSON before Docker metadata model deserialization Add shared JsonLimits/validate/from_slice contracts with encoded-string scratch bounds, decoded key/string byte limits, duplicate-key rejection, container depth and aggregate member budgets. Preflight creates no Value tree and errors contain static redacted classes. Consumers choose their own key limits; Docker's profile remains 256 bytes. Independent review/recheck found no remaining in-slice defect. Core has 39 tests and four doctests, including 21 JSON cases and two 512-case properties. Workspace build/test/fmt/Clippy/rustdoc pass with 1888 tests. Make test-json-boundaries runs the core suite and strict production lints; four existing core doc-format findings are corrected without ignores. No dependency versions/packages were added. Existing 16 B01 advisories and remaining B03 Lambda/auth/model debt remain open P1 gates. Bounded Docker HTTP/stream and deployment/pool consumers remain required before production Docker or full P1/M0 acceptance. --- Cargo.lock | 2 + Makefile | 9 +- crates/rustack-core/Cargo.toml | 2 + crates/rustack-core/src/config.rs | 2 +- crates/rustack-core/src/json.rs | 307 ++++++++++++++++++ crates/rustack-core/src/json/tests.rs | 195 +++++++++++ crates/rustack-core/src/lib.rs | 3 + crates/rustack-core/src/state.rs | 2 +- crates/rustack-core/src/types.rs | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 4 +- specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 10 +- 13 files changed, 538 insertions(+), 9 deletions(-) create mode 100644 crates/rustack-core/src/json.rs create mode 100644 crates/rustack-core/src/json/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 3f904e77..5fe902ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3399,6 +3399,8 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", + "proptest", + "rstest", "serde", "serde_json", "thiserror", diff --git a/Makefile b/Makefile index c755b7c9..cb74e0a8 100644 --- a/Makefile +++ b/Makefile @@ -251,6 +251,13 @@ integration: @cargo test -p rustack-integration -- --ignored # Shared runtime protocol tests and strict production boundary checks. +test-json-boundaries: + @cargo test -p rustack-core + @cargo clippy -p rustack-core --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + test-lambda-runtime: @cargo test -p rustack-lambda-runtime @cargo clippy -p rustack-lambda-runtime --lib --no-deps -- \ @@ -258,7 +265,7 @@ test-lambda-runtime: -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ -W clippy::indexing_slicing -W clippy::panic -.PHONY: test-lambda-runtime +.PHONY: test-json-boundaries test-lambda-runtime # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. diff --git a/crates/rustack-core/Cargo.toml b/crates/rustack-core/Cargo.toml index 9474e0cb..d4728ffe 100644 --- a/crates/rustack-core/Cargo.toml +++ b/crates/rustack-core/Cargo.toml @@ -29,4 +29,6 @@ tracing = { workspace = true } uuid = { workspace = true } [dev-dependencies] +proptest = { workspace = true } +rstest = { workspace = true } tokio = { workspace = true } diff --git a/crates/rustack-core/src/config.rs b/crates/rustack-core/src/config.rs index 30d7acd8..964446a2 100644 --- a/crates/rustack-core/src/config.rs +++ b/crates/rustack-core/src/config.rs @@ -1,6 +1,6 @@ //! Configuration management for Rustack services. //! -//! All configuration is driven by environment variables, matching LocalStack conventions. +//! All configuration is driven by environment variables, matching `LocalStack` conventions. use crate::types::AwsRegion; diff --git a/crates/rustack-core/src/json.rs b/crates/rustack-core/src/json.rs new file mode 100644 index 00000000..5791623b --- /dev/null +++ b/crates/rustack-core/src/json.rs @@ -0,0 +1,307 @@ +//! Bounded JSON preflight before typed deserialization. +//! +//! Encoded strings, decoded keys/values, aggregate members and container depth +//! are checked without building a dynamic Value tree. Errors redact input. + +use std::{collections::BTreeSet, fmt, num::NonZeroUsize}; + +use serde::{ + Deserializer, + de::{DeserializeOwned, DeserializeSeed, Error, MapAccess, SeqAccess, Visitor}, +}; + +/// Redacted JSON validation and target-schema failure classes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum JsonError { + /// Invalid parser limits. + #[error("invalid JSON limits")] + InvalidLimits, + /// Input exceeds its byte allowance. + #[error("JSON byte limit exceeded")] + Bytes, + /// Nested containers exceed their allowance. + #[error("JSON depth limit exceeded")] + Depth, + /// Aggregate object pairs/array elements exceed their allowance. + #[error("JSON member limit exceeded")] + Members, + /// An encoded/decoded scalar string exceeds its allowance. + #[error("JSON string limit exceeded")] + String, + /// A decoded object key exceeds its separate allowance. + #[error("JSON key limit exceeded")] + Key, + /// Two decoded keys in one object are equal. + #[error("duplicate JSON key")] + Duplicate, + /// Invalid JSON, including non-finite numbers. + #[error("invalid JSON syntax")] + Syntax, + /// Bounded JSON does not match the requested domain type. + #[error("invalid JSON schema")] + Schema, +} + +/// Validated byte, container-depth, member and string/key limits. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct JsonLimits { + bytes: NonZeroUsize, + depth: NonZeroUsize, + members: NonZeroUsize, + string_bytes: NonZeroUsize, + key_bytes: NonZeroUsize, +} + +impl JsonLimits { + /// Validate limits before accepting untrusted bytes. + /// + /// # Errors + /// Rejects zero limits, bytes over 16 MiB, depth over 64, members over 65,536, + /// strings over 1 MiB. Strings cannot exceed the body limit; keys cannot + /// exceed strings. Consumers choose their own tighter key allowance. + /// + /// ``` + /// let limits = rustack_core::json::JsonLimits::new(1024, 8, 64, 256, 64)?; + /// # Ok::<(), rustack_core::json::JsonError>(()) + /// ``` + pub fn new( + bytes: usize, + depth: usize, + members: usize, + string_bytes: usize, + key_bytes: usize, + ) -> Result { + if bytes > 16 * 1024 * 1024 + || depth > 64 + || members > 65_536 + || string_bytes > 1024 * 1024 + || string_bytes > bytes + || key_bytes > string_bytes + { + return Err(JsonError::InvalidLimits); + } + let nonzero = |value| NonZeroUsize::new(value).ok_or(JsonError::InvalidLimits); + Ok(Self { + bytes: nonzero(bytes)?, + depth: nonzero(depth)?, + members: nonzero(members)?, + string_bytes: nonzero(string_bytes)?, + key_bytes: nonzero(key_bytes)?, + }) + } +} + +/// Preflight bounded structure and decoded-key uniqueness without a value tree. +/// +/// Depth counts containers: an object/array of scalars has depth one. Members +/// count all object pairs and array elements across the entire input. +/// +/// # Errors +/// Returns a redacted syntax, duplicate or resource-limit class. +/// +/// ``` +/// use rustack_core::json::{JsonLimits, validate}; +/// validate(br#"{"items":[1,true,null]}"#, JsonLimits::new(1024, 8, 64, 256, 64)?)?; +/// # Ok::<(), rustack_core::json::JsonError>(()) +/// ``` +pub fn validate(input: &[u8], limits: JsonLimits) -> Result<(), JsonError> { + if input.len() > limits.bytes.get() { + return Err(JsonError::Bytes); + } + encoded_strings(input, limits.string_bytes.get())?; + let mut context = Scan { + limits, + remaining: limits.members.get(), + failure: None, + }; + let mut decoder = serde_json::Deserializer::from_slice(input); + let result = Node { + context: &mut context, + depth: 0, + } + .deserialize(&mut decoder); + if result.is_err() { + return Err(context.failure.unwrap_or(JsonError::Syntax)); + } + decoder.end().map_err(|_| JsonError::Syntax) +} + +/// Preflight JSON before materializing a typed model; domain validation follows. +/// +/// # Errors +/// Returns preflight classes or Schema without a deserializer's input text. +/// +/// ``` +/// use std::collections::BTreeMap; +/// use rustack_core::json::{JsonLimits, from_slice}; +/// let model: BTreeMap = from_slice(br#"{"count":7}"#, +/// JsonLimits::new(1024, 8, 64, 256, 64)?)?; +/// assert_eq!(model.get("count"), Some(&7)); +/// # Ok::<(), rustack_core::json::JsonError>(()) +/// ``` +pub fn from_slice(input: &[u8], limits: JsonLimits) -> Result { + validate(input, limits)?; + serde_json::from_slice(input).map_err(|_| JsonError::Schema) +} + +fn encoded_strings(input: &[u8], decoded_limit: usize) -> Result<(), JsonError> { + // A JSON escape uses at most six encoded bytes per decoded UTF-8 byte. + // Cap the parser's scratch space before processing escaped strings. + let maximum = decoded_limit + .checked_mul(6) + .ok_or(JsonError::InvalidLimits)?; + let mut inside = false; + let mut escaped = false; + let mut length = 0_usize; + for byte in input { + if !inside { + if *byte == b'"' { + inside = true; + length = 0; + } + } else if !escaped && *byte == b'"' { + inside = false; + } else { + length = length.checked_add(1).ok_or(JsonError::String)?; + if length > maximum { + return Err(JsonError::String); + } + escaped = !escaped && *byte == b'\\'; + } + } + Ok(()) +} + +#[derive(Debug)] +struct Scan { + limits: JsonLimits, + remaining: usize, + failure: Option, +} + +impl Scan { + fn reject(&mut self, error: JsonError) -> E { + self.failure.get_or_insert(error); + E::custom("invalid bounded JSON") + } + fn member(&mut self) -> Result<(), E> { + match self.remaining.checked_sub(1) { + Some(remaining) => { + self.remaining = remaining; + Ok(()) + } + None => Err(self.reject(JsonError::Members)), + } + } +} + +#[derive(Debug)] +struct Node<'a> { + context: &'a mut Scan, + depth: usize, +} + +impl<'de> DeserializeSeed<'de> for Node<'_> { + type Value = (); + fn deserialize>(self, decoder: D) -> Result<(), D::Error> { + decoder.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Node<'_> { + type Value = (); + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("bounded JSON") + } + fn visit_bool(self, _: bool) -> Result<(), E> { + Ok(()) + } + fn visit_i64(self, _: i64) -> Result<(), E> { + Ok(()) + } + fn visit_u64(self, _: u64) -> Result<(), E> { + Ok(()) + } + fn visit_f64(self, value: f64) -> Result<(), E> { + if value.is_finite() { + Ok(()) + } else { + Err(self.context.reject(JsonError::Syntax)) + } + } + fn visit_unit(self) -> Result<(), E> { + Ok(()) + } + fn visit_str(self, value: &str) -> Result<(), E> { + if value.len() > self.context.limits.string_bytes.get() { + return Err(self.context.reject(JsonError::String)); + } + Ok(()) + } + fn visit_seq>(self, mut array: A) -> Result<(), A::Error> { + if self.depth >= self.context.limits.depth.get() { + return Err(self.context.reject(JsonError::Depth)); + } + let depth = self + .depth + .checked_add(1) + .ok_or_else(|| self.context.reject(JsonError::Depth))?; + while array + .next_element_seed(Element { + context: self.context, + depth, + })? + .is_some() + {} + Ok(()) + } + fn visit_map>(self, mut object: A) -> Result<(), A::Error> { + if self.depth >= self.context.limits.depth.get() { + return Err(self.context.reject(JsonError::Depth)); + } + let depth = self + .depth + .checked_add(1) + .ok_or_else(|| self.context.reject(JsonError::Depth))?; + let mut keys = BTreeSet::new(); + while let Some(key) = object.next_key::()? { + self.context.member()?; + if key.len() > self.context.limits.key_bytes.get() { + return Err(self.context.reject(JsonError::Key)); + } + if !keys.insert(key) { + return Err(self.context.reject(JsonError::Duplicate)); + } + object.next_value_seed(Node { + context: self.context, + depth, + })?; + } + Ok(()) + } +} + +// Charge only actual elements, so an empty array remains valid after an +// enclosing object consumes the last available member. +#[derive(Debug)] +struct Element<'a> { + context: &'a mut Scan, + depth: usize, +} + +impl<'de> DeserializeSeed<'de> for Element<'_> { + type Value = (); + fn deserialize>(self, decoder: D) -> Result<(), D::Error> { + self.context.member()?; + Node { + context: self.context, + depth: self.depth, + } + .deserialize(decoder) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-core/src/json/tests.rs b/crates/rustack-core/src/json/tests.rs new file mode 100644 index 00000000..ccaacfb7 --- /dev/null +++ b/crates/rustack-core/src/json/tests.rs @@ -0,0 +1,195 @@ +use std::collections::BTreeMap; + +use proptest::{ + arbitrary::any, + collection::{btree_map, vec}, + prelude::{ProptestConfig, Strategy}, + prop_assert, prop_assert_eq, prop_oneof, proptest, +}; +use rstest::rstest; +use serde::Deserialize; +use serde_json::{Value, json}; + +use super::*; + +fn limits() -> JsonLimits { + JsonLimits::new(16 * 1024, 16, 1024, 256, 32).unwrap() +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct Model { + count: u32, +} + +#[test] +fn test_should_validate_before_typed_schema_and_redact_schema_errors() { + assert_eq!( + from_slice::(br#"{"count":7}"#, limits()).unwrap(), + Model { count: 7 } + ); + let error = from_slice::(br#"{"count":"secret-schema-value"}"#, limits()).unwrap_err(); + assert_eq!(error, JsonError::Schema); + assert!(!format!("{error:?} {error}").contains("secret-schema-value")); + assert_eq!( + from_slice::(br#"{"count":7,"unknown":1,"unknown":2}"#, limits()), + Err(JsonError::Duplicate) + ); +} + +#[rstest] +#[case(br#"{"a":1,"a":2}"#, JsonError::Duplicate)] +#[case(br#"{"a":1,"\u0061":2}"#, JsonError::Duplicate)] +#[case(br#"{"x":{"a":1,"a":2}}"#, JsonError::Duplicate)] +#[case(b"{", JsonError::Syntax)] +#[case(b"[1,]", JsonError::Syntax)] +#[case(b"null true", JsonError::Syntax)] +#[case(b"", JsonError::Syntax)] +#[case(b"01", JsonError::Syntax)] +#[case(b"1e99999", JsonError::Syntax)] +#[case(br#""\ud800""#, JsonError::Syntax)] +#[case(br#""\x00""#, JsonError::Syntax)] +#[case(&[b'"', 0xff, b'"'], JsonError::Syntax)] +fn test_should_reject_duplicates_and_malformed_syntax( + #[case] input: &[u8], + #[case] expected: JsonError, +) { + assert_eq!(validate(input, limits()), Err(expected)); +} + +#[test] +fn test_should_scope_duplicate_keys_to_their_own_object() { + validate(br#"{"a":1,"child":{"a":2}}"#, limits()).unwrap(); + validate(br#"[{"a":1},{"a":2}]"#, limits()).unwrap(); + validate(br#"{"\u00e9":1,"e\u0301":2}"#, limits()).unwrap(); +} + +#[test] +fn test_should_bound_container_depth_without_rejecting_scalar_children() { + let shallow = JsonLimits::new(1024, 1, 64, 256, 32).unwrap(); + for input in [ + b"{}".as_slice(), + b"[]", + b"[1,true,null]", + br#"{"a":1}"#, + b"false", + ] { + validate(input, shallow).unwrap(); + } + for input in [br#"{"a":{}}"#.as_slice(), b"[[]]", br#"[{"a":1}]"#] { + assert_eq!(validate(input, shallow), Err(JsonError::Depth)); + } + let deeply_nested = format!("{}0{}", "[".repeat(4000), "]".repeat(4000)); + let bounded = JsonLimits::new(16 * 1024, 64, 8192, 256, 32).unwrap(); + assert_eq!( + validate(deeply_nested.as_bytes(), bounded), + Err(JsonError::Depth) + ); +} + +#[test] +fn test_should_count_all_object_pairs_and_array_elements_in_one_budget() { + let three = JsonLimits::new(1024, 8, 3, 256, 32).unwrap(); + validate(br#"{"a":[1,2]}"#, three).unwrap(); + assert_eq!( + validate(br#"{"a":[1,2,3]}"#, three), + Err(JsonError::Members) + ); + validate(br#"{"a":{"b":1},"c":2}"#, three).unwrap(); + assert_eq!( + validate(br#"{"a":{"b":1},"c":2,"d":3}"#, three), + Err(JsonError::Members) + ); + let one = JsonLimits::new(1024, 8, 1, 256, 32).unwrap(); + validate(br#"{"a":[]}"#, one).unwrap(); + validate(br#"{"a":{}}"#, one).unwrap(); + validate(b"[]", one).unwrap(); + assert_eq!(validate(br#"{"a":[1]}"#, one), Err(JsonError::Members)); +} + +#[test] +fn test_should_bound_strings_and_keys_by_decoded_utf8_bytes_and_encoded_work() { + let small = JsonLimits::new(1024, 8, 64, 2, 1).unwrap(); + validate("\"é\"".as_bytes(), small).unwrap(); + validate(br#""\u00e9""#, small).unwrap(); + assert_eq!(validate("\"界\"".as_bytes(), small), Err(JsonError::String)); + assert_eq!(validate(br#""\u754c""#, small), Err(JsonError::String)); + assert_eq!(validate("{\"é\":1}".as_bytes(), small), Err(JsonError::Key)); + assert_eq!(validate(br#"{"\u00e9":1}"#, small), Err(JsonError::Key)); + let many_escapes = format!("\"{}\"", "\\u0061".repeat(100)); + assert_eq!( + validate(many_escapes.as_bytes(), small), + Err(JsonError::String) + ); + validate(br#""\\""#, small).unwrap(); + validate(br#""\"""#, small).unwrap(); + let raised_key_limit = JsonLimits::new(1024, 1, 1, 512, 512).unwrap(); + let long_key = format!("{{\"{}\":0}}", "k".repeat(300)); + validate(long_key.as_bytes(), raised_key_limit).unwrap(); + assert_eq!(validate(long_key.as_bytes(), limits()), Err(JsonError::Key)); +} + +#[test] +fn test_should_enforce_input_bytes_before_lexical_processing() { + let tiny = JsonLimits::new(4, 1, 1, 4, 4).unwrap(); + validate(b"null", tiny).unwrap(); + assert_eq!(validate(b"null ", tiny), Err(JsonError::Bytes)); + assert_eq!(validate(&[0xff; 5], tiny), Err(JsonError::Bytes)); +} + +#[test] +fn test_should_reject_invalid_limit_relationships_and_ranges() { + for args in [ + (0, 1, 1, 1, 1), + (1, 0, 1, 1, 1), + (1, 1, 0, 1, 1), + (1, 1, 1, 0, 1), + (1, 1, 1, 1, 0), + (16 * 1024 * 1024 + 1, 1, 1, 1, 1), + (1024, 65, 1, 1, 1), + (1024, 1, 65_537, 1, 1), + (2 * 1024 * 1024, 1, 1, 1024 * 1024 + 1, 1), + (1024, 1, 1, 256, 257), + (1024, 1, 1, 1, 2), + (1, 1, 1, 2, 1), + ] { + assert!(matches!( + JsonLimits::new(args.0, args.1, args.2, args.3, args.4), + Err(JsonError::InvalidLimits) + )); + } +} + +fn values() -> impl Strategy { + prop_oneof![ + any::().prop_map(Value::Bool), + any::().prop_map(|value| json!(value)), + "[a-zA-Z0-9é界\\\"\n]{0,16}".prop_map(Value::String), + proptest::strategy::Just(Value::Null), + ] + .prop_recursive(3, 64, 4, |inner| { + prop_oneof![ + vec(inner.clone(), 0..4).prop_map(Value::Array), + btree_map("[a-z]{1,8}", inner, 0..4) + .prop_map(|map: BTreeMap| Value::Object(map.into_iter().collect())), + ] + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn test_should_never_panic_on_bounded_malformed_json(input in vec(any::(), 0..4096)) { + let result = validate(&input, JsonLimits::new(4096, 8, 64, 64, 32).unwrap()); + if result.is_ok() { prop_assert!(serde_json::from_slice::(&input).is_ok()); } + } + + #[test] + fn test_should_preserve_bounded_recursive_values(value in values()) { + let bytes = serde_json::to_vec(&value).unwrap(); + prop_assert!(bytes.len() <= 16 * 1024); + let parsed: Value = from_slice(&bytes, limits()).unwrap(); + prop_assert_eq!(parsed, value); + } +} diff --git a/crates/rustack-core/src/lib.rs b/crates/rustack-core/src/lib.rs index 9a3f3640..6c1b8d54 100644 --- a/crates/rustack-core/src/lib.rs +++ b/crates/rustack-core/src/lib.rs @@ -4,9 +4,12 @@ //! Rustack service implementations, including multi-account/multi-region //! state management, configuration, and common AWS type definitions. +#![forbid(unsafe_code)] + mod config; mod error; pub mod http; +pub mod json; pub mod settings; mod state; mod types; diff --git a/crates/rustack-core/src/state.rs b/crates/rustack-core/src/state.rs index 16ef0774..d53f8d4d 100644 --- a/crates/rustack-core/src/state.rs +++ b/crates/rustack-core/src/state.rs @@ -1,7 +1,7 @@ //! Multi-account, multi-region state management. //! //! Provides [`AccountRegionStore`], a thread-safe concurrent store that -//! partitions state by AWS account ID and region, matching LocalStack's +//! partitions state by AWS account ID and region, matching `LocalStack`'s //! `AccountRegionBundle` pattern. use std::sync::Arc; diff --git a/crates/rustack-core/src/types.rs b/crates/rustack-core/src/types.rs index 76ee8100..aeb9f3a1 100644 --- a/crates/rustack-core/src/types.rs +++ b/crates/rustack-core/src/types.rs @@ -7,7 +7,7 @@ use std::fmt; pub struct AccountId(String); impl AccountId { - /// Default account ID used by LocalStack. + /// Default account ID used by `LocalStack`. pub const DEFAULT: &str = "000000000000"; /// Create a new account ID from a string. @@ -46,7 +46,7 @@ impl fmt::Display for AccountId { pub struct AwsRegion(String); impl AwsRegion { - /// Default region used by LocalStack. + /// Default region used by `LocalStack`. pub const DEFAULT: &str = "us-east-1"; /// Create a new region. diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index e33da214..ab2c7193 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -279,6 +279,8 @@ Each management request owns a fresh Hyper HTTP/1 connection to that socket, wit Compile-time wire quotas: management request/response bodies≤1 MiB each; response header buffer≤16 KiB,≤64 headers and≤4 KiB per value; request target≤4 KiB; decoded metadata JSON depth≤32,≤4,096 aggregate members/elements and≤16 KiB per scalar string. The larger scalar ceiling deliberately permits Docker inspection/configuration metadata; stricter image argv/environment/identity rules still validate before domain publication. Reject duplicate JSON object keys, HTTP trailers, wrong content framing and oversized Content-Length before collection. Stream chunked bodies through a byte limiter before materialization. Do not forward raw daemon error text, response bytes, registry credentials or URLs into infrastructure diagnostics; expose bounded operation/status/error classes with redacted sources. +Use a reusable rustack-core::json preflight before generated-model deserialization: JsonLimits::new(bytes, container_depth, aggregate_members, string_bytes, key_bytes), validate(bytes, limits) and from_slice(bytes, limits). Enforce the Docker key cap256 bytes in addition to the scalar ceiling. A linear lexical string pass bounds encoded escape expansion before Serde scratch allocation; a DeserializeSeed visitor validates depth/counts/decoded lengths and duplicate decoded keys without building a Value tree. Drop each object's bounded key set after preflight; only then materialize the requested typed model. Error classes distinguish invalid limits, size/depth/count/string/key bounds, duplicates, syntax and target schema without retaining untrusted error text. Tests pin decoded escaped-key duplicates, byte-based Unicode limits, root/container depth, aggregate arrays/maps, malformed numbers/escapes, input limits and panic-free bounded malformed input. These shared primitives may be reused by the ASL compiler, whose own profile/domain validation remains separate. + Negotiate API versions from a bounded /version response. Support the local1.44..1.53 profile: select min(daemon ApiVersion,1.53), require that selection≥daemon MinAPIVersion and1.44, and reject malformed/incompatible versions. Validate daemon Linux OS/architecture and required resource capabilities during startup. Use fixed typed endpoint templates and correct path-component/query encoding. Initial operation families are version/info, image inspect/pull, container create/start/inspect/kill/wait/remove/list, attach/logs, scoped network create/inspect/remove and bounded startup-probe exec. This is a deliberately scoped adapter; arbitrary user-selected Docker paths/methods/headers never reach it. Attach establishes the versioned HTTP upgrade before relay start. Its owner retains the Hyper driver until successful upgrade transfer, then owns the upgraded stream and supervised read/write pumps through shutdown. Attach, following logs and pull-progress jobs release their management slot only after transferring the stream **and** retained driver supervision into an owned response. A dropped/failed handoff signals and joins that connection's cleanup. Long-lived stream owners do not retain management slots: they are bounded by the existing environment reservations (control plus function-log streams, up to32 environments) or deployment reservations (pull streams). Thus warm environments cannot starve inspect/kill/remove admission. Do not pass streams through the ordinary Bollard LogOutput decoder. Require a non-TTY multiplexed eight-byte Docker output header: stdout1/stderr2, zero reserved bytes and declared payload≤8 MiB. Reject console/stdin/unknown channels, malformed/truncated headers/payload and oversized declarations before allocating. Stream payload in≤16 KiB chunks instead of buffering a whole Docker record. Docker record and CBOR frame boundaries are independent. Route relay stdout only into the bounded ControlClient byte stream, drain relay stderr into bounded diagnostics storage without logging its raw content, and preserve raw stdin writes separately. A read failure invalidates the environment without reconnect/replay. Missing Close Ack or uncertain stream cleanup cannot release an environment reservation as clean. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 27c3a0b2..b9df1342 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -1,6 +1,6 @@ # Workflows 70: Verification, Security and Performance Gates -Status: planned acceptance criteria; no new runtime tests executed · Applies to M0–M4 +Status: acceptance contract; executed foundation evidence is recorded in the implementation ledger/review; full M0–M4 acceptance remains open · Applies to M0–M4 ## 1. Evidence and oracle policy @@ -71,6 +71,8 @@ For Rust-relevant implementation work run cargo build, cargo test, cargo +nightl The shared runtime contract suite is available as `make test-lambda-runtime`; it runs the library tests/doctests and strict production boundary Clippy without lint ignores. Protocol property tests use [proptest1.12](https://docs.rs/proptest/1.12.0/proptest/) with512 cases per property, exercising bounded malformed bytes, mutations of valid envelopes and arbitrary binary payloads. Only its std feature is enabled; subprocess/fork machinery is unnecessary for these deterministic bounded parser checks. Runtime-session/relay tests extend this same target as their consumers land; a passing codec suite alone does not satisfy Docker acceptance. +`make test-json-boundaries` verifies the shared core JSON preflight and its strict production-boundary lints. Test escaped-key duplicates, per-object key scope, UTF-8 byte limits, encoded escape work, container depth, aggregate object/array members, empty collections, malformed syntax/numbers, typed schema redaction and configurable limits. Two512-case properties exercise arbitrary bounded malformed bytes and generated recursive valid values. This supports the Docker metadata/ASL parser boundaries; it does not establish their higher-level semantics or executor acceptance. + `make test-lambda-runtime-docker` builds the selected architecture's relay and pinned official Python RIC fixture images, then explicitly executes every resource-gated test in the shared runtime's Docker suite. It requires WORKFLOWS_DOCKER_SOCKET and a matching WORKFLOWS_ARCH; missing prerequisites fail rather than skip. The CLI is only the test transport. Assert computed42/7, warm counter1/2, request metadata, cgroup/user/rootfs restrictions, distinct FunctionError, running-handler cancellation, pre-function Close, EOF/truncated/partial frames, idle expiry, failed setup and expired helper reaping. The ordered production ControlClient also consumes real container pipes and drives computed warm calls followed by separate function-error/busy-cancellation scenarios; join its stream supervisor independently of container/CLI cleanup on all exits. Every case reconciles its preallocated owned names/run label and joins its CLI process. These are relay/client qualification checks, not provider/deployment/pool acceptance for L01–L09. Native x86, Docker Desktop and rootless qualification still require their M4 lanes. ## 4. Security acceptance diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 3124afac..24c33288 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,6 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3–1.4, P2–P5 | Pending | P1 transport consumer design is refined by the pinned Bollard boundary study and §3.6/D19: generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Runtime implementation/acceptance remains required; no Docker executor or workflow-engine acceptance is claimed yet. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire/stream/metadata/lifecycle/pool consumers remain required. | +| 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next consume the qualified client/relay/HTTP/session contracts from the P1 task1.3 Docker deployment/environment owner, then close startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement the task1.3 bounded DockerWire/stream ownership consumer, then deployment/environment actors consuming the qualified client/relay/HTTP/session contracts, followed by startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 559dc387..627e9ef5 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -135,13 +135,21 @@ The pinned [source study](../docs/research/study-bollard-docker-transport-bounda The final independent review found no remaining load-bearing design gap. W01 remains a mandatory consumer/filter/TRACE gate under the refined model-only production path. No Rust production/dependency source changed in this preflight; bounded fake-daemon and actual Docker wire/large-payload evidence remain required before the consumer lands. +## P1 task1.3 shared JSON preflight review + +The independent read-only review found no concrete in-slice defect. It verified per-object decoded-key uniqueness, escaped keys, UTF-8 byte limits, encoded-string scratch bounds, aggregate member/element counting, root/container depth, malformed/trailing JSON, schema-error redaction and controlled key-limit configuration. The final recheck confirmed the generic parser does not hardcode Docker's256-byte key ceiling; consumers explicitly choose that tighter profile. + +rustack-core::json now exposes validated nonzero JsonLimits, validate and from_slice. A linear encoded-string scan precedes DeserializeSeed traversal; the traversal retains only bounded object key sets and builds no dynamic Value tree. Typed materialization follows successful preflight, and domain validation still belongs to the service. Core now forbids unsafe code. Four existing core documentation-format findings were corrected so its complete strict production-boundary check passes without ignores; B03's Lambda/auth/model debt remains open. + +Core has39 passing tests (21 new JSON cases, including two512-case properties) and four passing doctests. `make test-json-boundaries` runs that suite and strict production Clippy. Workspace build/test/fmt/Clippy/rustdoc pass with1,888 tests and397 ignored resource tests. Existing proptest/rstest versions are reused only for core development; no dependency version/package was added. Audit/deny still report16 B01 advisories, with bans/licenses/sources passing. Docker metadata/stream consumers and all full P1/M0 exits remain required; a passing parser is not executor acceptance. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | -| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | Strict production boundary Clippy on lambda-core reports 66 existing findings: 56 missing Errors sections, four documentation-format findings and six indexing/slicing sites. The unfiltered dependency run additionally fails existing rustack-auth/core/model findings. The new identity/request/pool code has no finding in the strict core run; standard workspace Clippy passes | P1 quality work: fix the affected production boundaries and documentation, then rerun strict checks without lint ignores; standard checks alone do not close this gate | +| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | The initial strict lambda-core boundary run found66 baseline findings:56 missing Errors sections, four documentation-format findings and six indexing/slicing sites; its unfiltered dependency run also found auth/core/model debt. The later JSON slice fixes the four rustack-core formatting findings and passes that crate's full strict production check. Remaining Lambda/auth/model findings still require a fresh final audit; standard workspace Clippy passes | P1 quality work: fix the remaining affected boundaries/docs and rerun strict checks without ignores; standard checks alone do not close this gate | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. From 8f11974442698867b3454a28072062a51a2408c2 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 20:04:46 -0700 Subject: [PATCH 012/106] phase 1: own bounded Docker wire IO and qualify real image control streams Implement typed local Unix Docker management, bounded metadata before model deserialization, strict streaming attach and retained driver supervision. Separate lifecycle waits from reserved destructive-cleanup capacity (D20/W32). Apply mandatory dependency filtering and exercise actual Bollard log call sites. Workspace build/test/fmt/Clippy/rustdoc pass with 1903 tests. Near-6MiB official RIC wire qualification and five relay regressions pass on arm64 VZ and amd64/QEMU. Independent read-only review found no remaining in-slice issue. Deployment/pool/provider consumers and B01/B03 remain required before P1/M0 completion. --- Cargo.lock | 107 ++- Cargo.toml | 5 +- Makefile | 5 +- crates/rustack-lambda-core/Cargo.toml | 4 + .../src/docker/connection.rs | 269 ++++++ .../rustack-lambda-core/src/docker/framing.rs | 221 +++++ crates/rustack-lambda-core/src/docker/mod.rs | 838 ++++++++++++++++++ .../rustack-lambda-core/src/docker/tests.rs | 749 ++++++++++++++++ .../rustack-lambda-core/src/docker/types.rs | 205 +++++ crates/rustack-lambda-core/src/lib.rs | 2 + .../rustack-lambda-core/tests/docker_wire.rs | 381 ++++++++ .../rustack-lambda-runtime/src/diagnostics.rs | 14 +- crates/rustack-lambda-runtime/src/lib.rs | 2 +- docs/index.md | 2 +- ...udy-bollard-docker-transport-boundaries.md | 12 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 24 +- .../rustack-workflows-70-verification-plan.md | 4 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 18 +- specs/rustack-workflows-99-decisions.md | 9 + tools/workflows-spikes/Cargo.lock | 288 ++++++ tools/workflows-spikes/Cargo.toml | 11 +- tools/workflows-spikes/handler.py | 7 +- .../workflows-spikes/src/diagnostics_tests.rs | 166 ++++ tools/workflows-spikes/src/lib.rs | 3 + 26 files changed, 3296 insertions(+), 56 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/connection.rs create mode 100644 crates/rustack-lambda-core/src/docker/framing.rs create mode 100644 crates/rustack-lambda-core/src/docker/mod.rs create mode 100644 crates/rustack-lambda-core/src/docker/tests.rs create mode 100644 crates/rustack-lambda-core/src/docker/types.rs create mode 100644 crates/rustack-lambda-core/tests/docker_wire.rs create mode 100644 tools/workflows-spikes/src/diagnostics_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 5fe902ad..be8fca6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,6 +1103,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1176,6 +1182,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "bollard-stubs" +version = "1.53.1-rc.29.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +dependencies = [ + "serde", + "serde_json", + "serde_repr", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -1937,9 +1954,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -2104,9 +2121,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -2143,16 +2160,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http 1.5.0", "http-body 1.1.0", + "httparse", "hyper", "ipnet", "libc", @@ -2977,7 +2995,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "http 1.5.0", @@ -3015,7 +3033,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3114,7 +3132,7 @@ name = "rustack-apigatewayv2-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3168,7 +3186,7 @@ dependencies = [ name = "rustack-auth" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "digest 0.11.3", "hex", @@ -3300,7 +3318,7 @@ dependencies = [ name = "rustack-cloudfront-http" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -3460,7 +3478,7 @@ dependencies = [ name = "rustack-dynamodb-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3660,7 +3678,7 @@ dependencies = [ "aws-sdk-ssm", "aws-sdk-sts", "aws-smithy-runtime-api", - "base64", + "base64 0.22.1", "bytes", "futures", "rand", @@ -3684,7 +3702,7 @@ name = "rustack-kinesis-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3730,7 +3748,7 @@ dependencies = [ name = "rustack-kinesis-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3744,7 +3762,7 @@ version = "0.11.0" dependencies = [ "anyhow", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -3788,7 +3806,7 @@ dependencies = [ name = "rustack-kms-model" version = "0.10.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -3802,15 +3820,18 @@ version = "0.11.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", + "bollard-stubs", "bytes", "chrono", "dashmap", + "form_urlencoded", "http 1.5.0", "http-body-util", "hyper", "hyper-util", "parking_lot", + "percent-encoding", "rstest", "rustack-auth", "rustack-core", @@ -3825,6 +3846,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-subscriber", "typed-builder", "uuid", "winnow", @@ -3835,7 +3857,7 @@ dependencies = [ name = "rustack-lambda-echo-bootstrap" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "rustack-lambda-core", "rustack-lambda-model", "serde_json", @@ -3958,7 +3980,7 @@ name = "rustack-s3-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc32c", @@ -4045,7 +4067,7 @@ dependencies = [ name = "rustack-secretsmanager-core" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4087,7 +4109,7 @@ dependencies = [ name = "rustack-secretsmanager-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "http 1.5.0", @@ -4100,7 +4122,7 @@ name = "rustack-ses-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4157,7 +4179,7 @@ version = "0.11.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4212,7 +4234,7 @@ name = "rustack-sqs-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4259,7 +4281,7 @@ dependencies = [ name = "rustack-sqs-model" version = "0.9.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "http 1.5.0", "serde", @@ -4271,7 +4293,7 @@ name = "rustack-ssm-core" version = "0.11.0" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4323,7 +4345,7 @@ dependencies = [ name = "rustack-sts-core" version = "0.11.0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "dashmap", @@ -4624,6 +4646,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4966,7 +4999,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "312eb6febdc4336a85e5be8bd92e207e729651ce0ee7380225fcb9d8960d5069" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "parking_lot", "serde_json", @@ -5236,9 +5269,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -5295,9 +5328,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -5496,7 +5529,7 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64", + "base64 0.22.1", "log", "once_cell", "url", @@ -5574,9 +5607,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] diff --git a/Cargo.toml b/Cargo.toml index feeaa24f..80d75198 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,7 @@ rustack-cloudfront-dataplane = { version = "0.11.0", path = "crates/rustack-clou squib = "0.2.0" # Async runtime -tokio = { version = "1.53", features = [ +tokio = { version = "1.54", features = [ "rt-multi-thread", "macros", "signal", @@ -105,7 +105,7 @@ tokio = { version = "1.53", features = [ ] } # HTTP / networking -hyper = { version = "1.11", features = ["server", "http1", "http2"] } +hyper = { version = "1.12", features = ["server", "client", "http1", "http2"] } hyper-util = { version = "0.1", features = [ "tokio", "server-auto", @@ -120,6 +120,7 @@ config = { version = "0.15.25", default-features = false, features = ["yaml"] } url = "2.5.8" # Serialization +bollard-stubs = { version = "=1.53.1-rc.29.3.1", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" ciborium = "0.2" diff --git a/Makefile b/Makefile index cb74e0a8..60d1d767 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,10 @@ lambda-runtime-fixture-image: test-lambda-runtime-docker: lambda-runtime-image lambda-runtime-fixture-image @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-runtime --test docker -- --ignored --test-threads=1 -.PHONY: lambda-runtime-build lambda-runtime-image lambda-runtime-fixture-image test-lambda-runtime-docker +test-lambda-docker-wire: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_wire -- --ignored --test-threads=1 + +.PHONY: lambda-runtime-build lambda-runtime-image lambda-runtime-fixture-image test-lambda-runtime-docker test-lambda-docker-wire workflows-model-check: @python3 -c 'import hashlib,json,pathlib; p=pathlib.Path("codegen/smithy-model"); m=json.loads((p/"stepfunctions.provenance.json").read_text()); raw=(p/"stepfunctions.json").read_bytes(); assert len(raw)==m["bytes"] and hashlib.sha256(raw).hexdigest()==m["sha256"], "SFN source hash mismatch"; s=json.loads(raw)["shapes"][m["service"]]; assert s["version"]=="2016-11-23" and "aws.protocols#awsJson1_0" in s["traits"] and s["traits"]["aws.auth#sigv4"]["name"]=="states"; print("Pinned SFN model verified:",len(s["operations"]),"operations")' diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 06222d6a..37bce66b 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -19,6 +19,9 @@ rustack-auth = { workspace = true } squib = { workspace = true } http = { workspace = true } bytes = { workspace = true } +bollard-stubs = { workspace = true } +percent-encoding = { workspace = true } +form_urlencoded = { workspace = true } typed-builder = { workspace = true } winnow = { workspace = true } dashmap = { workspace = true } @@ -42,4 +45,5 @@ parking_lot = { workspace = true } [dev-dependencies] rstest = { workspace = true } +tracing-subscriber = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-core/src/docker/connection.rs b/crates/rustack-lambda-core/src/docker/connection.rs new file mode 100644 index 00000000..b4c16f04 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/connection.rs @@ -0,0 +1,269 @@ +//! One owned Unix HTTP exchange; the driver is joined on every outcome. + +use std::path::Path; + +use bytes::Bytes; +use http::{ + Request, Response, StatusCode, + header::{ + CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, HOST, TRANSFER_ENCODING, + UPGRADE, + }, +}; +use http_body_util::{BodyExt, Full}; +use hyper::{body::Incoming, client::conn::http1::Builder, upgrade::on}; +use hyper_util::rt::TokioIo; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use tokio::{ + net::UnixStream, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use super::{DockerError, MAX_BODY_BYTES, RawReply, RawRequest}; + +pub(super) async fn exchange( + path: &Path, + request: RawRequest, + deadline: Instant, + mut stop: watch::Receiver, +) -> Result { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let connected = async { + let stream = UnixStream::connect(path).await?; + let pair = Builder::new() + .max_headers(64) + .max_buf_size(16 * 1024) + .handshake::<_, Full>(TokioIo::new(stream)) + .await?; + Ok::<_, DockerError>(pair) + }; + let (mut sender, connection) = tokio::select! { + biased; + () = stopped(&mut stop) => return Err(DockerError::Stopped), + result = timeout_at(deadline, connected) => result.map_err(|_| DockerError::Deadline)??, + }; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if *stop.borrow() { + return Err(DockerError::Stopped); + } + let expected = request.expected; + let upgraded = request.expected == StatusCode::SWITCHING_PROTOCOLS; + let mut driver = tokio::spawn( + async move { + if upgraded { + connection.with_upgrades().await + } else { + // This fresh connection is never reused. Recover/drop its owned IO + // instead of half-closing an already-disconnected Unix socket. + connection.without_shutdown().await.map(drop) + } + } + .with_subscriber(runtime_dispatch()), + ); + let operation = async { + let mut builder = Request::builder() + .method(request.method) + .uri(request.target) + .header(HOST, "docker") + .header(CONTENT_LENGTH, request.body.len()) + .header(CONNECTION, if upgraded { "Upgrade" } else { "close" }); + if upgraded { + builder = builder.header(UPGRADE, "tcp"); + } + if !request.body.is_empty() { + builder = builder.header(CONTENT_TYPE, "application/json"); + } + let request = builder + .body(Full::new(request.body)) + .map_err(|_| DockerError::Protocol)?; + let response = sender.send_request(request).await?; + read_response(response, expected, deadline).await + }; + let outcome = tokio::select! { + biased; + () = stopped(&mut stop) => Err(DockerError::Stopped), + result = timeout_at(deadline, operation) => result.map_err(|_| DockerError::Deadline).and_then(|result| result), + }; + drop(sender); + let successful_upgrade = matches!(&outcome, Ok(RawReply::Upgrade(_))); + let joined = join_driver(&mut driver, deadline, successful_upgrade, &mut stop).await; + if matches!(&joined, Err(DockerError::Worker)) { + drop(outcome); + return Err(DockerError::Worker); + } + if Instant::now() >= deadline { + drop(outcome); + return Err(DockerError::Deadline); + } + if *stop.borrow() { + drop(outcome); + return Err(DockerError::Stopped); + } + match (outcome, joined) { + (Ok(reply), Ok(())) => Ok(reply), + (Err(error), _) | (_, Err(error)) => Err(error), + } +} + +async fn read_response( + response: Response, + expected: StatusCode, + deadline: Instant, +) -> Result { + let upgraded = expected == StatusCode::SWITCHING_PROTOCOLS; + check_headers(&response)?; + let status = response.status(); + if upgraded && status == StatusCode::SWITCHING_PROTOCOLS { + let headers = response.headers(); + if !headers + .get(UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"tcp")) + || !headers + .get(CONNECTION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) + }) + || headers.contains_key(TRANSFER_ENCODING) + || headers + .get(CONTENT_LENGTH) + .is_some_and(|value| value.as_bytes() != b"0") + { + return Err(DockerError::Protocol); + } + return Ok(RawReply::Upgrade(TokioIo::new(on(response).await?))); + } + if status == StatusCode::SWITCHING_PROTOCOLS { + return Err(DockerError::Protocol); + } + let body = collect(response.into_body(), deadline).await?; + if status != expected { + return Err(DockerError::Status(status)); + } + Ok(RawReply::Body { + status, + bytes: body, + }) +} + +async fn join_driver( + driver: &mut JoinHandle>, + deadline: Instant, + upgraded: bool, + stop: &mut watch::Receiver, +) -> Result<(), DockerError> { + if upgraded { + let joined = tokio::select! { + biased; + () = stopped(stop) => None, + result = timeout_at(deadline, &mut *driver) => result.ok(), + }; + if let Some(result) = joined { + return result + .map_err(|_| DockerError::Worker)? + .map_err(DockerError::from); + } + } + // Hyper connection IO is cancel-safe. Abort is followed by an actual join; + // it never replaces ownership of a container or an external backend job. + driver.abort(); + match driver.await { + Ok(result) => result.map_err(DockerError::from), + Err(error) if error.is_cancelled() => Ok(()), + Err(_) => Err(DockerError::Worker), + } +} + +pub(super) async fn stopped(stop: &mut watch::Receiver) { + let _ = stop.wait_for(|stop| *stop).await; +} + +fn check_headers(response: &Response) -> Result<(), DockerError> { + let headers = response.headers(); + if headers.len() > 64 { + return Err(DockerError::TooLarge); + } + let mut total = 0_usize; + for (name, value) in headers { + total = total + .checked_add(name.as_str().len()) + .and_then(|length| length.checked_add(value.as_bytes().len())) + .ok_or(DockerError::TooLarge)?; + if name.as_str().len() > 64 || value.as_bytes().len() > 4096 || total > 16 * 1024 { + return Err(DockerError::TooLarge); + } + if !value + .as_bytes() + .iter() + .all(|byte| *byte == b'\t' || (32..=126).contains(byte)) + { + return Err(DockerError::Protocol); + } + } + if headers.get_all(CONTENT_LENGTH).iter().count() > 1 { + return Err(DockerError::Protocol); + } + if let Some(value) = headers.get(CONTENT_LENGTH) { + let value = value.to_str().map_err(|_| DockerError::Protocol)?; + if value.is_empty() || value.len() > 20 || !value.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(DockerError::Protocol); + } + if value.parse::().map_err(|_| DockerError::Protocol)? + > u64::try_from(MAX_BODY_BYTES).map_err(|_| DockerError::TooLarge)? + { + return Err(DockerError::TooLarge); + } + } + if headers + .get(CONTENT_ENCODING) + .is_some_and(|value| !value.as_bytes().eq_ignore_ascii_case(b"identity")) + { + return Err(DockerError::Protocol); + } + if headers + .get(TRANSFER_ENCODING) + .is_some_and(|value| !value.as_bytes().eq_ignore_ascii_case(b"chunked")) + { + return Err(DockerError::Protocol); + } + Ok(()) +} + +async fn collect(mut body: Incoming, deadline: Instant) -> Result { + let mut output = Vec::new(); + let mut quantum = 0_u8; + while let Some(frame) = body.frame().await { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let frame = frame?; + let data = frame.into_data().map_err(|_| DockerError::Protocol)?; + let length = output + .len() + .checked_add(data.len()) + .ok_or(DockerError::TooLarge)?; + if length > MAX_BODY_BYTES { + return Err(DockerError::TooLarge); + } + output + .try_reserve(data.len()) + .map_err(|_| DockerError::Allocation)?; + output.extend_from_slice(&data); + quantum = quantum.saturating_add(1); + if quantum == 64 { + tokio::task::yield_now().await; + quantum = 0; + } + } + Ok(Bytes::from(output)) +} diff --git a/crates/rustack-lambda-core/src/docker/framing.rs b/crates/rustack-lambda-core/src/docker/framing.rs new file mode 100644 index 00000000..b95e2e5a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/framing.rs @@ -0,0 +1,221 @@ +//! Strict streaming non-TTY Docker output records; no full-record allocation. + +use std::{fmt, time::Duration}; + +use bytes::Bytes; +use tokio::{ + io::{AsyncRead, AsyncReadExt}, + time::{Instant, timeout_at}, +}; + +use super::DockerError; + +/// Maximum outer Docker record, distinct from the inner CBOR frame allowance. +pub const MAX_DOCKER_RECORD: usize = 8 * 1024 * 1024; +const CHUNK_BYTES: usize = 16 * 1024; + +/// Docker's validated non-TTY output channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum OutputChannel { + /// Standard output; relay output is binary control data. + Stdout, + /// Standard error; function logs are kept separately from relay control. + Stderr, +} + +/// At most16 KiB from one validated Docker record; Debug redacts content. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct OutputChunk { + /// Output channel advertised by the validated record header. + pub channel: OutputChannel, + /// Raw bytes; no implicit UTF-8 conversion or line splitting. + pub bytes: Bytes, +} +impl fmt::Debug for OutputChunk { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutputChunk") + .field("channel", &self.channel) + .field("bytes", &self.bytes.len()) + .finish() + } +} + +#[derive(Debug)] +struct Record { + channel: OutputChannel, + remaining: usize, + deadline: Instant, +} + +/// One retained parser with a fixed deadline for each partial record. +#[non_exhaustive] +pub struct DockerOutputReader { + reader: R, + record: Option, + io_budget: Duration, + poisoned: bool, + closed: bool, +} +impl fmt::Debug for DockerOutputReader { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DockerOutputReader") + .field("record", &self.record) + .field("poisoned", &self.poisoned) + .field("closed", &self.closed) + .finish_non_exhaustive() + } +} + +impl DockerOutputReader { + /// Own the reader with a nonzero partial-record budget≤60s. + /// + /// # Errors + /// Rejects a zero/excessive IO budget. + /// + /// ``` + /// let reader = rustack_lambda_core::docker::DockerOutputReader::new( + /// b"".as_slice(), std::time::Duration::from_secs(5))?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(reader: R, io_budget: Duration) -> Result { + if io_budget.is_zero() || io_budget > Duration::from_secs(60) { + return Err(DockerError::Configuration); + } + Ok(Self { + reader, + record: None, + io_budget, + poisoned: false, + closed: false, + }) + } + + /// Read one bounded chunk. Cancelling this future invalidates the reader. + /// + /// Boundary EOF returns None; a partial header/payload fails. The caller's + /// absolute deadline may tighten, but never extend, a partial record lease. + /// + /// # Errors + /// Returns framing, size, deadline, IO or interrupted-reader errors. + /// + /// ```no_run + /// # async fn example(reader: &mut rustack_lambda_core::docker::DockerOutputReader) { + /// let chunk = reader.next(tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await; + /// # } + /// ``` + pub async fn next(&mut self, deadline: Instant) -> Result, DockerError> { + if self.poisoned { + return Err(DockerError::Poisoned); + } + if self.closed { + return Ok(None); + } + self.poisoned = true; + let result = self.read(deadline).await; + if result.is_ok() { + self.poisoned = false; + } + result + } + + async fn read(&mut self, deadline: Instant) -> Result, DockerError> { + let mut empty = 0_u8; + loop { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if self.record.is_none() { + let mut header = [0_u8; 8]; + let first = header.get_mut(..1).ok_or(DockerError::Protocol)?; + let count = timeout_at(deadline, self.reader.read(first)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if count == 0 { + self.closed = true; + return Ok(None); + } + let record_deadline = Instant::now() + .checked_add(self.io_budget) + .ok_or(DockerError::Configuration)? + .min(deadline); + timeout_at( + record_deadline, + self.reader + .read_exact(header.get_mut(1..).ok_or(DockerError::Protocol)?), + ) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= record_deadline { + return Err(DockerError::Deadline); + } + let channel = match header.first() { + Some(1) => OutputChannel::Stdout, + Some(2) => OutputChannel::Stderr, + _ => return Err(DockerError::Protocol), + }; + if header.get(1..4) != Some([0_u8; 3].as_slice()) { + return Err(DockerError::Protocol); + } + let length: [u8; 4] = header + .get(4..) + .ok_or(DockerError::Protocol)? + .try_into() + .map_err(|_| DockerError::Protocol)?; + let remaining = usize::try_from(u32::from_be_bytes(length)) + .map_err(|_| DockerError::TooLarge)?; + if remaining > MAX_DOCKER_RECORD { + return Err(DockerError::TooLarge); + } + if remaining == 0 { + empty = empty.saturating_add(1); + if empty == 64 { + tokio::task::yield_now().await; + empty = 0; + } + continue; + } + self.record = Some(Record { + channel, + remaining, + deadline: record_deadline, + }); + } + let record = self.record.as_mut().ok_or(DockerError::Protocol)?; + let until = record.deadline.min(deadline); + if Instant::now() >= until { + return Err(DockerError::Deadline); + } + let size = record.remaining.min(CHUNK_BYTES); + let mut output = Vec::new(); + output + .try_reserve_exact(size) + .map_err(|_| DockerError::Allocation)?; + output.resize(size, 0); + timeout_at(until, self.reader.read_exact(&mut output)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= until { + return Err(DockerError::Deadline); + } + record.remaining = record + .remaining + .checked_sub(size) + .ok_or(DockerError::Protocol)?; + let channel = record.channel; + if record.remaining == 0 { + self.record = None; + } + return Ok(Some(OutputChunk { + channel, + bytes: Bytes::from(output), + })); + } + } +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs new file mode 100644 index 00000000..fa8a9071 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -0,0 +1,838 @@ +//! Bounded local Docker wire service and strict attach framing. +//! +//! This layer owns HTTP connections, not container policy or lifetime. Callers +//! retain preallocated identities and reservations through reconciliation. + +mod connection; +mod framing; +mod types; + +use std::{ + collections::BTreeMap, + fmt, + io::{self, Write}, + path::PathBuf, + time::Duration, +}; + +/// Generated Docker model contracts, without the upstream network client. +pub use bollard_stubs::models; +use bytes::Bytes; +use connection::exchange; +pub use framing::{DockerOutputReader, MAX_DOCKER_RECORD, OutputChannel, OutputChunk}; +use http::{Method, StatusCode}; +use hyper::upgrade::Upgraded; +use hyper_util::rt::TokioIo; +use models::{ + ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, + ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, +}; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +use rustack_core::json::{JsonLimits, from_slice}; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use serde::{Serialize, de::DeserializeOwned}; +use tokio::{ + io::{ReadHalf, WriteHalf, split}, + sync::{mpsc, oneshot, watch}, + task::{JoinHandle, JoinSet}, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; +pub use types::{ApiVersion, DockerError, DockerKey, DockerSocket}; +use uuid::Uuid; + +use crate::artifact::ImageReference; + +const MAX_BODY_BYTES: usize = 1024 * 1024; + +#[derive(Debug, Clone, Copy)] +enum Operation { + Version, + Info, + Image, + Create, + Start, + Inspect, + Kill, + Wait, + Remove, + List, + Attach, +} +impl Operation { + fn cleanup(self) -> bool { + matches!(self, Self::Inspect | Self::Kill | Self::Remove | Self::List) + } +} + +struct RawRequest { + operation: Operation, + method: Method, + target: String, + body: Bytes, + expected: StatusCode, +} +impl fmt::Debug for RawRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DockerRequest") + .field("operation", &self.operation) + .field("bytes", &self.body.len()) + .finish_non_exhaustive() + } +} +enum RawReply { + Body { status: StatusCode, bytes: Bytes }, + Upgrade(TokioIo), +} +impl fmt::Debug for RawReply { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Body { status, bytes } => formatter + .debug_struct("DockerReply") + .field("status", status) + .field("bytes", &bytes.len()) + .finish(), + Self::Upgrade(_) => formatter.write_str("DockerUpgrade()"), + } + } +} +#[derive(Debug)] +struct Envelope { + request: RawRequest, + deadline: Instant, + reply: oneshot::Sender>, +} + +#[derive(Debug, Clone)] +struct RawPort { + business: mpsc::Sender, + cleanup: mpsc::Sender, + waits: mpsc::Sender, + stop: watch::Sender, +} +impl RawPort { + async fn request( + &self, + request: RawRequest, + deadline: Instant, + ) -> Result { + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + check_deadline(deadline)?; + if request.target.len() > 4096 || request.body.len() > MAX_BODY_BYTES { + return Err(DockerError::TooLarge); + } + let sender = if matches!(request.operation, Operation::Wait) { + &self.waits + } else if request.operation.cleanup() { + &self.cleanup + } else { + &self.business + }; + let (reply, response) = oneshot::channel(); + sender + .try_send(Envelope { + request, + deadline, + reply, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Stopped, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } +} + +/// A strict upgraded attach stream whose HTTP driver has already joined. +#[non_exhaustive] +pub struct DockerAttachment(TokioIo); +impl fmt::Debug for DockerAttachment { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerAttachment()") + } +} + +/// Strict stdout/stderr reader for a split upgraded Docker socket. +pub type AttachmentReader = DockerOutputReader>>; +/// Raw container stdin; the owning relay writer supplies framing/IO deadlines. +pub type AttachmentWriter = WriteHalf>; + +impl DockerAttachment { + /// Transfer both socket halves into retained environment IO pumps. + /// + /// # Errors + /// Rejects a zero/over60s record budget. Dropping either returned owner does + /// not prove the external container has exited. + /// + /// ```no_run + /// # fn split(attach: rustack_lambda_core::docker::DockerAttachment) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let (reader, writer) = attach.split(std::time::Duration::from_secs(5))?; + /// # Ok(()) } + /// ``` + pub fn split( + self, + budget: Duration, + ) -> Result<(AttachmentReader, AttachmentWriter), DockerError> { + let (reader, writer) = split(self.0); + Ok((DockerOutputReader::new(reader, budget)?, writer)) + } +} + +/// Fixed ownership-label filter; authorization to reclaim remains above wire IO. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct OwnerFilter { + owner: Uuid, + run: Option, + environment: Option, +} +impl OwnerFilter { + /// Validate nonnil UUID scope before issuing a label-filtered list. + /// + /// # Errors + /// Rejects nil identities or an environment without a run scope. + /// + /// ``` + /// let filter = rustack_lambda_core::docker::OwnerFilter::new(uuid::Uuid::new_v4(), None, None)?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new( + owner: Uuid, + run: Option, + environment: Option, + ) -> Result { + if owner.is_nil() + || run.is_some_and(|id| id.is_nil()) + || environment.is_some_and(|id| id.is_nil()) + || (environment.is_some() && run.is_none()) + { + return Err(DockerError::Configuration); + } + Ok(Self { + owner, + run, + environment, + }) + } + fn encoded(self) -> Result { + let mut labels = vec![format!("rustack.lambda.ownerId={}", self.owner)]; + if let Some(run) = self.run { + labels.push(format!("rustack.lambda.runId={run}")); + } + if let Some(environment) = self.environment { + labels.push(format!("rustack.lambda.environmentId={environment}")); + } + let filters = BTreeMap::from([("label", labels)]); + let filters = serde_json::to_string(&filters).map_err(|_| DockerError::Schema)?; + Ok(form_urlencoded::Serializer::new(String::new()) + .append_pair("all", "true") + .append_pair("limit", "64") + .append_pair("filters", &filters) + .finish()) + } +} + +/// Cloneable typed request port; it does not own service shutdown. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct DockerPort { + raw: RawPort, + version: ApiVersion, +} + +macro_rules! container_method { + ($name:ident, $method:expr, $suffix:literal, $status:expr, $operation:ident, $result:ty, $decode:ident) => { + #[doc = concat!("Issue the scoped container ", stringify!($operation), " operation.")] + /// + /// # Errors + /// Returns bounded admission, deadline, transport, status or schema failure. + /// + #[doc = concat!("```no_run\n# async fn example(port: &rustack_lambda_core::docker::DockerPort, key: &rustack_lambda_core::docker::DockerKey) {\nlet result = port.", stringify!($name), "(key, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await;\n# }\n```")] + pub async fn $name(&self, key: &DockerKey, deadline: Instant) -> Result<$result, DockerError> { + let path = format!("/containers/{}{}", escaped(key.as_str()), $suffix); + let reply = self.call(Operation::$operation, $method, path, Bytes::new(), $status, deadline).await?; + $decode(reply, deadline) + } + }; +} + +impl DockerPort { + /// Negotiated API version, immutable after successful connection startup. + /// + /// ```no_run + /// # fn inspect(port: &rustack_lambda_core::docker::DockerPort) { + /// let version = port.api_version(); + /// # } + /// ``` + #[must_use] + pub fn api_version(&self) -> ApiVersion { + self.version + } + + /// Inspect bounded daemon capabilities; validate execution policy above IO. + /// + /// # Errors + /// Returns admission, deadline, transport or bounded metadata/schema failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort) { + /// let info = port.info(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn info(&self, deadline: Instant) -> Result { + decode( + self.call( + Operation::Info, + Method::GET, + "/info".into(), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Inspect an already validated image reference without pulling it. + /// + /// # Errors + /// Returns missing-image status, deadline, transport or metadata failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, image: &rustack_lambda_core::artifact::ImageReference) { + /// let info = port.inspect_image(image, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_image( + &self, + image: &ImageReference, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::Image, + Method::GET, + format!("/images/{}/json", escaped(image.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Create under a preallocated name retained by the environment owner. + /// + /// # Errors + /// Returns bounded request/schema, status, deadline or transport failure. + /// A failed/dropped response still requires name/label reconciliation. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, name: &rustack_lambda_core::docker::DockerKey, body: &rustack_lambda_core::docker::models::ContainerCreateBody) { + /// let created = port.create_container(name, body, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_container( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + deadline: Instant, + ) -> Result { + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("name", name.as_str()) + .finish(); + decode( + self.call( + Operation::Create, + Method::POST, + format!("/containers/create?{query}"), + encode(body)?, + StatusCode::CREATED, + deadline, + ) + .await?, + deadline, + ) + } + + container_method!( + start_container, + Method::POST, + "/start", + StatusCode::NO_CONTENT, + Start, + (), + unit + ); + container_method!( + inspect_container, + Method::GET, + "/json", + StatusCode::OK, + Inspect, + ContainerInspectResponse, + decode + ); + container_method!( + kill_container, + Method::POST, + "/kill?signal=SIGKILL", + StatusCode::NO_CONTENT, + Kill, + (), + unit + ); + container_method!( + wait_container, + Method::POST, + "/wait?condition=not-running", + StatusCode::OK, + Wait, + ContainerWaitResponse, + decode + ); + container_method!( + remove_container, + Method::DELETE, + "?force=true&v=false", + StatusCode::NO_CONTENT, + Remove, + (), + unit + ); + + /// List only a fixed ownership-label scope for reconciliation. + /// + /// # Errors + /// Returns admission, deadline, status, transport or metadata-budget failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, filter: rustack_lambda_core::docker::OwnerFilter) { + /// let found = port.list_containers(filter, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn list_containers( + &self, + filter: OwnerFilter, + deadline: Instant, + ) -> Result, DockerError> { + decode( + self.call( + Operation::List, + Method::GET, + format!("/containers/json?{}", filter.encoded()?), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Establish binary attach before starting the relay; no log replay. + /// + /// # Errors + /// Returns invalid upgrade, status, deadline or joined transport failure. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, relay: &rustack_lambda_core::docker::DockerKey) { + /// let attached = port.attach(relay, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn attach( + &self, + relay: &DockerKey, + deadline: Instant, + ) -> Result { + let reply = self + .call( + Operation::Attach, + Method::POST, + format!( + "/containers/{}/attach?stdin=true&stdout=true&stderr=true&stream=true&\ + logs=false", + escaped(relay.as_str()) + ), + Bytes::new(), + StatusCode::SWITCHING_PROTOCOLS, + deadline, + ) + .await?; + match reply { + RawReply::Upgrade(io) => Ok(DockerAttachment(io)), + RawReply::Body { .. } => Err(DockerError::Protocol), + } + } + + async fn call( + &self, + operation: Operation, + method: Method, + path: String, + body: Bytes, + expected: StatusCode, + deadline: Instant, + ) -> Result { + let target = format!("/v{}{path}", self.version); + self.raw + .request( + RawRequest { + operation, + method, + target, + body, + expected, + }, + deadline, + ) + .await + } +} + +/// Owner of bounded request scheduling and all in-flight management drivers. +#[derive(Debug)] +#[non_exhaustive] +pub struct DockerWire { + port: DockerPort, + owner: Owner, +} + +impl DockerWire { + /// Verify the explicit socket and negotiate a bounded API response. + /// + /// # Errors + /// Returns unavailable socket, malformed/incompatible API, IO or supervision + /// failure. Dropped startup waiters retain cleanup through the owner task. + /// + /// ```no_run + /// # async fn example() -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let socket = "unix:///var/run/docker.sock".parse()?; + /// let mut wire = rustack_lambda_core::docker::DockerWire::connect(socket, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// wire.shutdown().await?; + /// # Ok(()) } + /// ``` + pub async fn connect(socket: DockerSocket, deadline: Instant) -> Result { + tokio::runtime::Handle::try_current().map_err(|_| DockerError::Worker)?; + let path = socket.verified(deadline).await?; + let (mut owner, raw) = Owner::start(path); + let result = async { + let response = raw + .request( + RawRequest { + operation: Operation::Version, + method: Method::GET, + target: "/version".into(), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline, + ) + .await?; + let version: SystemVersion = decode(response, deadline)?; + ApiVersion::negotiate( + version.api_version.as_deref().ok_or(DockerError::Version)?, + version + .min_api_version + .as_deref() + .ok_or(DockerError::Version)?, + ) + } + .await; + match result { + Ok(version) => Ok(Self { + port: DockerPort { raw, version }, + owner, + }), + Err(error) => { + owner.shutdown().await?; + Err(error) + } + } + } + + /// Clone a typed port; the original wire keeps shutdown responsibility. + /// + /// ```no_run + /// # fn port(wire: &rustack_lambda_core::docker::DockerWire) { + /// let port = wire.port(); + /// # } + /// ``` + #[must_use] + pub fn port(&self) -> DockerPort { + self.port.clone() + } + + /// Stop admission and join every retained management request/driver. + /// + /// # Errors + /// Caches task supervision failures. Exported stream owners and external + /// containers must be joined/removed by their environment owner separately. + /// + /// ```no_run + /// # async fn shutdown(wire: &mut rustack_lambda_core::docker::DockerWire) { + /// let settled = wire.shutdown().await; + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result<(), DockerError> { + self.owner.shutdown().await + } +} + +#[derive(Debug)] +struct Owner { + stop: watch::Sender, + task: Option>>, + settlement: Option>, +} +impl Owner { + fn start(socket: PathBuf) -> (Self, RawPort) { + let (business, requests) = mpsc::channel(32); + let (cleanup, priority) = mpsc::channel(8); + let (waits, pending_waits) = mpsc::channel(8); + let (stop, _) = watch::channel(false); + let actor = Actor::builder() + .socket(socket) + .business(requests) + .cleanup(priority) + .waits(pending_waits) + .control(stop.clone()) + .build(); + let task = tokio::spawn(actor.run().with_subscriber(runtime_dispatch())); + ( + Self { + stop: stop.clone(), + task: Some(task), + settlement: None, + }, + RawPort { + business, + cleanup, + waits, + stop, + }, + ) + } + async fn shutdown(&mut self) -> Result<(), DockerError> { + self.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + let result = match self.task.as_mut() { + Some(task) => task + .await + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Ok(()), + }; + self.task.take(); + self.settlement = Some(result.clone()); + result + } +} +impl Drop for Owner { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[derive(Debug, TypedBuilder)] +struct Actor { + socket: PathBuf, + business: mpsc::Receiver, + cleanup: mpsc::Receiver, + waits: mpsc::Receiver, + control: watch::Sender, + #[builder(default)] + jobs: JoinSet<(Lane, bool)>, + #[builder(default)] + counts: Counts, +} + +#[derive(Debug, Clone, Copy)] +enum Lane { + Business, + Cleanup, + Wait, +} +#[derive(Debug, Default)] +struct Counts { + business: usize, + cleanup: usize, + waits: usize, +} +impl Counts { + fn get_mut(&mut self, lane: Lane) -> &mut usize { + match lane { + Lane::Business => &mut self.business, + Lane::Cleanup => &mut self.cleanup, + Lane::Wait => &mut self.waits, + } + } +} +impl Actor { + async fn run(mut self) -> Result<(), DockerError> { + let mut stop = self.control.subscribe(); + let mut outcome = loop { + tokio::select! { + biased; + () = connection::stopped(&mut stop) => break Ok(()), + job = self.jobs.join_next(), if !self.jobs.is_empty() => match job { + Some(Ok((lane, false))) => { + let count = self.counts.get_mut(lane); + if let Some(value) = count.checked_sub(1) { *count = value; } else { break Err(DockerError::Worker); } + } + _ => break Err(DockerError::Worker), + }, + envelope = self.cleanup.recv(), if self.counts.cleanup < 2 => match envelope { Some(envelope) => self.start(envelope, Lane::Cleanup), None => break Err(DockerError::Stopped) }, + envelope = self.waits.recv(), if self.counts.waits < 32 => match envelope { Some(envelope) => self.start(envelope, Lane::Wait), None => break Err(DockerError::Stopped) }, + envelope = self.business.recv(), if self.counts.business < 6 => match envelope { Some(envelope) => self.start(envelope, Lane::Business), None => break Err(DockerError::Stopped) }, + } + }; + self.control.send_replace(true); + self.business.close(); + self.cleanup.close(); + self.waits.close(); + while let Ok(envelope) = self.business.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Ok(envelope) = self.cleanup.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Ok(envelope) = self.waits.try_recv() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + } + while let Some(result) = self.jobs.join_next().await { + if !matches!(result, Ok((_, false))) { + outcome = Err(DockerError::Worker); + } + } + outcome + } + fn start(&mut self, envelope: Envelope, lane: Lane) { + if *self.control.borrow() { + let _ = envelope.reply.send(Err(DockerError::Stopped)); + return; + } + if envelope.reply.is_closed() { + return; + } + if Instant::now() >= envelope.deadline { + let _ = envelope.reply.send(Err(DockerError::Deadline)); + return; + } + let socket = self.socket.clone(); + let stopped = self.control.subscribe(); + *self.counts.get_mut(lane) += 1; + self.jobs.spawn( + async move { + let result = exchange(&socket, envelope.request, envelope.deadline, stopped).await; + let failed = matches!(&result, Err(DockerError::Worker)); + let _ = envelope.reply.send(result); + (lane, failed) + } + .with_subscriber(runtime_dispatch()), + ); + } +} + +fn escaped(value: &str) -> String { + utf8_percent_encode(value, NON_ALPHANUMERIC).to_string() +} +fn check_deadline(deadline: Instant) -> Result<(), DockerError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(600) { + return Err(DockerError::Configuration); + } + Ok(()) +} +fn decode(reply: RawReply, deadline: Instant) -> Result { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let result = match reply { + RawReply::Body { bytes, .. } => from_slice( + &bytes, + JsonLimits::new(MAX_BODY_BYTES, 32, 4096, 16 * 1024, 256)?, + ) + .map_err(DockerError::from), + RawReply::Upgrade(_) => Err(DockerError::Protocol), + }; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result +} +fn unit(reply: RawReply, deadline: Instant) -> Result<(), DockerError> { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + match reply { + RawReply::Body { bytes, .. } if bytes.is_empty() => Ok(()), + _ => Err(DockerError::Protocol), + } +} + +#[derive(Default)] +struct Encoded { + bytes: Vec, + failure: Option, +} +impl fmt::Debug for Encoded { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Encoded") + .field("bytes", &self.bytes.len()) + .field("failure", &self.failure) + .finish() + } +} +impl Write for Encoded { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let length = self + .bytes + .len() + .checked_add(bytes.len()) + .filter(|length| *length <= MAX_BODY_BYTES); + if length.is_none() { + self.failure = Some(DockerError::TooLarge); + return Err(io::Error::other("Docker byte budget exceeded")); + } + if self.bytes.try_reserve(bytes.len()).is_err() { + self.failure = Some(DockerError::Allocation); + return Err(io::Error::other("Docker allocation failed")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn encode(value: &impl Serialize) -> Result { + let mut output = Encoded::default(); + if serde_json::to_writer(&mut output, value).is_err() { + return Err(output.failure.unwrap_or(DockerError::Schema)); + } + Ok(Bytes::from(output.bytes)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/tests.rs b/crates/rustack-lambda-core/src/docker/tests.rs new file mode 100644 index 00000000..b57c7058 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/tests.rs @@ -0,0 +1,749 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use tempfile::TempDir; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, duplex}, + net::{UnixListener, UnixStream}, +}; + +use super::*; + +#[derive(Debug, Clone)] +struct LogWriter(std::sync::Arc>>); +impl Write for LogWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self.0.lock().unwrap(); + if output.len().saturating_add(bytes.len()) > 2 * MAX_BODY_BYTES { + return Err(io::Error::other("capture exceeded limit")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} + +struct IncomingRequest { + head: String, + body: Vec, + stream: UnixStream, +} +impl IncomingRequest { + async fn reply(mut self, status: u16, body: &[u8]) { + let head = format!( + "HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + timeout_at(deadline(), async { + self.stream.write_all(head.as_bytes()).await?; + self.stream.write_all(body).await + }) + .await + .unwrap() + .unwrap(); + } + async fn raw(mut self, response: &[u8]) { + timeout_at(deadline(), self.stream.write_all(response)) + .await + .unwrap() + .unwrap(); + } +} + +struct Daemon { + directory: TempDir, + requests: mpsc::Receiver, + stop: watch::Sender, + task: JoinHandle<()>, +} +impl Daemon { + fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let listener = UnixListener::bind(directory.path().join("docker.sock")).unwrap(); + let (requests, received) = mpsc::channel(32); + let (stop, mut stopped) = watch::channel(false); + let task = tokio::spawn(async move { + let mut jobs = JoinSet::new(); + loop { + tokio::select! { + biased; + () = connection::stopped(&mut stopped) => break, + job = jobs.join_next(), if !jobs.is_empty() => { job.unwrap().unwrap(); }, + stream = listener.accept(), if jobs.len() < 16 => { + let requests = requests.clone(); + let (stream, _) = stream.unwrap(); + jobs.spawn(async move { + // Closing a fixture connection during shutdown is + // normal. Panics still fail the joined server task. + if let Ok(incoming) = read_request(stream).await { + if incoming.head.starts_with("GET /version ") { + incoming.reply(200, br#"{"ApiVersion":"1.53","MinAPIVersion":"1.44"}"#).await; + } else { let _ = requests.send(incoming).await; } + } + }); + } + } + } + jobs.abort_all(); + while let Some(result) = jobs.join_next().await { + if let Err(error) = result { + assert!(error.is_cancelled()); + } + } + }); + Self { + directory, + requests: received, + stop, + task, + } + } + fn socket(&self) -> DockerSocket { + format!( + "unix://{}", + self.directory.path().join("docker.sock").display() + ) + .parse() + .unwrap() + } + async fn wire(&self) -> DockerWire { + match DockerWire::connect(self.socket(), deadline()).await { + Ok(wire) => wire, + Err(DockerError::Http(error)) => panic!("fixture HTTP startup failed: {error:?}"), + Err(error) => panic!("fixture startup failed: {error}"), + } + } + async fn next(&mut self) -> IncomingRequest { + timeout_at(deadline(), self.requests.recv()) + .await + .unwrap() + .unwrap() + } + async fn close(self) { + self.stop.send_replace(true); + self.task.await.unwrap(); + } +} + +async fn read_request(mut stream: UnixStream) -> io::Result { + timeout_at(deadline(), async { + let mut input = Vec::new(); + let boundary = loop { + if let Some(index) = input.windows(4).position(|value| value == b"\r\n\r\n") { + break index + 4; + } + if input.len() > 16 * 1024 { + return Err(io::Error::other("fixture header exceeded cap")); + } + let mut block = [0; 512]; + let count = stream.read(&mut block).await?; + if count == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + input.extend_from_slice(block.get(..count).unwrap()); + }; + let head = + String::from_utf8(input.get(..boundary).unwrap().to_vec()).map_err(io::Error::other)?; + let length = head + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + if length > MAX_BODY_BYTES { + return Err(io::Error::other("fixture body exceeded cap")); + } + let mut body = input.get(boundary..).unwrap().to_vec(); + let missing = length + .checked_sub(body.len()) + .ok_or_else(|| io::Error::other("unexpected pipelining"))?; + let mut remainder = vec![0; missing]; + stream.read_exact(&mut remainder).await?; + body.extend(remainder); + Ok(IncomingRequest { head, body, stream }) + }) + .await + .map_err(|_| io::Error::other("fixture deadline"))? +} + +fn record(channel: u8, bytes: &[u8]) -> Vec { + let mut output = vec![channel, 0, 0, 0]; + output.extend_from_slice(&u32::try_from(bytes.len()).unwrap().to_be_bytes()); + output.extend_from_slice(bytes); + output +} + +#[tokio::test] +async fn test_should_negotiate_bound_metadata_encode_paths_and_preserve_status() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + assert_eq!(port.api_version().to_string(), "1.53"); + let (info, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + assert!(request.head.starts_with("GET /v1.53/info ")); + request + .reply(200, br#"{"ID":"daemon-id","OSType":"linux"}"#) + .await; + }); + assert_eq!(info.unwrap().id.as_deref(), Some("daemon-id")); + let image: ImageReference = "localhost:5000/team/worker:v1".parse().unwrap(); + let (result, ()) = tokio::join!(port.inspect_image(&image, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("GET /v1.53/images/localhost%3A5000%2Fteam%2Fworker%3Av1/json ") + ); + request.reply(404, b"secret daemon error").await; + }); + let error = result.unwrap_err(); + assert!(matches!(error, DockerError::Status(StatusCode::NOT_FOUND))); + assert!(!format!("{error:?} {error}").contains("secret")); + wire.shutdown().await.unwrap(); + assert!(matches!( + port.info(deadline()).await, + Err(DockerError::Stopped) + )); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_reject_oversized_truncated_chunked_trailer_and_duplicate_metadata() { + for mode in 0..6 { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + match mode { + 0 => request.raw(b"HTTP/1.1 200 OK\r\nContent-Length: 1048577\r\n\r\n").await, + 1 => request.raw(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n{}").await, + 2 => request.raw(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\n{}\r\n0\r\nSecret: trailer\r\n\r\n").await, + 3 => request.reply(200, br#"{"ID":"one","ID":"two"}"#).await, + 4 => request.raw(format!("HTTP/1.1 200 OK\r\nSecret: {}\r\nContent-Length: 2\r\n\r\n{{}}", "x".repeat(4097)).as_bytes()).await, + _ => { + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + response.extend_from_slice(b"100001\r\n"); + response.extend(std::iter::repeat_n(b'x', MAX_BODY_BYTES + 1)); + response.extend_from_slice(b"\r\n0\r\n\r\n"); + // The client closes as soon as its byte cap is reached. + let mut stream = request.stream; + let _ = timeout_at(deadline(), stream.write_all(&response)).await; + } + } + }); + assert!(result.is_err(), "case {mode}"); + wire.shutdown().await.unwrap(); + daemon.close().await; + } +} + +#[tokio::test] +async fn test_should_send_typed_container_create_and_management_operations() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key: DockerKey = "owned-test".parse().unwrap(); + let body = ContainerCreateBody { + image: Some("sha256:fixture".into()), + env: Some(vec!["SECRET=value".into()]), + ..Default::default() + }; + let (created, ()) = tokio::join!(port.create_container(&key, &body, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("POST /v1.53/containers/create?name=owned-test ") + ); + let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + assert_eq!(body["Env"], serde_json::json!(["SECRET=value"])); + request + .reply(201, br#"{"Id":"owned-test","Warnings":[]}"#) + .await; + }); + assert_eq!(created.unwrap().id, "owned-test"); + for operation in 0..5 { + let (result, ()) = tokio::join!( + async { + match operation { + 0 => port.start_container(&key, deadline()).await, + 1 => port + .inspect_container(&key, deadline()) + .await + .map(|inspect| assert_eq!(inspect.id.as_deref(), Some("owned-test"))), + 2 => port.kill_container(&key, deadline()).await, + 3 => port + .wait_container(&key, deadline()) + .await + .map(|wait| assert_eq!(wait.status_code, 0)), + _ => port.remove_container(&key, deadline()).await, + } + }, + async { + let request = daemon.next().await; + assert!(request.head.contains("/containers/owned%2Dtest")); + match operation { + 1 => request.reply(200, br#"{"Id":"owned-test"}"#).await, + 3 => request.reply(200, br#"{"StatusCode":0}"#).await, + _ => request.reply(204, b"").await, + } + } + ); + result.unwrap(); + } + let filter = OwnerFilter::new(Uuid::new_v4(), Some(Uuid::new_v4()), None).unwrap(); + let (found, ()) = tokio::join!(port.list_containers(filter, deadline()), async { + let request = daemon.next().await; + assert!(request.head.contains("rustack.lambda.ownerId")); + request.reply(200, b"[]").await; + }); + assert_eq!(found.unwrap(), Vec::::new()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_request_serialization_and_redact_request_debug() { + let body = ContainerCreateBody { + env: Some(vec!["TOP_SECRET=value".into()]), + ..Default::default() + }; + let request = RawRequest { + operation: Operation::Create, + method: Method::POST, + target: "/secret-target".into(), + body: encode(&body).unwrap(), + expected: StatusCode::CREATED, + }; + let shown = format!("{request:?}"); + assert!(!shown.contains("TOP_SECRET") && !shown.contains("secret-target")); + let oversized = ContainerCreateBody { + env: Some(vec!["x".repeat(MAX_BODY_BYTES)]), + ..Default::default() + }; + assert!(matches!(encode(&oversized), Err(DockerError::TooLarge))); + assert!( + !format!( + "{:?}", + Encoded { + bytes: b"secret".to_vec(), + failure: None + } + ) + .contains("secret") + ); +} + +#[tokio::test] +async fn test_should_upgrade_handoff_and_decode_fragmented_binary_output_without_a_management_slot() +{ + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key: DockerKey = "relay-test".parse().unwrap(); + let (attached, mut server) = tokio::join!(port.attach(&key, deadline()), async { + let mut request = daemon.next().await; + assert!(request.head.contains("logs=false")); + request + .stream + .write_all( + b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n", + ) + .await + .unwrap(); + request.stream + }); + let attach = attached.unwrap(); + assert!(!format!("{attach:?}").contains("secret")); + let (mut reader, mut writer) = attach.split(Duration::from_secs(1)).unwrap(); + let mut records = record(1, b"binary\0secret"); + records.extend(record(2, b"stderr secret")); + let (chunks, ()) = tokio::join!( + async { + let first = reader.next(deadline()).await.unwrap().unwrap(); + let second = reader.next(deadline()).await.unwrap().unwrap(); + (first, second) + }, + async { + for byte in records { + server.write_all(&[byte]).await.unwrap(); + } + } + ); + assert_eq!(chunks.0.bytes.as_ref(), b"binary\0secret"); + assert_eq!(chunks.1.channel, OutputChannel::Stderr); + assert!(!format!("{:?} {reader:?}", chunks.0).contains("secret")); + writer.write_all(b"raw stdin").await.unwrap(); + let mut stdin = [0; 9]; + server.read_exact(&mut stdin).await.unwrap(); + assert_eq!(&stdin, b"raw stdin"); + // Exported attach has no management driver left to block shutdown. + wire.shutdown().await.unwrap(); + drop(reader); + drop(writer); + drop(server); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_cleanup_capacity_while_business_calls_and_mailbox_are_full() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let mut calls = JoinSet::new(); + for _ in 0..6 { + let port = port.clone(); + calls.spawn(async move { port.info(deadline()).await }); + } + let mut held = Vec::new(); + for _ in 0..6 { + held.push(daemon.next().await); + } + let mut queued = Vec::new(); + for _ in 0..32 { + let (reply, response) = oneshot::channel(); + port.raw + .business + .try_send(Envelope { + request: RawRequest { + operation: Operation::Info, + method: Method::GET, + target: "/v1.53/info".into(), + body: Bytes::new(), + expected: StatusCode::OK, + }, + deadline: deadline(), + reply, + }) + .unwrap(); + queued.push(response); + } + assert!(matches!( + port.info(deadline()).await, + Err(DockerError::Busy) + )); + let key: DockerKey = "owned-container".parse().unwrap(); + let (removed, ()) = tokio::join!(port.remove_container(&key, deadline()), async { + let request = daemon.next().await; + assert!( + request + .head + .starts_with("DELETE /v1.53/containers/owned%2Dcontainer?") + ); + request.reply(204, b"").await; + }); + removed.unwrap(); + wire.shutdown().await.unwrap(); + for response in queued { + assert!(matches!(response.await.unwrap(), Err(DockerError::Stopped))); + } + while let Some(result) = calls.join_next().await { + assert!(result.unwrap().is_err()); + } + for mut request in held { + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + } + daemon.close().await; +} + +#[tokio::test] +async fn test_should_admit_kill_while_two_lifecycle_waits_are_blocked() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let mut calls = JoinSet::new(); + let key: DockerKey = "running-container".parse().unwrap(); + for _ in 0..2 { + let port = port.clone(); + let key = key.clone(); + calls.spawn(async move { port.wait_container(&key, deadline()).await }); + } + let mut held = Vec::new(); + for _ in 0..2 { + let request = daemon.next().await; + assert!(request.head.contains("/wait?condition=not-running")); + held.push(request); + } + let (killed, ()) = tokio::join!( + port.kill_container(&key, Instant::now() + Duration::from_secs(1)), + async { + let request = daemon.next().await; + assert!(request.head.contains("/kill?signal=SIGKILL")); + request.reply(204, b"").await; + } + ); + killed.unwrap(); + wire.shutdown().await.unwrap(); + while let Some(result) = calls.join_next().await { + assert!(result.unwrap().is_err()); + } + for mut request in held { + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + } + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_inflight_calls_after_waiter_drop_and_join_after_cancelled_shutdown() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let caller = tokio::spawn(async move { port.info(deadline()).await }); + let request = daemon.next().await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + // The accepted request still owns its driver even though its response + // waiter is gone. Finish the body before stopping the retained wire owner. + request.reply(200, b"{}").await; + let mut shutdown = Box::pin(wire.shutdown()); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(shutdown); + wire.shutdown().await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_filter_secret_requests_responses_and_partial_trailers_under_trace() { + let capture = LogWriter(std::sync::Arc::default()); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + async { + tracing::debug!("application_event_is_preserved"); + let dispatch = runtime_dispatch(); + tracing::dispatcher::with_default(&dispatch, || { + tracing::trace!(target: "bollard::docker", "secret_constructed_bollard"); + }); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let key = "secret-request-target".parse().unwrap(); + let body = ContainerCreateBody { env: Some(vec!["SECRET_ENV=secret_request_body".into()]), ..Default::default() }; + let (created, ()) = tokio::join!(port.create_container(&key, &body, deadline()), async { + let request = daemon.next().await; + request.reply(201, br#"{"Id":"known","Warnings":["secret_response_body"]}"#).await; + }); + assert_eq!(created.unwrap().id, "known"); + let (result, ()) = tokio::join!(port.info(deadline()), async { + let request = daemon.next().await; + let mut stream = request.stream; + stream.write_all(b"HTTP/1.1 200 OK\r\nSecret: secret_response_header\r\nTransfer-Encoding: chunked\r\n\r\n2\r\n{}\r\n0\r\nSecret: secret_partial_trailer").await.unwrap(); + // Force at least one partial trailer-buffer parse before finishing. + tokio::task::yield_now().await; + stream.write_all(b"\r\n\r\n").await.unwrap(); + }); + assert!(result.is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; + }.with_subscriber(subscriber).await; + let shown = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); + assert!(shown.contains("application_event_is_preserved")); + for secret in [ + "secret_constructed_bollard", + "secret-request-target", + "secret_request_body", + "secret_response_body", + "secret_response_header", + "secret_partial_trailer", + ] { + assert!( + !shown.contains(secret), + "transport secret escaped the filter" + ); + } +} + +#[tokio::test] +async fn test_should_stream_large_records_in_bounded_chunks_and_reject_bad_outer_framing() { + let data = record(1, &vec![7; 100_000]); + let mut reader = DockerOutputReader::new(data.as_slice(), Duration::from_secs(1)).unwrap(); + let mut total = 0; + while let Some(chunk) = reader.next(deadline()).await.unwrap() { + assert!(chunk.bytes.len() <= 16 * 1024); + total += chunk.bytes.len(); + } + assert_eq!(total, 100_000); + assert!(reader.next(deadline()).await.unwrap().is_none()); + for bytes in [ + vec![0, 0, 0, 0, 0, 0, 0, 0], + vec![3, 0, 0, 0, 0, 0, 0, 0], + vec![1, 1, 0, 0, 0, 0, 0, 0], + vec![1, 0], + vec![1, 0, 0, 0, 0xff, 0xff, 0xff, 0xff], + vec![1, 0, 0, 0, 0, 0, 0, 1], + ] { + let mut reader = DockerOutputReader::new(bytes.as_slice(), Duration::from_secs(1)).unwrap(); + assert!(reader.next(deadline()).await.is_err()); + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Poisoned) + )); + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_not_extend_partial_record_deadlines_across_chunks_or_restart_cancelled_reads() +{ + let data = record(1, &vec![9; 32 * 1024]); + let mut reader = DockerOutputReader::new(data.as_slice(), Duration::from_secs(1)).unwrap(); + assert_eq!( + reader.next(deadline()).await.unwrap().unwrap().bytes.len(), + 16 * 1024 + ); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Deadline) + )); + let (mut input, source) = duplex(8); + input.write_all(&[1]).await.unwrap(); + let mut reader = DockerOutputReader::new(source, Duration::from_secs(1)).unwrap(); + let mut waiting = Box::pin(reader.next(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(matches!( + reader.next(deadline()).await, + Err(DockerError::Poisoned) + )); +} + +#[test] +fn test_should_validate_explicit_sockets_identifiers_and_version_negotiation() { + for socket in [ + "/var/run/docker.sock", + "tcp://localhost:2375", + "unix://relative", + "unix:///bad\0socket", + ] { + assert!(socket.parse::().is_err()); + } + for key in ["", "../bad", "path/name", "-bad", "space name"] { + assert!(key.parse::().is_err()); + } + assert!("a".repeat(129).parse::().is_err()); + assert_eq!(ApiVersion::negotiate("1.99", "1.44").unwrap().minor(), 53); + assert!(ApiVersion::negotiate("1.43", "1.24").is_err()); + assert!(ApiVersion::negotiate("1.99", "1.54").is_err()); + for version in ["1.053", "2.53", "1.-1", "1.1000"] { + assert!(version.parse::().is_err()); + } + assert!(OwnerFilter::new(Uuid::nil(), None, None).is_err()); + assert!(OwnerFilter::new(Uuid::new_v4(), None, Some(Uuid::new_v4())).is_err()); +} + +#[tokio::test] +async fn test_should_close_a_timed_out_body_driver_without_stopping_the_wire() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!(port.info(Instant::now() + Duration::from_secs(1)), async { + let mut request = daemon.next().await; + request + .stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n") + .await + .unwrap(); + let mut byte = [0; 1]; + assert_eq!( + timeout_at(deadline(), request.stream.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + assert!(matches!(result, Err(DockerError::Deadline))); + let (next, ()) = tokio::join!(port.info(deadline()), async { + daemon.next().await.reply(200, b"{}").await; + }); + next.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_cache_supervision_failure_when_priority_stop_wins() { + let (_business, requests) = mpsc::channel(32); + let (_cleanup, priority) = mpsc::channel(8); + let (_waits, waits) = mpsc::channel(8); + let (control, _) = watch::channel(true); + let mut actor = Actor::builder() + .socket(PathBuf::from("/unused-fixture-socket")) + .business(requests) + .cleanup(priority) + .waits(waits) + .control(control.clone()) + .build(); + let failed = actor + .jobs + .spawn(async { panic!("controlled wire worker failure") }); + while !failed.is_finished() { + tokio::task::yield_now().await; + } + let task = tokio::spawn(actor.run()); + let mut owner = Owner { + stop: control, + task: Some(task), + settlement: None, + }; + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); + assert!(matches!(owner.shutdown().await, Err(DockerError::Worker))); +} + +#[tokio::test] +async fn test_should_reject_expired_and_excessive_deadlines_before_metadata_publication() { + assert!(matches!( + check_deadline(Instant::now()), + Err(DockerError::Deadline) + )); + assert!(matches!( + check_deadline(Instant::now() + Duration::from_secs(601)), + Err(DockerError::Configuration) + )); + let response = RawReply::Body { + status: StatusCode::OK, + bytes: Bytes::from_static(b"{}"), + }; + let decoded: Result = decode(response, Instant::now()); + assert!(matches!(decoded, Err(DockerError::Deadline))); +} diff --git a/crates/rustack-lambda-core/src/docker/types.rs b/crates/rustack-lambda-core/src/docker/types.rs new file mode 100644 index 00000000..4ada90d0 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/types.rs @@ -0,0 +1,205 @@ +//! Validated local Docker identities, versions and redacted errors. + +use std::{fmt, io, os::unix::fs::FileTypeExt, path::PathBuf, str::FromStr, sync::Arc}; + +use http::StatusCode; +use rustack_core::json::JsonError; +use tokio::time::{Instant, timeout_at}; + +use super::check_deadline; + +/// Docker boundary failure classes; no peer response body is displayed. +#[derive(Clone, thiserror::Error)] +#[non_exhaustive] +pub enum DockerError { + /// Invalid local configuration or identifier. + #[error("invalid Docker configuration")] + Configuration, + /// The local daemon's API range is incompatible. + #[error("unsupported Docker API version")] + Version, + /// A declared/actual byte or collection allowance was exceeded. + #[error("Docker byte budget exceeded")] + TooLarge, + /// A framing/header/response is malformed. + #[error("invalid Docker protocol")] + Protocol, + /// A cancelled partial stream cannot be restarted. + #[error("Docker stream is invalidated")] + Poisoned, + /// Absolute IO deadline expired. + #[error("Docker deadline exceeded")] + Deadline, + /// Wire service has stopped admission. + #[error("Docker wire is stopped")] + Stopped, + /// A bounded mailbox has no capacity. + #[error("Docker wire is busy")] + Busy, + /// An owned task panicked or failed to join. + #[error("Docker supervision failed")] + Worker, + /// A bounded output allocation could not be reserved. + #[error("Docker allocation failed")] + Allocation, + /// Request/model encoding failed without exposing input text. + #[error("invalid Docker request schema")] + Schema, + /// Non-success HTTP response; daemon error text is intentionally excluded. + #[error("Docker response status {0}")] + Status(StatusCode), + /// Local socket/stream IO failed. + #[error("Docker IO failed")] + Io(#[source] Arc), + /// HTTP transport failed. + #[error("Docker HTTP failed")] + Http(#[source] Arc), + /// Bounded metadata validation failed. + #[error(transparent)] + Json(#[from] JsonError), +} + +impl fmt::Debug for DockerError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} +impl From for DockerError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} +impl From for DockerError { + fn from(error: hyper::Error) -> Self { + Self::Http(Arc::new(error)) + } +} + +/// Explicit local Unix socket, independent of ambient Docker environment. +#[derive(Clone)] +#[non_exhaustive] +pub struct DockerSocket(PathBuf); + +impl fmt::Debug for DockerSocket { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerSocket()") + } +} +impl FromStr for DockerSocket { + type Err = DockerError; + fn from_str(value: &str) -> Result { + let path = value + .strip_prefix("unix://") + .ok_or(DockerError::Configuration)?; + if !path.starts_with('/') || path.len() > 4096 || path.chars().any(char::is_control) { + return Err(DockerError::Configuration); + } + Ok(Self(PathBuf::from(path))) + } +} +impl DockerSocket { + pub(super) async fn verified(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + let path = timeout_at(deadline, tokio::fs::canonicalize(&self.0)) + .await + .map_err(|_| DockerError::Deadline)??; + let metadata = timeout_at(deadline, tokio::fs::metadata(&path)) + .await + .map_err(|_| DockerError::Deadline)??; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if !metadata.file_type().is_socket() { + return Err(DockerError::Configuration); + } + Ok(path) + } +} + +/// Validated Docker object name or content identifier used as a path component. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct DockerKey(String); + +impl fmt::Debug for DockerKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DockerKey()") + } +} +impl FromStr for DockerKey { + type Err = DockerError; + fn from_str(value: &str) -> Result { + if value.len() > 128 + || !value + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_alphanumeric()) + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) + { + return Err(DockerError::Configuration); + } + Ok(Self(value.to_owned())) + } +} +impl DockerKey { + /// Borrow the validated spelling; it is still encoded when placed in a URI. + /// + /// ``` + /// let key: rustack_lambda_core::docker::DockerKey = "rustack-relay-123".parse()?; + /// assert_eq!(key.as_str(), "rustack-relay-123"); + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// A bounded Docker API version; negotiations select the local1.44..1.53 range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub struct ApiVersion(u16); + +impl FromStr for ApiVersion { + type Err = DockerError; + fn from_str(value: &str) -> Result { + let minor = value.strip_prefix("1.").ok_or(DockerError::Version)?; + if minor.is_empty() || minor.len() > 3 || !minor.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(DockerError::Version); + } + let parsed = minor.parse::().map_err(|_| DockerError::Version)?; + if parsed.to_string() != minor { + return Err(DockerError::Version); + } + Ok(Self(parsed)) + } +} +impl ApiVersion { + pub(super) fn negotiate(maximum: &str, minimum: &str) -> Result { + let selected = maximum.parse::()?.min(Self(53)); + let minimum: Self = minimum.parse()?; + if selected < Self(44) || selected < minimum { + return Err(DockerError::Version); + } + Ok(selected) + } + + /// Numeric API minor version, without its major version or URI prefix. + /// + /// ``` + /// let version: rustack_lambda_core::docker::ApiVersion = "1.53".parse()?; + /// assert_eq!(version.minor(), 53); + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + #[must_use] + pub fn minor(self) -> u16 { + self.0 + } +} +impl fmt::Display for ApiVersion { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "1.{}", self.0) + } +} diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 84169357..4f51b405 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -13,6 +13,8 @@ pub mod artifact; pub mod code; pub mod config; +#[cfg(unix)] +pub mod docker; pub mod error; pub mod executor; mod function_config; diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs new file mode 100644 index 00000000..f260183c --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -0,0 +1,381 @@ +//! Explicit local Docker qualification of production HTTP/upgrade/control IO. + +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{collections::HashMap, env, time::Duration}; + +use anyhow::{Context, Result, bail, ensure}; +use bytes::Bytes; +use http::StatusCode; +use rustack_lambda_core::{ + artifact::{ImageReference, Sha256Digest}, + docker::{ + DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, OutputChannel, OwnerFilter, + models::{ContainerCreateBody, HostConfig}, + }, +}; +use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + metadata::local_trace_id, + protocol::{Architecture, AttemptId, EnvironmentId, Payload}, + session::{Invocation, RuntimeResult}, +}; +use serde_json::{Value, json}; +use tokio::{ + io::AsyncWriteExt, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use uuid::Uuid; + +const IO: Duration = Duration::from_secs(60); +const LARGE: usize = 6 * 1024 * 1024 - 4096; +fn deadline() -> Instant { + Instant::now() + IO +} + +#[derive(Debug)] +struct Names { + relay: DockerKey, + function: DockerKey, + owner: Uuid, + run: Uuid, + environment: EnvironmentId, +} +impl Names { + fn labels(&self) -> HashMap { + HashMap::from([ + ("rustack.lambda.ownerId".into(), self.owner.to_string()), + ("rustack.lambda.runId".into(), self.run.to_string()), + ( + "rustack.lambda.environmentId".into(), + self.environment.to_string(), + ), + ( + "rustack.lambda.functionResourceId".into(), + self.environment.to_string(), + ), + ]) + } +} + +#[derive(Debug)] +struct Runtime { + client: ControlClient, + stop: watch::Sender, + pump: Option>>, + settlement: Option>, +} +impl Runtime { + async fn shutdown(&mut self) -> Result<()> { + let protocol = self + .client + .shutdown(deadline()) + .await + .map_err(anyhow::Error::from); + self.stop.send_replace(true); + let pump = if let Some(result) = &self.settlement { + result.clone() + } else { + let result = if let Some(pump) = &mut self.pump { + pump.await + .map_err(|_| DockerError::Worker) + .and_then(|result| result) + } else { + Ok(()) + }; + self.pump.take(); + self.settlement = Some(result.clone()); + result + }; + protocol.and(pump.map_err(anyhow::Error::from)) + } +} +impl Drop for Runtime { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[derive(Debug)] +struct Environment { + wire: DockerWire, + names: Names, + runtime: Option, +} +impl Environment { + async fn new() -> Result { + let socket = env::var("RUSTACK_RUNTIME_DOCKER_SOCKET") + .context("explicit test Docker socket required")?; + let socket: DockerSocket = format!("unix://{socket}").parse()?; + let wire = DockerWire::connect(socket, deadline()).await?; + let run = Uuid::new_v4(); + Ok(Self { + wire, + names: Names { + relay: format!("rustack-wire-relay-{run}").parse()?, + function: format!("rustack-wire-function-{run}").parse()?, + owner: Uuid::new_v4(), + run, + environment: EnvironmentId::default(), + }, + runtime: None, + }) + } + + async fn image(port: &DockerPort, reference: &str, architecture: &str) -> Result { + let reference: ImageReference = reference.parse()?; + let image = port.inspect_image(&reference, deadline()).await?; + ensure!( + image.os.as_deref() == Some("linux") + && image.architecture.as_deref() == Some(architecture), + "fixture image platform mismatch" + ); + let id = image.id.context("image content id")?; + let _: Sha256Digest = id.parse()?; + Ok(id) + } + + async fn start(&mut self) -> Result<()> { + let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; + let (architecture, daemon_arch) = match arch.as_str() { + "arm64" => (Architecture::Arm64, "aarch64"), + "amd64" => (Architecture::X86_64, "x86_64"), + _ => bail!("architecture must be arm64 or amd64"), + }; + let relay = env::var("RUSTACK_RUNTIME_IMAGE").context("relay fixture image required")?; + let port = self.wire.port(); + let info = port.info(deadline()).await?; + ensure!( + info.os_type.as_deref() == Some("linux") + && info.architecture.as_deref() == Some(daemon_arch), + "daemon platform mismatch" + ); + let relay = Self::image(&port, &relay, &arch).await?; + let function = Self::image(&port, &format!("rustack-p0-handler:{arch}"), &arch).await?; + port.create_container( + &self.names.relay, + &ContainerCreateBody { + image: Some(relay), + user: Some("1000:1000".into()), + env: Some(vec![ + format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={}", self.names.environment), + "RUSTACK_LAMBDA_RELAY_IO_SECONDS=60".into(), + "RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS=60".into(), + ]), + labels: Some(self.names.labels()), + attach_stdin: Some(true), + attach_stdout: Some(true), + attach_stderr: Some(true), + open_stdin: Some(true), + stdin_once: Some(true), + tty: Some(false), + host_config: Some(limits("none".into(), 64 * 1024 * 1024, 100_000_000, 64)), + ..Default::default() + }, + deadline(), + ) + .await?; + self.attach(&port, architecture).await?; + port.start_container(&self.names.relay, deadline()).await?; + let client = &mut self.runtime.as_mut().context("runtime owner")?.client; + ensure!( + client.next(deadline()).await? == ClientEvent::RelayReady, + "relay handshake mismatch" + ); + port.create_container( + &self.names.function, + &ContainerCreateBody { + image: Some(function), + user: Some("1000:1000".into()), + labels: Some(self.names.labels()), + env: Some(vec![ + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001".into(), + "AWS_LAMBDA_FUNCTION_NAME=wire-fixture".into(), + "AWS_LAMBDA_FUNCTION_VERSION=$LATEST".into(), + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192".into(), + "AWS_REGION=us-east-1".into(), + "AWS_DEFAULT_REGION=us-east-1".into(), + ]), + host_config: Some(limits( + format!("container:{}", self.names.relay.as_str()), + 192 * 1024 * 1024, + 1_000_000_000, + 256, + )), + ..Default::default() + }, + deadline(), + ) + .await?; + port.start_container(&self.names.function, deadline()) + .await?; + ensure!( + client.next(deadline()).await? == ClientEvent::BootstrapReady, + "function readiness mismatch" + ); + Ok(()) + } + + async fn attach(&mut self, port: &DockerPort, architecture: Architecture) -> Result<()> { + let attached = port.attach(&self.names.relay, deadline()).await?; + let (mut output, input) = attached.split(IO)?; + let (mut bridge, reader) = tokio::io::duplex(64 * 1024); + let client = ControlClient::start( + ClientConfig::new(self.names.environment, architecture, IO)?, + reader, + input, + )?; + let (stop, mut stopped) = watch::channel(false); + let pump = tokio::spawn(async move { + loop { + let chunk = tokio::select! { + biased; + () = async { let _ = stopped.wait_for(|stop| *stop).await; } => return Ok(()), + chunk = output.next(Instant::now() + Duration::from_secs(600)) => chunk?, + }; + match chunk { + Some(chunk) if chunk.channel == OutputChannel::Stdout => { + timeout_at(deadline(), bridge.write_all(&chunk.bytes)) + .await + .map_err(|_| DockerError::Deadline)??; + } + Some(_) => {} // Relay diagnostics are deliberately not copied into logs. + None => return Ok(()), + } + } + }); + self.runtime = Some(Runtime { + client, + stop, + pump: Some(pump), + settlement: None, + }); + Ok(()) + } + + async fn call(&mut self, event: &Value, failed: bool) -> Result { + let call = Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(120), + "arn:aws:lambda:us-east-1:000000000000:function:wire-fixture:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?; + let id = call.parts().0; + let client = &mut self.runtime.as_mut().context("runtime owner")?.client; + client.invoke(call, deadline()).await?; + match client + .next(Instant::now() + Duration::from_secs(120)) + .await? + { + ClientEvent::InvocationResult { attempt, result } if attempt == id => match result { + RuntimeResult::Success(payload) if !failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + RuntimeResult::FunctionError(payload) if failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + _ => bail!("unexpected function outcome"), + }, + _ => bail!("unexpected control outcome"), + } + } + + async fn cleanup(&mut self) -> Result<()> { + let port = self.wire.port(); + let function = remove(&port, &self.names.function).await; + let runtime = if let Some(runtime) = &mut self.runtime { + runtime.shutdown().await + } else { + Ok(()) + }; + let relay = remove(&port, &self.names.relay).await; + let remaining = port + .list_containers( + OwnerFilter::new(self.names.owner, Some(self.names.run), None)?, + deadline(), + ) + .await; + let listed = remaining.map_err(anyhow::Error::from).and_then(|items| { + ensure!( + items.is_empty(), + "owned containers remain after wire cleanup" + ); + Ok(()) + }); + let joined = self.wire.shutdown().await.map_err(anyhow::Error::from); + function.and(runtime).and(relay).and(listed).and(joined) + } +} + +fn limits(network_mode: String, memory: i64, nano_cpus: i64, pids_limit: i64) -> HostConfig { + HostConfig { + network_mode: Some(network_mode), + memory: Some(memory), + nano_cpus: Some(nano_cpus), + pids_limit: Some(pids_limit), + readonly_rootfs: Some(true), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + tmpfs: Some(HashMap::from([( + "/tmp".into(), + "rw,noexec,nosuid,size=536870912".into(), + )])), + ..Default::default() + } +} + +async fn remove(port: &DockerPort, key: &DockerKey) -> Result<()> { + match port.remove_container(key, deadline()).await { + Ok(()) | Err(DockerError::Status(StatusCode::NOT_FOUND)) => Ok(()), + Err(error) => Err(error.into()), + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker wire qualification images/socket"] +async fn test_should_execute_real_ric_over_production_wire_with_large_binary_frames_and_cleanup() +-> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + environment.start().await?; + let event = json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE}); + let computed = environment.call(&event, false).await?; + ensure!( + computed["computed"] == 42 + && computed["counter"] == 1 + && computed["inputBytes"] == LARGE, + "computed/large input mismatch" + ); + ensure!( + computed["large"].as_str().is_some_and( + |value| value.len() == LARGE && value.bytes().all(|byte| byte == b'r') + ), + "large output mismatch" + ); + ensure!( + computed["uid"] == 1000 + && computed["rootReadOnly"] == true + && computed["memoryMax"] == "201326592" + && computed["pidsMax"] == "256", + "function resource controls differ" + ); + let warm = environment.call(&json!({"value":0}), false).await?; + ensure!( + warm["computed"] == 7 && warm["counter"] == 2, + "warm reuse mismatch" + ); + let failure = environment.call(&json!({"fail":true}), true).await?; + ensure!( + failure["errorType"] == "ValueError", + "function error mismatch" + ); + Ok(()) + } + .await; + let cleanup = environment.cleanup().await; + result.and(cleanup) +} diff --git a/crates/rustack-lambda-runtime/src/diagnostics.rs b/crates/rustack-lambda-runtime/src/diagnostics.rs index 32828159..c5e28489 100644 --- a/crates/rustack-lambda-runtime/src/diagnostics.rs +++ b/crates/rustack-lambda-runtime/src/diagnostics.rs @@ -15,7 +15,7 @@ struct RuntimeSubscriber { fn allowed(metadata: &Metadata<'_>) -> bool { let target = metadata.target(); - let dependency = ["hyper", "hyper_util"].iter().any(|name| { + let dependency = ["hyper", "hyper_util", "bollard"].iter().any(|name| { target == *name || target .strip_prefix(name) @@ -67,7 +67,17 @@ impl Subscriber for RuntimeSubscriber { } } -pub(crate) fn runtime_dispatch() -> Dispatch { +/// Capture application context while denying verbose transport dependency logs. +/// +/// The policy covers Hyper, `hyper_util` and Bollard independently of `RUST_LOG`. +/// Apply the dispatch to construction and every poll/spawned transport worker. +/// +/// ``` +/// let dispatch = rustack_lambda_runtime::diagnostics::runtime_dispatch(); +/// tracing::dispatcher::with_default(&dispatch, || tracing::debug!("application event")); +/// ``` +#[must_use] +pub fn runtime_dispatch() -> Dispatch { dispatcher::get_default(|parent| { Dispatch::new(RuntimeSubscriber { parent: parent.clone(), diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index d89bb144..3a097ecf 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -7,7 +7,7 @@ #![forbid(unsafe_code)] pub mod client; -mod diagnostics; +pub mod diagnostics; pub mod http; pub mod metadata; pub mod protocol; diff --git a/docs/index.md b/docs/index.md index d3d128da..35becd5c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ ## Research -- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse and the bounded production transport contract. +- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation and actual RIC/diagnostic qualification. - [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. - [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. - [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. diff --git a/docs/research/study-bollard-docker-transport-boundaries.md b/docs/research/study-bollard-docker-transport-boundaries.md index 3f943cb4..0ec735d5 100644 --- a/docs/research/study-bollard-docker-transport-boundaries.md +++ b/docs/research/study-bollard-docker-transport-boundaries.md @@ -1,6 +1,6 @@ # Study: Bollard's Docker Response and Attach Boundaries -Status: source review complete; no new runtime experiment · Date:2026-10-25 · Vendor pin: `vendors/bollard` @ `f9ec79e7546edfb674b1066c44a68815cb52251c` (`v0.21.1`) +Status: pinned source review complete; bounded wire implementation and qualification recorded below · Date:2026-10-25 · Vendor pin: `vendors/bollard` @ `f9ec79e7546edfb674b1066c44a68815cb52251c` (`v0.21.1`) ## Question @@ -71,3 +71,13 @@ Use the generated model/query contracts, with a small bounded Unix HTTP/upgrade The earlier production suggestion to use Bollard's ordinary pipe client is superseded by D19; the retained P0 driver remains valid as a feasibility experiment. This is an in-phase transport refinement, not a request to rewrite the Docker API or support another container platform. Required implementation evidence remains bounded fake-daemon tests, actual attach/large-payload/truncation checks, TRACE capture and the established arm64/amd64 RIC lanes. No unresolved source assumption authorizes bypassing those tests. A source review is not proof that the forthcoming bounded adapter works. + +## Implementation follow-through + +P1 now reuses only generated bollard-stubs models in production. DockerWire owns bounded version/info/image inspection, typed container create/start/inspect/kill/wait/remove/list and strict upgraded attach. Its actor retains request jobs after caller cancellation; ordinary HTTP drivers are aborted/joined or recover/drop their one-shot IO, and upgraded drivers join before socket transfer. A separate32-job wait lane prevents long-running lifecycle waits from starving the two destructive-cleanup slots. Deployment actors, log/pull/network/probe consumers, pool policy and provider integration remain required; this is not executor acceptance. + +The current workspace and standalone probe align on [Tokio1.54.1](https://docs.rs/crate/tokio/1.54.1) and [Hyper1.12.0](https://docs.rs/crate/hyper/1.12.0), rechecked against official package documentation on2026-10-25. The original Hyper1.11.1 audit remains historical evidence; mandatory task-bound filtering and fragmented trailer tests remain in effect on the updated stack. + +A retained compatibility regression calls the actual pinned Bollard0.21.1 create and logs APIs against a fake local Unix daemon under TRACE with LogTracer enabled. It covers synchronous stream construction, asynchronous request/response polls and the decoder's truncated-EOF Console fallback. Test-only markers in request environment, response warnings, container target and trailing bytes are absent from infrastructure capture while an application event remains present. The production boundary independently captures create requests, response bytes/headers and fragmented trailers. These are executed call-site tests, not only synthetic logging-target tests. + +The explicit `make test-lambda-docker-wire` target builds the real relay and unchanged official Python RIC image and uses DockerWire for daemon IO without the Docker CLI. It sends and receives6MiB-minus4096-byte-class data through actual multiplexed attach, checks computed warm results and a handler error, closes/joins the relay stream and verifies the run-label scope is empty after removal. Stress qualification uses60s IO/120s attempt budgets; it does not establish default5s latency or throughput. The existing five-test `test-lambda-runtime-docker` suite remains a separate relay-contract regression. Exact platform results and current review state are recorded in [verification](../../specs/rustack-workflows-70-verification-plan.md) and [implementation review](../../specs/rustack-workflows-93-review.md). diff --git a/specs/index.md b/specs/index.md index e94058f0..8c84b3af 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Production Docker execution and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO is also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index ab2c7193..0e4d7d49 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -217,7 +217,7 @@ The runtime library's HTTP adapter binds only an explicitly supplied loopback So HTTP parsing limits are compile-time constants:16 KiB header buffer/aggregate header bytes,64 headers,256 bytes per value and256 bytes per request target. Reject absolute-form targets, query strings, duplicate Invocation-Id headers and nonprintable header values. Routes match the entire method/path and canonical attempt UUID; extra path segments do not reach the session. GET bodies must be empty. POST bodies are raw bounded bytes≤6 MiB, collected with a five-second absolute deadline shared with command processing. Reject all HTTP trailers before session mutation; streaming responses are outside this profile. Content-Length over the limit rejects before reading the body. Header parsing has its own five-second deadline. GET /next waits within the configured session idle lease instead of a five-second business timeout. Responses to rejected requests close the connection; normal RIC keepalive remains supported. -Wrap the HTTP supervisor and every spawned connection future with an independent task-bound subscriber filter that suppresses Hyper/hyper_util DEBUG/TRACE regardless of the application's subscriber/RUST_LOG. Preserve allowed application span IDs/context and events through delegation. Pinned Hyper1.11.1 logs its partial chunk-trailer buffer at TRACE before application validation, so rejecting trailers alone cannot establish secrecy. Capture a fragmented secret trailer under TRACE as well as ordinary header/body/target values. This shared adapter filter is separate from the mandatory Bollard filter at the future Docker boundary. +Wrap the HTTP supervisor and every spawned connection future with an independent task-bound subscriber filter that suppresses Hyper/hyper_util DEBUG/TRACE regardless of the application's subscriber/RUST_LOG. Preserve allowed application span IDs/context and events through delegation. The initial Hyper1.11.1 source audit found partial chunk-trailer buffers in TRACE before application validation, so rejecting trailers alone cannot establish secrecy. The current Hyper1.12.0/Tokio1.54.1 stack retains this mandatory filter and the captured-secret regression. Capture a fragmented secret trailer under TRACE as well as ordinary header/body/target values. This shared adapter filter is separate from the mandatory Bollard filter at the future Docker boundary. GET responses include Request-Id, Invocation-Id (the same synthetic attempt UUID), checked epoch-millisecond Deadline-Ms, requested Function-Arn and a syntactically valid local Trace-Id. This local profile generates a fresh request/attempt UUID for every Invoke rather than implementing event-source event identity across redelivery. If Invocation-Id is present on completion it must match the path and delivered attempt; omission stays compatible with older RICs. Mismatch uses400/InvalidInvocationId without echoing values, Busy maps409, unknown attempts400, expired requests408 and closed sessions503. Successful completion/init error returns202. Infrastructure traces contain bounded classes/counts only, not targets, headers or payloads. This follows the [current AWS Runtime API contract](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html). @@ -233,7 +233,7 @@ Separate first-byte waiting from partial-frame IO: silence is bounded by the lar The `rustack-lambda-runtime` executable is Linux-only and serves127.0.0.1:9001. Required private child environment is RUSTACK_LAMBDA_ENVIRONMENT_ID; optional RUSTACK_LAMBDA_RELAY_IDLE_SECONDS / IO_SECONDS / RETIRE_SECONDS forward the owner's validated configuration without adding a new user configuration loader. Reject invalid/noncanonical values without echoing them. Advertise the binary's compiled x86_64/arm64 architecture. Stdout contains control frames only; diagnostic errors use bounded static tracing classes on stderr. -Use safe OwnedFd clones of stdin/stdout with Tokio Unix pipe Receiver/Sender::from_owned_fd inside the entered IO runtime. The pinned Tokio1.53.1 constructors validate pipe/access mode and enable nonblocking IO. Do not use tokio::io::stdin or a blocking read task: cancelling its future does not stop the blocking read and can keep runtime shutdown waiting after a lease expires. Reject nonpipe descriptors/unsupported platforms explicitly. Unit tests use owned cancel-safe duplex streams; process tests exercise pipe EOF/Close and actual Linux builds precede Docker qualification. +Use safe OwnedFd clones of stdin/stdout with Tokio Unix pipe Receiver/Sender::from_owned_fd inside the entered IO runtime. The current Tokio1.54.1 constructors validate pipe/access mode and enable nonblocking IO. Do not use tokio::io::stdin or a blocking read task: cancelling its future does not stop the blocking read and can keep runtime shutdown waiting after a lease expires. Reject nonpipe descriptors/unsupported platforms explicitly. Unit tests use owned cancel-safe duplex streams; process tests exercise pipe EOF/Close and actual Linux builds precede Docker qualification. The optional local `test-lambda-runtime-docker` target builds the relay image and explicitly runs the resource-gated process suite against the selected local Unix socket. Missing Docker/images fail this explicit target; ordinary workspace tests do not pretend to provide Docker evidence. The test-only Docker CLI supplies non-TTY binary stdin/stdout; the production executor still requires the separately supervised Docker attach client. Preallocate unique container names and a run label before allocation so setup failures retain reconciliation ownership. Always remove both owned names, reap the CLI process and verify no run-labeled containers remain, even when the assertions fail. Exercise Hello/Ready/Close before a function exists, control EOF/truncation, and an unchanged official RIC image's computed result, warm counter, handler error and busy cancellation. These relay checks do not establish SDK/provider, deployment/pool or full M0 acceptance. @@ -273,9 +273,25 @@ Tests pair real relay/shared HTTP with the host client over fragmented duplex st The pinned [Bollard transport study](../docs/research/study-bollard-docker-transport-boundaries.md) supersedes the initial ordinary-pipe-client recommendation. Reuse its independently published generated `bollard-stubs` models/query types (`1.53.1-rc.29.3.1`, default features disabled); own the small Unix HTTP/upgrade adapter for the operations this feature needs. Ordinary Bollard response collection and its permissive log decoder do not enforce these local pre-materialization limits. This changes the transport implementation, not the Docker API or runtime protocol. No production shell/CLI transport, TCP/SSH fallback or TLS feature is introduced. -DockerSocket is a validated operator-owned unix:// absolute path≤4,096 bytes, without controls/NUL. Resolve/check its Unix-socket metadata under an IO deadline; never consult ambient DOCKER_HOST or discover another socket. DockerWire owns bounded business/cleanup mailboxes of32/8 commands and at most eight management jobs: six ordinary slots and two reserved cleanup slots. Kill/wait/remove/reconciliation use the priority cleanup lane; owner stop has its own signal. Ordinary deployment/start work cannot consume the reserved cleanup slots. Cloneable ports submit typed request envelopes with an absolute deadline; Drop of a response waiter never authorizes replay or loses connection-task ownership. The environment/deployment actor retains names/labels and side-effect reconciliation above the wire layer. +DockerSocket is a validated operator-owned unix:// absolute path≤4,096 bytes, without controls/NUL. Resolve/check its Unix-socket metadata under an IO deadline; never consult ambient DOCKER_HOST or discover another socket. DockerWire owns bounded business/cleanup mailboxes of32/8 commands and at most eight short management jobs: six ordinary slots and two reserved cleanup slots. Kill/remove/inspection/reconciliation use the priority cleanup lane; owner stop has its own signal. Blocking lifecycle waits use a separate eight-command mailbox and at most32 retained wait jobs, bounded to one per allocated environment by the environment owner. They cannot occupy management or kill/remove reservations: two waits on running containers must never prevent the kills needed to release them. Ordinary deployment/start work cannot consume the reserved cleanup slots. Cloneable ports submit typed request envelopes with an absolute deadline; Drop of a response waiter never authorizes replay or loses connection-task ownership. The environment/deployment actor retains names/labels and side-effect reconciliation above the wire layer. -Each management request owns a fresh Hyper HTTP/1 connection to that socket, with no keepalive pool. One retained job owns connect, handshake, request, bounded response consumption and connection-driver termination through join. Use a single absolute request deadline and recheck the clock after awaits. Bound socket connect, HTTP header parsing, upgrade negotiation and body/frame IO; do not leave a detached Hyper driver or count response headers as complete body success. Cancellation/expiry closes that request's IO and joins its driver. Caller cancellation does not implicitly cancel/retry the service-side resource operation; preallocated identities and reconciliation handle uncertain create results. +```text +┌─ typed DockerPort ────────────────────────────────────────────────────┐ +│ business mailbox32 ──▶ six short management jobs ──┐ │ +│ cleanup mailbox8 ──▶ two reserved cleanup jobs ──┼─▶ owned Unix HTTP │ +│ wait mailbox8 ──▶≤32 lifecycle wait jobs ─────┘ drivers │ +│ priority owner stop ─▶ close admission + signal and join all jobs │ +└───────────────────────────────┬───────────────────────────────────────┘ + │ supervised handoff (release short slot) +┌─ environment / deployment owners ────────────────────────────────────┐ +│ attach / following logs / pull streams retained under their existing │ +│ allocation reservations; stop/join independently before wire shutdown │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +The host boundary lives in rustack-lambda-core::docker. DockerWire::connect(socket, deadline) starts its retained owner before awaiting version negotiation; dropping connection startup signals that owner rather than losing request drivers. port() returns a cloneable DockerPort; shutdown(&mut self) stops admission, signals and joins requests, and caches failure. DockerPort exposes typed version/info/image inspection and container create/start/inspect/kill/wait/remove/list/attach methods; stream/network/pull methods land with their consumers. Endpoint templates are private. DockerKey validates bounded identifier/name syntax and all path components are percent encoded. Labels supplied by the environment owner remain authoritative for mutation/reconciliation authorization above this transport layer. Request Debug exposes operation class/byte counts, never target/body values. Each upgraded attachment joins its HTTP driver before handing off an owned stream; split(io_budget) supplies a strict DockerOutputReader and raw stdin writer for the retained environment pumps. Reader next(deadline) yields redacted typed stdout/stderr chunks≤16 KiB; its record deadline survives chunk boundaries, and dropping an in-progress read poisons that reader rather than restarting partial framing. + +Each management request owns a fresh Hyper HTTP/1 connection to that socket, with no keepalive pool. One retained job owns connect, handshake, request, bounded response consumption and connection-driver termination through join. Use a single absolute request deadline≤600s and recheck the clock after awaits and typed deserialization. Bound socket connect, HTTP header parsing, upgrade negotiation and body/frame IO; do not leave a detached Hyper driver or count response headers as complete body success. Ordinary one-shot HTTP drivers use without_shutdown to recover/drop their owned IO, avoiding redundant half-close errors after Unix peer EOF; upgraded drivers join before transferring the socket. Cancellation/expiry closes that request's IO and joins its driver. Caller cancellation does not implicitly cancel/retry the service-side resource operation; preallocated identities and reconciliation handle uncertain create results. Environment cancellation retains an already-issued create under its independent bounded Docker IO budget, then cleans its outcome; caller Task expiry is not permission to drop that create future. An uncertain create cannot be declared clean solely because one immediate name lookup returned404. Compile-time wire quotas: management request/response bodies≤1 MiB each; response header buffer≤16 KiB,≤64 headers and≤4 KiB per value; request target≤4 KiB; decoded metadata JSON depth≤32,≤4,096 aggregate members/elements and≤16 KiB per scalar string. The larger scalar ceiling deliberately permits Docker inspection/configuration metadata; stricter image argv/environment/identity rules still validate before domain publication. Reject duplicate JSON object keys, HTTP trailers, wrong content framing and oversized Content-Length before collection. Stream chunked bodies through a byte limiter before materialization. Do not forward raw daemon error text, response bytes, registry credentials or URLs into infrastructure diagnostics; expose bounded operation/status/error classes with redacted sources. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index b9df1342..77d12ec9 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -75,6 +75,10 @@ The shared runtime contract suite is available as `make test-lambda-runtime`; it `make test-lambda-runtime-docker` builds the selected architecture's relay and pinned official Python RIC fixture images, then explicitly executes every resource-gated test in the shared runtime's Docker suite. It requires WORKFLOWS_DOCKER_SOCKET and a matching WORKFLOWS_ARCH; missing prerequisites fail rather than skip. The CLI is only the test transport. Assert computed42/7, warm counter1/2, request metadata, cgroup/user/rootfs restrictions, distinct FunctionError, running-handler cancellation, pre-function Close, EOF/truncated/partial frames, idle expiry, failed setup and expired helper reaping. The ordered production ControlClient also consumes real container pipes and drives computed warm calls followed by separate function-error/busy-cancellation scenarios; join its stream supervisor independently of container/CLI cleanup on all exits. Every case reconciles its preallocated owned names/run label and joins its CLI process. These are relay/client qualification checks, not provider/deployment/pool acceptance for L01–L09. Native x86, Docker Desktop and rootless qualification still require their M4 lanes. +`make test-lambda-docker-wire` builds those images and explicitly qualifies production DockerWire management/attach IO without using the CLI for execution. Assert near6MiB input/output fragmentation, real computed warm calls, handler failure, Close/Ack, joined control/bridge tasks and empty owned run labels after removal. The15 fake-daemon/framing tests additionally pin bounded malformed responses/headers/trailers, typed encoding/version negotiation, reserved cleanup during business saturation, Kill while two waits are blocked, dropped callers, cancelled shutdown waiters, cached worker panic, slow response-body expiry and poisoned partial reads. Captured TRACE tests cover production Hyper requests/responses/trailers; `make test-workflows-spikes` separately exercises actual pinned Bollard request/response/EOF decoder log sites, including synchronous stream construction. + +Current wire qualification on2026-10-25 passes on Engine29.5.2 Linux arm64 in native VZ and Linux amd64 under QEMU. Both near6MiB cases pass with60s IO/120s attempt stress budgets. These do not establish the default5s IO latency budget, native x86 performance, Docker Desktop/rootless support or full Lambda provider/pool/deployment acceptance. Independent daemon queries find no remaining containers bearing the wire harness ownership label after the runs. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 24c33288..d592a652 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,7 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire/stream/metadata/lifecycle/pool consumers remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Deployment/image actors, remaining stream/network/probe consumers and environment/pool/provider integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement the task1.3 bounded DockerWire/stream ownership consumer, then deployment/environment actors consuming the qualified client/relay/HTTP/session contracts, followed by startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement task1.3 deployment/image and environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts, together with remaining stream/network/probe operations, followed by startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 627e9ef5..92f9752b 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -8,7 +8,7 @@ This is the canonical review backlog for P0–P5. A corrected specification is n | ID | Priority / source | Finding and resolution | Remaining implementation gate | |---|---|---|---| -| W01 | P1; `apps/rustack/src/main.rs:373`, runtime transport contract | Bollard logs request/response/attach bytes at debug/trace. Specification now requires an independent mandatory dependency filter, even with RUST_LOG=trace | P1: implement filter and captured-log secret/payload regression | +| W01 | P1; `apps/rustack/src/main.rs:373`, runtime transport contract | Bollard logs request/response/attach bytes at debug/trace. Specification now requires an independent mandatory dependency filter, even with RUST_LOG=trace | Closed for implemented runtime/wire paths: mandatory task-bound filter plus captured Hyper and actual Bollard call-site regressions. Future Docker stream consumers must retain the same policy | | W02 | P1; runtime protocol contract | BootstrapReady was missing from the message variants despite being required by initialization. Specification fixed; real relay probe uses the separate signal | P1: implement production typed signal and initialization tests | | W03 | P1; runtime create contract | Omitted Architectures must default to x86_64, unlike an explicit empty list. Specification corrected | P1: normalize before validation and cover omission/empty/multiple | | W04 | P1; `codegen/src/shapes.rs:180`, `codegen/src/codegen.rs:263` | Generated chrono fields do not accept numeric AWS JSON timestamps. Independently reproduced; required/optional/fractional/negative codec behavior pinned | P2 task2.1: generator fix and executable generated round trips before service generation | @@ -143,6 +143,22 @@ rustack-core::json now exposes validated nonzero JsonLimits, validate and from_s Core has39 passing tests (21 new JSON cases, including two512-case properties) and four passing doctests. `make test-json-boundaries` runs that suite and strict production Clippy. Workspace build/test/fmt/Clippy/rustdoc pass with1,888 tests and397 ignored resource tests. Existing proptest/rstest versions are reused only for core development; no dependency version/package was added. Audit/deny still report16 B01 advisories, with bans/licenses/sources passing. Docker metadata/stream consumers and all full P1/M0 exits remain required; a passing parser is not executor acceptance. +## P1 task1.3 bounded Docker wire review + +The independent read-only review and final recheck found no remaining concrete in-slice issue after the lifecycle-wait correction below. Review covered typed operation paths and status handling, version negotiation, bounded serialization/JSON and HTTP framing, retained driver/handoff ownership, deadline/cancel races, worker panic settlement, strict multiplexed streaming and secret capture. No production DockerExecutor/provider acceptance is claimed. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W32 | P2; `crates/rustack-lambda-core/src/docker/mod.rs`, request admission | Long-lived Wait jobs could fill both reserved cleanup slots and prevent Kill/Remove. Dedicated bounded32-job/eight-mailbox wait lane now leaves two slots for inspect/kill/remove/list; six short business slots remain separate | Held-two-waits regression completes Kill before either wait is released; business saturation admits Remove; exported attach survives wire shutdown without retaining a management slot. D20 records the ownership rule | + +DockerWire and DockerPort implement bounded Unix management and strict upgraded attach using generated model-only bollard-stubs1.53.1-rc.29.3.1. All15 new fake-daemon/framing tests pass within227 Lambda core tests. The workspace passes1,903 tests (398 explicitly resource-gated), build/fmt/Clippy/rustdoc; the unchanged strict Lambda boundary debt remains B03, with no new Docker-path findings. Runtime contracts pass97 tests and48 doctests plus strict production checks and actual Linux entry-point Clippy. Eleven real Native fixture tests pass; the native SDK target passes its six actual native/DryRun cases, with its separately gated Squib case still excluded as VM evidence. + +The actual production-wire integration passes against official Python RIC images on arm64 VZ and amd64/QEMU, computing42/7 with warm counters1/2, transferring near6MiB requests/responses and surfacing a real FunctionError. Each run independently removes both named containers, joins stream/wire owners and checks its run-label scope; subsequent daemon queries find no owned wire containers. These stress cases use60s IO/120s attempt budgets rather than claiming default5s performance. The separate five-case relay suite remains required regression evidence on each daemon. + +W01's implemented filter now covers Hyper/hyper_util/Bollard independent of application TRACE. Production capture excludes request/response/header/partial-trailer markers while preserving application logs. The standalone compatibility suite passes eight tests/five doctests and Clippy, including actual Bollard create/logs construction/polls and truncated decoder fallback under LogTracer. It does not replace the stricter production framing decoder. Workspace/probe stacks align on Tokio1.54.1/Hyper1.12.0; root audit still has the same16 B01 advisories, deny bans/licenses/sources pass, and the standalone probe audit is clean. No ignore was added. + +Deployment/image actors, remaining stream/network/probe consumers, warm pooling, startup/resource checks, full SDK/provider acceptance and B01/B03 remain required before P1 closes. W30 is closed for implemented wire admission/handoff; future log/pull owners remain subject to its existing-reservation rule. W31's source citations remain pinned and checked. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 41a89598..28c0b291 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -150,6 +150,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [bounded Docker API boundary](./rustack-workflows-20-lambda-runtime-design.md#36-bounded-docker-api-boundary), [pinned source study](../docs/research/study-bollard-docker-transport-boundaries.md). - Date:2026-10-25. Production acceptance still requires the specified fake-daemon and real image tests; source inspection alone closes no runtime gate. +## D20 — Separate lifecycle waits from destructive cleanup capacity + +- Context: a Docker wait request can remain pending for an environment lifetime. Sharing two reserved management slots with waits can prevent Kill/Remove from ever reaching the daemon. +- Alternatives: use the ordinary business lane; increase one shared queue; poll container inspection repeatedly; allocate a dedicated bounded wait lane. +- Decision: retain six short business jobs and two reserved inspect/kill/remove/list jobs, plus a separate lane of at most32 concurrent wait jobs with an eight-command mailbox. Environment owners issue at most one wait per environment. Attach handoff releases its short management slot before warm reuse. +- Why: waiting for external termination must not consume the capacity needed to cause that termination. Separate admission and retained supervision provide bounded ownership without cleanup starvation. +- Pinned by: [Docker request ownership and quotas](./rustack-workflows-20-lambda-runtime-design.md#36-bounded-docker-api-boundary); executable held-wait and business-saturation regressions in the Docker wire tests. +- Date:2026-10-25. The wire lane does not itself own container lifetime or prove successful cleanup. + ## Glossary | Term | Meaning in this specification | diff --git a/tools/workflows-spikes/Cargo.lock b/tools/workflows-spikes/Cargo.lock index b4d7d869..2fe28111 100644 --- a/tools/workflows-spikes/Cargo.lock +++ b/tools/workflows-spikes/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -20,6 +29,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + [[package]] name = "bollard" version = "0.21.1" @@ -128,6 +143,34 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -236,6 +279,25 @@ dependencies = [ "r-efi", ] +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -247,6 +309,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hex" version = "0.4.3" @@ -306,7 +374,9 @@ checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", + "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -472,6 +542,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.18" @@ -489,12 +569,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + [[package]] name = "libc" version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.3" @@ -507,6 +599,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.3" @@ -524,6 +625,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -575,6 +685,43 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustack-lambda-runtime" +version = "0.9.0" +dependencies = [ + "bytes", + "ciborium", + "http", + "http-body-util", + "hyper", + "hyper-util", + "serde", + "thiserror", + "tokio", + "tracing", + "tracing-core", + "tracing-subscriber", + "typed-builder", + "uuid", +] + [[package]] name = "rustack-workflows-spikes" version = "0.1.0" @@ -587,13 +734,31 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "rustack-lambda-runtime", "serde", "serde_json", + "tempfile", "thiserror", "tokio", + "tracing", + "tracing-log", + "tracing-subscriber", "uuid", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -672,6 +837,25 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "slab" version = "0.4.12" @@ -733,6 +917,19 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.21" @@ -753,6 +950,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "tinystr" version = "0.8.5" @@ -773,6 +979,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys", @@ -816,9 +1023,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -826,6 +1045,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -834,6 +1096,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "unicode-ident" version = "1.0.27" @@ -869,6 +1151,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "want" version = "0.3.2" diff --git a/tools/workflows-spikes/Cargo.toml b/tools/workflows-spikes/Cargo.toml index 68dbfb54..c113952b 100644 --- a/tools/workflows-spikes/Cargo.toml +++ b/tools/workflows-spikes/Cargo.toml @@ -13,14 +13,21 @@ bollard = { version = "=0.21.1", default-features = false, features = ["pipe"] } bytes = { version = "1.12", features = ["serde"] } ciborium = "0.2" futures = "0.3" -hyper = { version = "1.11", features = ["server", "http1"] } +hyper = { version = "1.12", features = ["server", "http1"] } hyper-util = { version = "0.1", features = ["tokio"] } http-body-util = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "net", "sync", "time", "io-util", "io-std"] } +tokio = { version = "1.54", features = ["rt-multi-thread", "macros", "net", "sync", "time", "io-util", "io-std"] } uuid = { version = "1.26", features = ["v4"] } +[dev-dependencies] +rustack-lambda-runtime = { path = "../../crates/rustack-lambda-runtime" } +tempfile = "3.27" +tracing = "0.1" +tracing-log = "0.2" +tracing-subscriber = { version = "0.3", features = ["fmt"] } + [lints.rust] rust_2024_compatibility = "warn" missing_docs = "warn" diff --git a/tools/workflows-spikes/handler.py b/tools/workflows-spikes/handler.py index c44fa764..807a0a3b 100644 --- a/tools/workflows-spikes/handler.py +++ b/tools/workflows-spikes/handler.py @@ -16,7 +16,7 @@ def handler(event, context): print("P0_BUSY_START=" + context.aws_request_id, flush=True) time.sleep(60) cgroup = Path("/sys/fs/cgroup") - return { + response = { "computed": event["value"] + 7, "counter": counter, "uid": os.getuid(), @@ -25,3 +25,8 @@ def handler(event, context): "pidsMax": (cgroup / "pids.max").read_text().strip(), "rootReadOnly": bool(os.statvfs("/").f_flag & os.ST_RDONLY), } + if "padding" in event: + response["inputBytes"] = len(event["padding"].encode("utf-8")) + if event.get("response_bytes"): + response["large"] = "r" * event["response_bytes"] + return response diff --git a/tools/workflows-spikes/src/diagnostics_tests.rs b/tools/workflows-spikes/src/diagnostics_tests.rs new file mode 100644 index 00000000..02590006 --- /dev/null +++ b/tools/workflows-spikes/src/diagnostics_tests.rs @@ -0,0 +1,166 @@ +//! Verify the production filter against actual Bollard log-facade call sites. + +use std::{ + io::{self, Write}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use bollard::{ + API_DEFAULT_VERSION, Docker, container::LogOutput, models::ContainerCreateBody, + query_parameters::LogsOptionsBuilder, +}; +use futures::StreamExt; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{UnixListener, UnixStream}, + time::timeout, +}; +use tracing::instrument::WithSubscriber; +use tracing_log::{LogTracer, log::LevelFilter}; + +#[derive(Clone, Debug, Default)] +struct Capture(Arc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut output = self + .0 + .lock() + .map_err(|_| io::Error::other("capture lock failed"))?; + if output.len().saturating_add(bytes.len()) > 1024 * 1024 { + return Err(io::Error::other("capture exceeded cap")); + } + output.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +async fn request(listener: &UnixListener) -> Result { + let (mut stream, _) = listener.accept().await?; + let mut bytes = Vec::new(); + let end = loop { + if let Some(position) = bytes.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + break position + 4; + } + ensure!(bytes.len() < 16 * 1024, "fixture header exceeded cap"); + let mut buffer = [0; 512]; + let read = stream.read(&mut buffer).await?; + ensure!(read > 0, "fixture header EOF"); + bytes.extend_from_slice(buffer.get(..read).context("fixture slice")?); + }; + let header = std::str::from_utf8(bytes.get(..end).context("fixture header")?)?; + let length = header + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(key, _)| key.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim().parse::()) + }) + .transpose()? + .unwrap_or(0); + ensure!(length <= 4096, "fixture body exceeded cap"); + let received = bytes + .len() + .checked_sub(end) + .context("fixture body offset")?; + let remaining = length + .checked_sub(received) + .context("fixture body length")?; + stream.read_exact(&mut vec![0; remaining]).await?; + Ok(stream) +} + +async fn respond(stream: &mut UnixStream, body: &[u8]) -> Result<()> { + let header = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(header.as_bytes()).await?; + stream.write_all(body).await?; + Ok(()) +} + +#[tokio::test] +async fn test_should_filter_actual_bollard_request_response_and_truncated_decoder_logs() +-> Result<()> { + LogTracer::builder() + .with_max_level(LevelFilter::Trace) + .init()?; + let directory = tempfile::tempdir()?; + let socket = directory.path().join("docker.sock"); + let listener = UnixListener::bind(&socket)?; + let mut server = tokio::spawn(async move { + let mut create = request(&listener).await?; + respond( + &mut create, + br#"{"Id":"probe","Warnings":["secret_bollard_response"]}"#, + ) + .await?; + drop(create); + let mut logs = request(&listener).await?; + let mut truncated = vec![1, 0, 0, 0, 0, 0, 0, 100]; + truncated.extend_from_slice(b"secret_bollard_trailing"); + respond(&mut logs, &truncated).await?; + Ok::<_, anyhow::Error>(()) + }); + let capture = Capture::default(); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let outcome = async { + tracing::debug!("compatibility_application_marker"); + let dispatch = runtime_dispatch(); + let docker = Docker::connect_with_socket(socket.to_str().context("fixture socket UTF-8")?, 3, API_DEFAULT_VERSION)?; + let created = docker.create_container(None, ContainerCreateBody { env: Some(vec!["SECRET=secret_bollard_request".into()]), ..Default::default() }).with_subscriber(dispatch.clone()).await?; + ensure!(created.id == "probe", "Bollard create response mismatch"); + // logs() constructs its HTTP future synchronously. Cover construction + // as well as every stream poll with the production dispatch policy. + let mut logs = tracing::dispatcher::with_default(&dispatch, || docker.logs("secret_bollard_target", Some(LogsOptionsBuilder::default().stdout(true).build()))); + async { + let output = logs.next().await.context("expected EOF fallback record")??; + ensure!(matches!(output, LogOutput::Console { message } if message.as_ref() == b"secret_bollard_trailing"), "upstream EOF behavior changed"); + ensure!(logs.next().await.is_none(), "unexpected extra fallback record"); + Ok::<_, anyhow::Error>(()) + }.with_subscriber(dispatch).await + }.with_subscriber(subscriber).await; + let joined = match timeout(Duration::from_secs(3), &mut server).await { + Ok(result) => result.context("join fixture server")?, + Err(_) => { + server.abort(); + let _ = server.await; + Err(anyhow::anyhow!("fixture server deadline")) + } + }; + outcome.and(joined)?; + let shown = String::from_utf8( + capture + .0 + .lock() + .map_err(|_| anyhow::anyhow!("capture lock failed"))? + .clone(), + )?; + ensure!( + shown.contains("compatibility_application_marker"), + "application logs were lost" + ); + for secret in [ + "secret_bollard_request", + "secret_bollard_response", + "secret_bollard_target", + "secret_bollard_trailing", + ] { + ensure!( + !shown.contains(secret), + "Bollard payload reached infrastructure logs" + ); + } + Ok(()) +} diff --git a/tools/workflows-spikes/src/lib.rs b/tools/workflows-spikes/src/lib.rs index 266149cc..6ef203de 100644 --- a/tools/workflows-spikes/src/lib.rs +++ b/tools/workflows-spikes/src/lib.rs @@ -4,6 +4,9 @@ //! the production Runtime API or a hardened external CBOR deserializer. #![forbid(unsafe_code)] +#[cfg(all(test, unix))] +mod diagnostics_tests; + use std::{ fmt, io::{self, Cursor}, From 906a259a7eba06eae886a099fc3bdb6cf5a232e4 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 20:29:56 -0700 Subject: [PATCH 013/106] phase 1: resolve immutable image platforms and retain validated execution plans Implement read-only local resolution, retained redacted image defaults, explicit Docker argv reset semantics and restore-verified canonical fingerprints. Bind platform inspection to the addressable parent ID, retain selected manifests independently, and create with a captured platform. Validate classic-store evidence while preserving null VFS status. Pin historical Moby source for merge and compatibility guards. Independent W33-W38 findings are fixed and rechecked. Workspace build/test/fmt/Clippy/rustdoc pass with 1964 tests; Lambda core has 288 tests and 58 doctests. Both real Docker two-case suites pass on arm64 VZ and amd64/QEMU with no owned containers left. Deployment/CAS, availability leases, allowed pulls, unpacked quota proof, pool/provider integration and B01/B03 remain required before P1/M0 completion. --- .gitmodules | 3 + Cargo.lock | 1 + crates/rustack-lambda-core/Cargo.toml | 1 + crates/rustack-lambda-core/src/artifact.rs | 2 + .../src/artifact/execution.rs | 384 +++++++++++ .../rustack-lambda-core/src/artifact/image.rs | 219 ++++++- .../rustack-lambda-core/src/docker/image.rs | 601 ++++++++++++++++++ crates/rustack-lambda-core/src/docker/mod.rs | 48 +- .../rustack-lambda-core/src/docker/tests.rs | 97 ++- .../rustack-lambda-core/src/docker/types.rs | 4 + .../src/function_config.rs | 2 +- .../rustack-lambda-core/tests/docker_wire.rs | 120 +++- docs/index.md | 2 +- ...udy-bollard-docker-transport-boundaries.md | 24 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 33 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 2 +- specs/rustack-workflows-93-review.md | 19 + specs/rustack-workflows-99-decisions.md | 9 + vendors/moby | 1 + 21 files changed, 1532 insertions(+), 44 deletions(-) create mode 100644 crates/rustack-lambda-core/src/artifact/execution.rs create mode 100644 crates/rustack-lambda-core/src/docker/image.rs create mode 160000 vendors/moby diff --git a/.gitmodules b/.gitmodules index e625a759..28970f37 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "vendors/bollard"] path = vendors/bollard url = https://github.com/fussybeaver/bollard.git +[submodule "vendors/moby"] + path = vendors/moby + url = https://github.com/moby/moby.git diff --git a/Cargo.lock b/Cargo.lock index be8fca6b..af4050d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3826,6 +3826,7 @@ dependencies = [ "chrono", "dashmap", "form_urlencoded", + "hex", "http 1.5.0", "http-body-util", "hyper", diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 37bce66b..7bbdb784 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -34,6 +34,7 @@ thiserror = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } sha2 = { workspace = true } +hex = { workspace = true } base64 = { workspace = true } async-trait = { workspace = true } tempfile = { workspace = true } diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs index d1bf7a9d..d762d08c 100644 --- a/crates/rustack-lambda-core/src/artifact.rs +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -3,6 +3,7 @@ //! ZIP leases keep successfully staged code alive through invocation cleanup. //! Images carry immutable daemon identity rather than an unresolved mutable tag. +mod execution; mod image; use std::{ @@ -13,6 +14,7 @@ use std::{ }; use bytes::Bytes; +pub use execution::{ImageExecutionDefaults, ImageExecutionMetadata, ImageExecutionPlan}; pub use image::{ Architecture, ArtifactError, ImageMetadata, ImageReference, ResolvedImage, Sha256Digest, }; diff --git a/crates/rustack-lambda-core/src/artifact/execution.rs b/crates/rustack-lambda-core/src/artifact/execution.rs new file mode 100644 index 00000000..f5c230e5 --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/execution.rs @@ -0,0 +1,384 @@ +//! Validated image defaults retained through capture and snapshot restoration. + +use std::{fmt, sync::Arc}; + +use rustack_lambda_model::types::ImageConfig; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::ArtifactError; +use crate::function_config::validate_image_config; + +/// Primitive image execution metadata; validate with `ImageExecutionDefaults`. +#[derive(Clone, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ImageExecutionMetadata { + /// Inherited entrypoint, before function overrides. + pub entrypoint: Vec, + /// Inherited command, before function overrides. + pub command: Vec, + /// Image-internal working directory; empty normalizes to root. + pub working_directory: String, + /// Ordered inherited environment, including Docker deletion markers. + pub environment: Vec, + /// Image user declaration; container security policy is applied separately. + pub user: String, +} +impl fmt::Debug for ImageExecutionMetadata { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ImageExecutionMetadata()") + } +} + +/// Immutable normalized image defaults, revalidated on deserialization. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(try_from = "ImageExecutionMetadata", into = "ImageExecutionMetadata")] +#[non_exhaustive] +pub struct ImageExecutionDefaults(Arc); + +impl TryFrom for ImageExecutionDefaults { + type Error = ArtifactError; + fn try_from(mut metadata: ImageExecutionMetadata) -> Result { + if metadata.entrypoint.len() == 1 + && metadata.entrypoint.first().is_some_and(String::is_empty) + { + metadata.entrypoint.clear(); + } + if metadata.working_directory.is_empty() { + metadata.working_directory = "/".into(); + } + validate_image_config(&ImageConfig { + entry_point: Some(metadata.entrypoint.clone()), + command: Some(metadata.command.clone()), + working_directory: Some(metadata.working_directory.clone()), + }) + .map_err(|_| ArtifactError::Inspection)?; + validate_environment(&metadata.environment)?; + if metadata.user.len() > 256 + || !metadata.user.bytes().all(|byte| (32..=126).contains(&byte)) + { + return Err(ArtifactError::Inspection); + } + Ok(Self(Arc::new(metadata))) + } +} +impl From for ImageExecutionMetadata { + fn from(defaults: ImageExecutionDefaults) -> Self { + Arc::unwrap_or_clone(defaults.0) + } +} + +impl ImageExecutionDefaults { + /// Borrow normalized defaults; diagnostics must not print their values. + /// + /// ```no_run + /// # fn defaults(defaults: &rustack_lambda_core::artifact::ImageExecutionDefaults) { + /// let environment = &defaults.metadata().environment; + /// # } + /// ``` + #[must_use] + pub fn metadata(&self) -> &ImageExecutionMetadata { + &self.0 + } + + /// Materialize the exact Docker argv/workdir, preserving empty overrides. + /// + /// # Errors + /// Rejects invalid overrides, an empty executable or composed argv bounds. + /// + /// ```no_run + /// # fn plan(defaults: &rustack_lambda_core::artifact::ImageExecutionDefaults) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let plan = defaults.plan(None)?; + /// assert!(!plan.entrypoint().is_empty()); // Includes Docker's reset sentinel. + /// # Ok(()) } + /// ``` + pub fn plan( + &self, + overrides: Option<&ImageConfig>, + ) -> Result { + if let Some(config) = overrides { + validate_image_config(config).map_err(|_| ArtifactError::Inspection)?; + } + let mut entrypoint = overrides + .and_then(|config| config.entry_point.as_ref()) + .unwrap_or(&self.0.entrypoint) + .clone(); + let command = overrides + .and_then(|config| config.command.as_ref()) + .unwrap_or(&self.0.command) + .clone(); + let workdir = overrides + .and_then(|config| config.working_directory.as_ref()) + .unwrap_or(&self.0.working_directory) + .clone(); + if entrypoint.len() == 1 && entrypoint.first().is_some_and(String::is_empty) { + entrypoint.clear(); + } + validate_image_config(&ImageConfig { + entry_point: Some(entrypoint.clone()), + command: Some(command.clone()), + working_directory: Some(workdir.clone()), + }) + .map_err(|_| ArtifactError::Inspection)?; + if entrypoint + .iter() + .chain(&command) + .next() + .is_none_or(String::is_empty) + { + return Err(ArtifactError::Inspection); + } + if entrypoint.is_empty() { + entrypoint.push(String::new()); + } + Ok(ImageExecutionPlan { + entrypoint, + command, + workdir, + }) + } +} + +/// Validated Docker-create fields; no daemon inheritance is needed. +#[derive(Clone)] +#[non_exhaustive] +pub struct ImageExecutionPlan { + entrypoint: Vec, + command: Vec, + workdir: String, +} +impl fmt::Debug for ImageExecutionPlan { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ImageExecutionPlan()") + } +} +impl ImageExecutionPlan { + /// Exact create entrypoint; semantic emptiness uses the `[""]` reset sentinel. + /// + /// ```no_run + /// # fn entry(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let entrypoint = plan.entrypoint().to_vec(); + /// # } + /// ``` + #[must_use] + pub fn entrypoint(&self) -> &[String] { + &self.entrypoint + } + /// Exact create command, including an explicitly empty array. + /// + /// ```no_run + /// # fn command(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let command = plan.command().to_vec(); + /// # } + /// ``` + #[must_use] + pub fn command(&self) -> &[String] { + &self.command + } + /// Exact absolute working directory inside the image. + /// + /// ```no_run + /// # fn workdir(plan: &rustack_lambda_core::artifact::ImageExecutionPlan) { + /// let workdir = plan.working_directory(); + /// # } + /// ``` + #[must_use] + pub fn working_directory(&self) -> &str { + &self.workdir + } +} + +fn validate_environment(environment: &[String]) -> Result<(), ArtifactError> { + if environment.len() > 128 { + return Err(ArtifactError::Inspection); + } + let mut total = 0_usize; + for entry in environment { + let key = entry.split_once('=').map_or(entry.as_str(), |(key, _)| key); + if entry.len() > 4096 + || entry.contains('\0') + || key.len() > 256 + || !key + .as_bytes() + .first() + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err(ArtifactError::Inspection); + } + total = total + .checked_add(entry.len()) + .filter(|total| *total <= 64 * 1024) + .ok_or(ArtifactError::Inspection)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn metadata() -> ImageExecutionMetadata { + ImageExecutionMetadata::builder() + .entrypoint(vec!["/entry".into()]) + .command(vec!["handler".into()]) + .working_directory("/task".into()) + .environment(vec!["PATH=/bin".into(), "REMOVED".into()]) + .user("1000:1000".into()) + .build() + } + + #[test] + fn test_should_materialize_inherited_and_explicit_empty_docker_argv() { + let defaults = ImageExecutionDefaults::try_from(metadata()).unwrap(); + let inherited = defaults.plan(None).unwrap(); + assert_eq!(inherited.entrypoint(), ["/entry"]); + assert_eq!(inherited.command(), ["handler"]); + assert_eq!(inherited.working_directory(), "/task"); + let clear_command = defaults + .plan(Some(&ImageConfig { + command: Some(vec![]), + ..Default::default() + })) + .unwrap(); + assert_eq!(clear_command.command(), [] as [String; 0]); + assert_eq!(clear_command.entrypoint(), ["/entry"]); + for reset in [vec![], vec![String::new()]] { + let override_config = ImageConfig { + entry_point: Some(reset), + command: Some(vec!["/other".into()]), + working_directory: Some("/tmp".into()), + }; + let plan = defaults.plan(Some(&override_config)).unwrap(); + assert_eq!(plan.entrypoint(), [""]); + assert_eq!(plan.command(), ["/other"]); + assert_eq!(plan.working_directory(), "/tmp"); + } + assert!( + defaults + .plan(Some(&ImageConfig { + entry_point: Some(vec![]), + command: Some(vec![]), + ..Default::default() + })) + .is_err() + ); + } + + #[test] + fn test_should_allow_empty_defaults_with_an_executable_override_and_normalize_root() { + let mut input = metadata(); + input.entrypoint = vec![String::new()]; + input.command.clear(); + input.working_directory.clear(); + let defaults = ImageExecutionDefaults::try_from(input).unwrap(); + assert_eq!(defaults.metadata().entrypoint, [] as [String; 0]); + assert_eq!(defaults.metadata().working_directory, "/"); + assert!(defaults.plan(None).is_err()); + let plan = defaults + .plan(Some(&ImageConfig { + command: Some(vec!["/start".into()]), + ..Default::default() + })) + .unwrap(); + assert_eq!(plan.entrypoint(), [""]); + assert_eq!(plan.command(), ["/start"]); + let mut invalid = metadata(); + invalid.entrypoint.clear(); + invalid.command.clear(); + invalid.working_directory = "relative".into(); + assert!(ImageExecutionDefaults::try_from(invalid).is_err()); + } + + #[rstest::rstest] + #[case("argv_count")] + #[case("argv_item")] + #[case("argv_total")] + #[case("argv_nul")] + #[case("directory")] + #[case("directory_size")] + #[case("env_count")] + #[case("env_item")] + #[case("env_total")] + #[case("env_key")] + #[case("env_key_size")] + #[case("env_nul")] + #[case("user_size")] + #[case("user_control")] + fn test_should_reject_invalid_defaults_before_capture(#[case] field: &str) { + let mut input = metadata(); + match field { + "argv_count" => input.command = vec!["x".into(); 1501], + "argv_item" => input.command = vec!["x".repeat(4097)], + "argv_total" => input.command = vec!["x".repeat(4096); 17], + "argv_nul" => input.entrypoint = vec!["/bin/x\0".into()], + "directory" => input.working_directory = "relative".into(), + "directory_size" => input.working_directory = format!("/{}", "x".repeat(1000)), + "env_count" => input.environment = vec!["X=1".into(); 129], + "env_item" => input.environment = vec![format!("X={}", "x".repeat(4095))], + "env_total" => input.environment = vec![format!("X={}", "x".repeat(4094)); 17], + "env_key" => input.environment = vec!["1KEY=value".into()], + "env_key_size" => input.environment = vec![format!("{}=x", "K".repeat(257))], + "env_nul" => input.environment = vec!["KEY=\0".into()], + "user_size" => input.user = "x".repeat(257), + "user_control" => input.user = "1000\n".into(), + _ => panic!("unknown test field"), + } + assert!(matches!( + ImageExecutionDefaults::try_from(input), + Err(ArtifactError::Inspection) + )); + } + + #[test] + fn test_should_check_composed_argv_and_environment_exact_byte_boundaries() { + let mut input = metadata(); + input.entrypoint = vec!["e".repeat(4096); 16]; + input.command.clear(); + input.environment = vec![format!("X={}", "x".repeat(4094)); 16]; + let defaults = ImageExecutionDefaults::try_from(input).unwrap(); + defaults.plan(None).unwrap(); + assert!( + defaults + .plan(Some(&ImageConfig { + command: Some(vec!["x".into()]), + ..Default::default() + })) + .is_err() + ); + assert!( + defaults + .plan(Some(&ImageConfig { + working_directory: Some(String::new()), + ..Default::default() + })) + .is_err() + ); + } + + #[test] + fn test_should_revalidate_saved_defaults_and_redact_all_debug_surfaces() { + let mut input = metadata(); + input.entrypoint = vec!["secret_argv".into()]; + input.environment = vec!["SECRET=secret_value".into()]; + input.working_directory = "/secret_path".into(); + input.user = "secret_user".into(); + let defaults = ImageExecutionDefaults::try_from(input.clone()).unwrap(); + let saved = serde_json::to_value(&defaults).unwrap(); + let restored: ImageExecutionDefaults = serde_json::from_value(saved.clone()).unwrap(); + assert_eq!(restored.plan(None).unwrap().entrypoint(), ["secret_argv"]); + for shown in [ + format!("{input:?}"), + format!("{defaults:?}"), + format!("{:?}", defaults.plan(None).unwrap()), + ] { + assert!(!shown.contains("secret_")); + } + let mut invalid = saved; + invalid["environment"] = serde_json::json!(["BAD-KEY=value"]); + assert!(serde_json::from_value::(invalid).is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/artifact/image.rs b/crates/rustack-lambda-core/src/artifact/image.rs index ae52eaa5..6d4d1fa1 100644 --- a/crates/rustack-lambda-core/src/artifact/image.rs +++ b/crates/rustack-lambda-core/src/artifact/image.rs @@ -3,6 +3,7 @@ use std::{fmt, net::Ipv6Addr, str::FromStr}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use typed_builder::TypedBuilder; use winnow::{ ModalResult, Parser, @@ -10,6 +11,8 @@ use winnow::{ token::take_while, }; +use super::ImageExecutionDefaults; + /// Closed Lambda architecture, also used to select a Docker platform. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Architecture { @@ -63,7 +66,7 @@ impl FromStr for Architecture { } /// Invalid artifact metadata; errors never include the untrusted input. -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Clone, Copy, thiserror::Error)] #[non_exhaustive] pub enum ArtifactError { /// Reference does not match the bounded Docker grammar. @@ -99,6 +102,64 @@ impl ImageReference { pub fn as_str(&self) -> &str { &self.0 } + + /// Repository spelling, including an explicit registry, without tag/digest. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "localhost:5000/team/worker:v1".parse()?; + /// assert_eq!(image.repository(), "localhost:5000/team/worker"); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn repository(&self) -> &str { + let name = self + .0 + .split_once('@') + .map_or(self.0.as_str(), |(name, _)| name); + if name + .rsplit('/') + .next() + .is_some_and(|part| part.contains(':')) + { + name.rsplit_once(':').map_or(name, |(name, _)| name) + } else { + name + } + } + + /// Explicit registry component; short local names do not invent provenance. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "worker:v1".parse()?; + /// assert_eq!(image.registry(), None); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn registry(&self) -> Option<&str> { + self.repository() + .split_once('/') + .and_then(|(first, _)| is_registry_component(first).then_some(first)) + } + + /// Requested digest suffix, if present, already checked by the constructor. + /// + /// ``` + /// let image: rustack_lambda_core::artifact::ImageReference = "worker:v1".parse()?; + /// assert_eq!(image.digest(), None); + /// # Ok::<(), rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub fn digest(&self) -> Option<&str> { + self.0.split_once('@').map(|(_, digest)| digest) + } +} + +fn is_registry_component(first: &str) -> bool { + first.contains('.') + || first.contains(':') + || first == "localhost" + || first.starts_with('[') + || first.bytes().any(|byte| byte.is_ascii_uppercase()) } impl FromStr for ImageReference { @@ -134,12 +195,7 @@ impl FromStr for ImageReference { return Err(ArtifactError::Reference); } let repository = if let Some((first, remaining)) = name.split_once('/') { - if first.contains('.') - || first.contains(':') - || first == "localhost" - || first.starts_with('[') - || first.bytes().any(|byte| byte.is_ascii_uppercase()) - { + if is_registry_component(first) { validate_registry(first)?; remaining } else { @@ -314,6 +370,8 @@ pub struct ImageMetadata { pub requested_reference: ImageReference, /// Immutable daemon image content ID. pub image_id: Sha256Digest, + /// Selected platform manifest, separate from an addressable parent image ID. + pub selected_manifest: Option, /// Registry repository and selected manifest digest, when available. pub registry_manifest: Option, /// Inspected operating system, required to equal linux. @@ -322,8 +380,58 @@ pub struct ImageMetadata { pub architecture: Architecture, /// SHA-256 of the normalized inspected execution metadata. pub inspection_hash: Sha256Digest, - /// Inspected uncompressed size, bounded to 10 GiB locally. + /// Selected image size reported by the daemon, bounded to 10 GiB locally. pub size_bytes: u64, + /// Validated inherited execution defaults, retained without re-inspection. + pub execution: ImageExecutionDefaults, +} + +impl ImageMetadata { + /// Compute the canonical fingerprint also verified during snapshot restore. + /// + /// # Errors + /// Rejects an invalid inspected size or bounded metadata serialization. + /// + /// ```no_run + /// # fn hash(metadata: &rustack_lambda_core::artifact::ImageMetadata) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let hash = rustack_lambda_core::artifact::ImageMetadata::execution_fingerprint( + /// &metadata.image_id, metadata.selected_manifest.as_ref(), + /// metadata.architecture, metadata.size_bytes, &metadata.execution)?; + /// # Ok(()) } + /// ``` + pub fn execution_fingerprint( + image_id: &Sha256Digest, + selected_manifest: Option<&Sha256Digest>, + architecture: Architecture, + size: u64, + defaults: &ImageExecutionDefaults, + ) -> Result { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct Fingerprint<'a> { + version: u8, + image_id: &'a Sha256Digest, + descriptor_digest: Option<&'a Sha256Digest>, + architecture: Architecture, + size: u64, + defaults: &'a ImageExecutionDefaults, + } + if !(1..=10 * 1024 * 1024 * 1024).contains(&size) { + return Err(ArtifactError::Inspection); + } + // Validated defaults contain <=128KiB of argv/environment, whose worst + // JSON escape expansion is sixfold; fixed-size identities keep this <1MiB. + let bytes = serde_json::to_vec(&Fingerprint { + version: 1, + image_id, + descriptor_digest: selected_manifest, + architecture, + size, + defaults, + }) + .map_err(|_| ArtifactError::Inspection)?; + format!("sha256:{}", hex::encode(Sha256::digest(bytes))).parse() + } } /// Immutable Linux image metadata that passed all inspection invariants. @@ -345,6 +453,24 @@ impl TryFrom for ResolvedImage { { return Err(ArtifactError::Inspection); } + if metadata.inspection_hash + != ImageMetadata::execution_fingerprint( + &metadata.image_id, + metadata.selected_manifest.as_ref(), + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + )? + { + return Err(ArtifactError::Inspection); + } + if let (Some(selected), Some(registry)) = + (&metadata.selected_manifest, &metadata.registry_manifest) + { + if registry.digest() != Some(selected.as_str()) { + return Err(ArtifactError::Inspection); + } + } Ok(Self(metadata)) } } @@ -357,26 +483,35 @@ impl From for ImageMetadata { impl ResolvedImage { #[cfg(test)] pub(crate) fn fixture(architecture: Architecture) -> Self { + let image_id = format!("sha256:{}", "a".repeat(64)) + .parse() + .expect("valid unit digest"); + let execution = super::ImageExecutionMetadata::builder() + .entrypoint(vec!["/bootstrap".into()]) + .command(Vec::new()) + .working_directory("/".into()) + .environment(Vec::new()) + .user("1000".into()) + .build() + .try_into() + .expect("valid execution defaults"); + let hash = + ImageMetadata::execution_fingerprint(&image_id, None, architecture, 1024, &execution) + .expect("unit fingerprint"); ImageMetadata::builder() .requested_reference( "rustack-fixture:unit" .parse() .expect("valid unit reference"), ) - .image_id( - format!("sha256:{}", "a".repeat(64)) - .parse() - .expect("valid unit digest"), - ) + .image_id(image_id) + .selected_manifest(None) .registry_manifest(None) .os("linux".into()) .architecture(architecture) - .inspection_hash( - format!("sha256:{}", "b".repeat(64)) - .parse() - .expect("valid unit inspection digest"), - ) + .inspection_hash(hash) .size_bytes(1024) + .execution(execution) .build() .try_into() .expect("valid unit image metadata") @@ -564,4 +699,52 @@ mod tests { } Ok(()) } + #[rstest::rstest] + #[case("/imageId", serde_json::json!(format!("sha256:{}", "d".repeat(64))))] + #[case("/selectedManifest", serde_json::json!(format!("sha256:{}", "d".repeat(64))))] + #[case("/sizeBytes", serde_json::json!(1025))] + #[case("/architecture", serde_json::json!("arm64"))] + #[case("/execution/command", serde_json::json!(["changed-valid-argv"]))] + #[case("/execution/environment", serde_json::json!(["KEY=changed-valid-value"]))] + #[case("/execution/user", serde_json::json!("1001"))] + #[case("/execution/entrypoint", serde_json::json!(["/other"]))] + #[case("/execution/workingDirectory", serde_json::json!("/other"))] + #[case("/inspectionHash", serde_json::json!(format!("sha256:{}", "f".repeat(64))))] + fn test_should_reject_valid_saved_metadata_mutations_with_a_stale_fingerprint( + #[case] pointer: &str, + #[case] replacement: serde_json::Value, + ) { + let image = ResolvedImage::fixture(Architecture::X86_64); + let mut saved = serde_json::to_value(image).unwrap(); + *saved.pointer_mut(pointer).unwrap() = replacement; + assert!(serde_json::from_value::(saved).is_err()); + } + + #[test] + fn test_should_extract_registry_repository_and_digest_without_confusing_ports() { + for (reference, repository, registry) in [ + ( + "localhost:5000/team/worker:v1", + "localhost:5000/team/worker", + Some("localhost:5000"), + ), + ( + "[::1]:5000/team/worker:v1", + "[::1]:5000/team/worker", + Some("[::1]:5000"), + ), + ("Registry/worker:v1", "Registry/worker", Some("Registry")), + ("team/worker:v1", "team/worker", None), + ("worker:v1", "worker", None), + ] { + let with_digest = format!("{reference}@sha256:{}", "e".repeat(64)); + let image: ImageReference = with_digest.parse().unwrap(); + assert_eq!(image.repository(), repository); + assert_eq!(image.registry(), registry); + assert_eq!( + image.digest(), + Some(format!("sha256:{}", "e".repeat(64)).as_str()) + ); + } + } } diff --git a/crates/rustack-lambda-core/src/docker/image.rs b/crates/rustack-lambda-core/src/docker/image.rs new file mode 100644 index 00000000..ce47bddd --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/image.rs @@ -0,0 +1,601 @@ +//! Read-only platform selection and executable-image validation before deployment. + +use bytes::Bytes; +use http::{Method, StatusCode}; +use rustack_lambda_model::types::ImageConfig; +use tokio::time::Instant; + +use super::{ + DockerError, DockerPort, Operation, decode, escaped, + models::{ImageInspect, SystemInfo}, +}; +use crate::{ + artifact::{ + Architecture, ArtifactError, ImageExecutionDefaults, ImageExecutionMetadata, ImageMetadata, + ImageReference, ResolvedImage, Sha256Digest, + }, + function_config::validate_image_config, +}; + +/// Read-only local resolution; deployment publication remains with its actor. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ImageResolver { + port: DockerPort, +} + +impl ImageResolver { + /// Use an already negotiated bounded Docker port; never discover a daemon. + /// + /// ```no_run + /// # fn resolver(port: rustack_lambda_core::docker::DockerPort) { + /// let resolver = rustack_lambda_core::docker::ImageResolver::new(port); + /// # } + /// ``` + #[must_use] + pub fn new(port: DockerPort) -> Self { + Self { port } + } + + /// Resolve a preloaded image and validate effective entry configuration. + /// + /// This never pulls, builds or reads registry credentials. Function settings + /// and deployment status/CAS are validated and owned by the provider. + /// + /// # Errors + /// Returns wire failure, invalid overrides or invalid/ambiguous inspection + /// metadata. No error includes image argv, environment or daemon text. + /// + /// ```no_run + /// # async fn example(port: rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// use rustack_lambda_core::{artifact::Architecture, docker::ImageResolver}; + /// let image = ImageResolver::new(port).resolve_local(&"worker:v1".parse()?, + /// Architecture::Arm64, None, + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn resolve_local( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + ) -> Result { + super::check_deadline(deadline)?; + if let Some(overrides) = overrides { + validate_image_config(overrides).map_err(|_| DockerError::Configuration)?; + } + let root = self.port.inspect_image(reference, deadline).await?; + let launch_id: Sha256Digest = root.id.as_deref().ok_or_else(invalid)?.parse()?; + let inspected = if self.port.api_version().minor() >= 49 { + drop(root); + let platform = match architecture { + Architecture::X86_64 => r#"{"os":"linux","architecture":"amd64"}"#, + Architecture::Arm64 => r#"{"os":"linux","architecture":"arm64"}"#, + }; + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("platform", platform) + .finish(); + let path = format!("/images/{}/json?{query}", escaped(launch_id.as_str())); + decode( + self.port + .call( + Operation::Image, + Method::GET, + path, + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + )? + } else { + root + }; + if inspected.descriptor.is_none() { + let info = self.port.info(deadline).await?; + verify_classic_store(&inspected, &info)?; + } + let result = resolve(reference, architecture, overrides, inspected, &launch_id); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } +} + +fn verify_classic_store(inspected: &ImageInspect, info: &SystemInfo) -> Result<(), DockerError> { + let driver = inspected.graph_driver.as_ref().ok_or_else(invalid)?; + if !matches!( + driver.name.as_str(), + "overlay2" | "aufs" | "btrfs" | "devicemapper" | "vfs" | "zfs" | "fuse-overlayfs" + ) || info.driver.as_deref() != Some(driver.name.as_str()) + || inspected + .manifests + .as_ref() + .is_some_and(|manifests| !manifests.is_empty()) + { + return Err(invalid()); + } + let statuses = info.driver_status.as_deref().unwrap_or_default(); + if statuses.len() > 128 { + return Err(invalid()); + } + for status in statuses { + let [key, value] = status.as_slice() else { + return Err(invalid()); + }; + if key.len() > 256 + || value.len() > 16 * 1024 + || (key.eq_ignore_ascii_case("driver-type") && value == "io.containerd.snapshotter.v1") + { + return Err(invalid()); + } + } + Ok(()) +} + +fn invalid() -> DockerError { + ArtifactError::Inspection.into() +} + +fn resolve( + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + inspected: ImageInspect, + launch_id: &Sha256Digest, +) -> Result { + let docker_arch = match architecture { + Architecture::X86_64 => "amd64", + Architecture::Arm64 => "arm64", + }; + if inspected.os.as_deref() != Some("linux") + || inspected.architecture.as_deref() != Some(docker_arch) + { + return Err(invalid()); + } + let _: Sha256Digest = inspected.id.as_deref().ok_or_else(invalid)?.parse()?; + let size = inspected + .size + .and_then(|value| u64::try_from(value).ok()) + .filter(|size| (1..=10 * 1024 * 1024 * 1024).contains(size)) + .ok_or_else(invalid)?; + let descriptor = selected_descriptor(&inspected, docker_arch)?; + let registry = registry_manifest(reference, &inspected, descriptor.as_ref())?; + let config = inspected.config.ok_or_else(invalid)?; + if config + .volumes + .as_ref() + .is_some_and(|volumes| !volumes.is_empty()) + { + return Err(invalid()); + } + let defaults: ImageExecutionDefaults = ImageExecutionMetadata::builder() + .entrypoint(config.entrypoint.unwrap_or_default()) + .command(config.cmd.unwrap_or_default()) + .working_directory(config.working_dir.unwrap_or_default()) + .environment(config.env.unwrap_or_default()) + .user(config.user.unwrap_or_default()) + .build() + .try_into()?; + defaults.plan(overrides)?; + let inspection_hash = ImageMetadata::execution_fingerprint( + launch_id, + descriptor.as_ref(), + architecture, + size, + &defaults, + )?; + ImageMetadata::builder() + .requested_reference(reference.clone()) + .image_id(launch_id.clone()) + .selected_manifest(descriptor) + .registry_manifest(registry) + .os("linux".into()) + .architecture(architecture) + .inspection_hash(inspection_hash) + .size_bytes(size) + .execution(defaults) + .build() + .try_into() + .map_err(DockerError::from) +} + +fn selected_descriptor( + inspected: &ImageInspect, + architecture: &str, +) -> Result, DockerError> { + if inspected + .manifests + .as_ref() + .is_some_and(|manifests| !manifests.is_empty()) + { + return Err(invalid()); + } + let Some(descriptor) = &inspected.descriptor else { + return Ok(None); + }; + if !matches!( + descriptor.media_type.as_deref(), + Some( + "application/vnd.oci.image.manifest.v1+json" + | "application/vnd.docker.distribution.manifest.v2+json" + ) + ) || descriptor.artifact_type.is_some() + || descriptor.platform.as_ref().is_some_and(|platform| { + platform.os.as_deref().is_some_and(|os| os != "linux") + || platform + .architecture + .as_deref() + .is_some_and(|arch| arch != architecture) + }) + || descriptor.annotations.as_ref().is_some_and(|annotations| { + annotations + .get("vnd.docker.reference.type") + .is_some_and(|kind| kind == "attestation-manifest") + }) + { + return Err(invalid()); + } + Ok(Some( + descriptor.digest.as_deref().ok_or_else(invalid)?.parse()?, + )) +} + +fn repository_identity(reference: &ImageReference) -> String { + let (registry, repository) = + reference + .registry() + .map_or(("docker.io", reference.repository()), |registry| { + ( + registry, + reference + .repository() + .split_once('/') + .map_or("", |(_, repository)| repository), + ) + }); + let registry = registry.to_ascii_lowercase(); + let registry = match registry.as_str() { + "index.docker.io" => "docker.io", + value => value, + }; + if registry == "docker.io" && !repository.contains('/') { + format!("{registry}/library/{repository}") + } else { + format!("{registry}/{repository}") + } +} + +fn registry_manifest( + reference: &ImageReference, + inspected: &ImageInspect, + descriptor: Option<&Sha256Digest>, +) -> Result, DockerError> { + let digests = inspected.repo_digests.as_deref().unwrap_or_default(); + if digests.len() > 128 { + return Err(invalid()); + } + let repository = repository_identity(reference); + let mut matches = Vec::new(); + for candidate in digests { + let candidate: ImageReference = candidate.parse()?; + let digest: Sha256Digest = candidate.digest().ok_or_else(invalid)?.parse()?; + if repository_identity(&candidate) == repository { + matches.push(digest); + } + } + if let Some(requested) = reference.digest() { + if descriptor.is_none_or(|digest| digest.as_str() != requested) + && !matches.iter().any(|digest| digest.as_str() == requested) + { + return Err(invalid()); + } + } + if reference.registry().is_none() || matches.is_empty() { + return Ok(None); + } + let selected = if let Some(descriptor) = descriptor { + descriptor + } else if let Some(requested) = reference.digest() { + matches + .iter() + .find(|digest| digest.as_str() == requested) + .ok_or_else(invalid)? + } else { + matches + .iter() + .min_by(|left, right| left.as_str().cmp(right.as_str())) + .ok_or_else(invalid)? + }; + Ok(Some( + format!("{}@{}", reference.repository(), selected.as_str()).parse()?, + )) +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn digest(letter: char) -> String { + format!("sha256:{}", letter.to_string().repeat(64)) + } + fn inspection() -> Value { + json!({"Id":digest('a'),"Os":"linux","Architecture":"arm64","Size":1024, + "Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json", "digest":digest('b'), + "platform":{"os":"linux","architecture":"arm64"}}, + "RepoDigests":[format!("registry.test/team/worker@{}",digest('c'))], + "Config":{"Entrypoint":["/entry"],"Cmd":["handler"],"WorkingDir":"/task","Env":["SECRET=secret_value"],"User":"1000"}}) + } + fn selected(reference: &str, value: Value) -> Result { + resolve( + &reference.parse()?, + Architecture::Arm64, + None, + serde_json::from_value(value).unwrap(), + &digest('a').parse()?, + ) + } + + #[test] + fn test_should_pin_selected_child_and_preserve_validated_defaults() { + for suffix in [ + ":v1".to_owned(), + format!("@{}", digest('c')), + format!("@{}", digest('b')), + ] { + let image = + selected(&format!("registry.test/team/worker{suffix}"), inspection()).unwrap(); + assert_eq!(image.metadata().image_id.as_str(), digest('a')); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('b')) + ); + assert_eq!( + image.metadata().execution.plan(None).unwrap().entrypoint(), + ["/entry"] + ); + assert_eq!( + image.metadata().execution.metadata().environment, + ["SECRET=secret_value"] + ); + assert!(!format!("{image:?}").contains("secret_value")); + let restored: ResolvedImage = + serde_json::from_slice(&serde_json::to_vec(&image).unwrap()).unwrap(); + assert_eq!( + restored.metadata().execution.plan(None).unwrap().command(), + ["handler"] + ); + } + assert!( + selected( + &format!("registry.test/team/worker@{}", digest('d')), + inspection() + ) + .is_err() + ); + } + + #[test] + fn test_should_keep_short_local_provenance_and_ignore_unrelated_repository_digests() { + let mut value = inspection(); + value["RepoDigests"] = json!([format!("team/worker@{}", digest('c'))]); + let image = selected("team/worker:v1", value).unwrap(); + assert_eq!(image.resolved_uri(), format!("local://{}", digest('a'))); + let mut value = inspection(); + value["RepoDigests"] = json!([format!("other.test/team/worker@{}", digest('c'))]); + assert!( + selected("registry.test/team/worker:v1", value) + .unwrap() + .metadata() + .registry_manifest + .is_none() + ); + } + + #[rstest::rstest] + #[case("docker.io/library/python:v1", "python")] + #[case("index.docker.io/python:v1", "docker.io/library/python")] + #[case("Registry.test/team/worker:v1", "registry.test/team/worker")] + fn test_should_normalize_familiar_repository_evidence( + #[case] requested: &str, + #[case] repository: &str, + ) { + let mut value = inspection(); + value["RepoDigests"] = json!([format!("{repository}@{}", digest('c'))]); + let image = selected(requested, value).unwrap(); + assert_eq!( + image + .metadata() + .registry_manifest + .as_ref() + .unwrap() + .digest(), + Some(digest('b').as_str()) + ); + } + + #[test] + fn test_should_select_legacy_manifest_evidence_deterministically() { + let mut value = inspection(); + value.as_object_mut().unwrap().remove("Descriptor"); + value["RepoDigests"] = json!([ + format!("registry.test/team/worker@{}", digest('d')), + format!("registry.test/team/worker@{}", digest('c')) + ]); + let image = selected("registry.test/team/worker:v1", value.clone()).unwrap(); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('c')) + ); + let image = selected(&format!("registry.test/team/worker@{}", digest('d')), value).unwrap(); + assert_eq!( + image.resolved_uri(), + format!("registry.test/team/worker@{}", digest('d')) + ); + } + + #[rstest::rstest] + #[case("os")] + #[case("architecture")] + #[case("id")] + #[case("size_zero")] + #[case("size_negative")] + #[case("size_large")] + #[case("descriptor_index")] + #[case("descriptor_unknown")] + #[case("descriptor_digest")] + #[case("descriptor_arch")] + #[case("descriptor_os")] + #[case("descriptor_artifact")] + #[case("descriptor_attestation")] + #[case("digests_count")] + #[case("digests_invalid")] + #[case("digests_tag_only")] + #[case("config_missing")] + #[case("config_volumes")] + #[case("config_nul")] + #[case("config_empty_executable")] + fn test_should_reject_incompatible_or_ambiguous_image_metadata(#[case] field: &str) { + let mut value = inspection(); + match field { + "os" => value["Os"] = json!("windows"), + "architecture" => value["Architecture"] = json!("amd64"), + "id" => value["Id"] = json!("worker:v1"), + "size_zero" => value["Size"] = json!(0), + "size_negative" => value["Size"] = json!(-1), + "size_large" => value["Size"] = json!(10_u64 * 1024 * 1024 * 1024 + 1), + "descriptor_index" => { + value["Descriptor"]["mediaType"] = json!("application/vnd.oci.image.index.v1+json"); + } + "descriptor_unknown" => value["Descriptor"]["mediaType"] = json!("application/unknown"), + "descriptor_digest" => value["Descriptor"]["digest"] = json!("bad"), + "descriptor_arch" => value["Descriptor"]["platform"]["architecture"] = json!("amd64"), + "descriptor_os" => value["Descriptor"]["platform"]["os"] = json!("windows"), + "descriptor_artifact" => { + value["Descriptor"]["artifactType"] = json!("application/attestation"); + } + "descriptor_attestation" => { + value["Descriptor"]["annotations"] = + json!({"vnd.docker.reference.type":"attestation-manifest"}); + } + "digests_count" => { + value["RepoDigests"] = + json!(vec![ + format!("registry.test/team/worker@{}", digest('c')); + 129 + ]); + } + "digests_invalid" => value["RepoDigests"] = json!(["garbage@digest"]), + "digests_tag_only" => value["RepoDigests"] = json!(["registry.test/team/worker:v1"]), + "config_missing" => value["Config"] = Value::Null, + "config_volumes" => value["Config"]["Volumes"] = json!({"/data":{}}), + "config_nul" => value["Config"]["Cmd"] = json!(["bad\0"]), + "config_empty_executable" => { + value["Config"]["Entrypoint"] = json!([]); + value["Config"]["Cmd"] = json!([]); + } + _ => panic!("unknown test case"), + } + assert!(selected("registry.test/team/worker:v1", value).is_err()); + } + + #[test] + fn test_should_hash_only_normalized_base_execution_identity() { + let base = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let mut value = inspection(); + value["Config"]["Labels"] = json!({"ignored":"changed"}); + value["Created"] = json!("2025-01-01T00:00:00Z"); + value["RepoDigests"] = json!([ + format!("unrelated.test/other@{}", digest('e')), + format!("registry.test/team/worker@{}", digest('c')) + ]); + let same = selected("registry.test/team/worker:v2", value).unwrap(); + assert_eq!( + base.metadata().inspection_hash, + same.metadata().inspection_hash + ); + for field in ["Entrypoint", "Cmd", "Env", "User", "WorkingDir"] { + let mut value = inspection(); + value["Config"][field] = match field { + "Entrypoint" | "Cmd" => json!(["changed"]), + "Env" => json!(["SECRET=changed"]), + "User" => json!("1001"), + _ => json!("/changed"), + }; + let changed = selected("registry.test/team/worker:v1", value).unwrap(); + assert_ne!( + base.metadata().inspection_hash, + changed.metadata().inspection_hash + ); + } + let mut value = inspection(); + value["Config"]["Entrypoint"] = json!([]); + value["Config"]["Cmd"] = json!([]); + let overrides = ImageConfig { + command: Some(vec!["/provided".into()]), + ..Default::default() + }; + let image = resolve( + &"registry.test/team/worker:v1".parse().unwrap(), + Architecture::Arm64, + Some(&overrides), + serde_json::from_value(value).unwrap(), + &digest('a').parse().unwrap(), + ) + .unwrap(); + assert!(image.metadata().execution.plan(None).is_err()); + assert_eq!( + image + .metadata() + .execution + .plan(Some(&overrides)) + .unwrap() + .command(), + ["/provided"] + ); + } + #[test] + fn test_should_require_consistent_classic_evidence_and_reject_manifest_summaries() { + let mut value = inspection(); + value.as_object_mut().unwrap().remove("Descriptor"); + value["GraphDriver"] = json!({"Name":"overlay2","Data":{}}); + let inspected: ImageInspect = serde_json::from_value(value.clone()).unwrap(); + let classic: SystemInfo = serde_json::from_value( + json!({"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"]]}), + ) + .unwrap(); + verify_classic_store(&inspected, &classic).unwrap(); + for statuses in [ + json!({"Driver":"vfs"}), + json!({"Driver":"vfs","DriverStatus":null}), + ] { + let mut vfs = inspected.clone(); + vfs.graph_driver.as_mut().unwrap().name = "vfs".into(); + verify_classic_store(&vfs, &serde_json::from_value(statuses).unwrap()).unwrap(); + } + for name in ["btrfs", "zfs"] { + let mut containerd = inspected.clone(); + containerd.graph_driver.as_mut().unwrap().name = name.into(); + let info:SystemInfo = serde_json::from_value(json!({"Driver":name,"DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]})).unwrap(); + assert!(verify_classic_store(&containerd, &info).is_err()); + } + for info in [ + json!({"Driver":"overlayfs","DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]}), + json!({"Driver":"overlay2","DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]]}), + json!({"Driver":"overlay2","DriverStatus":[["malformed"]]}), + ] { + assert!( + verify_classic_store(&inspected, &serde_json::from_value(info).unwrap()).is_err() + ); + } + value["Manifests"] = json!([{"ID":digest('b'),"Descriptor":{},"Available":true,"Kind":"image","ImageData":null,"AttestationData":null,"Size":{"Content":1,"Total":1}}]); + let inspected: ImageInspect = serde_json::from_value(value).unwrap(); + assert!(verify_classic_store(&inspected, &classic).is_err()); + assert!(selected_descriptor(&inspected, "arm64").is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index fa8a9071..07da1b24 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -5,6 +5,7 @@ mod connection; mod framing; +mod image; mod types; use std::{ @@ -23,6 +24,7 @@ pub use framing::{DockerOutputReader, MAX_DOCKER_RECORD, OutputChannel, OutputCh use http::{Method, StatusCode}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; +pub use image::ImageResolver; use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, @@ -42,7 +44,7 @@ use typed_builder::TypedBuilder; pub use types::{ApiVersion, DockerError, DockerKey, DockerSocket}; use uuid::Uuid; -use crate::artifact::ImageReference; +use crate::artifact::{Architecture, ImageReference}; const MAX_BODY_BYTES: usize = 1024 * 1024; @@ -351,9 +353,47 @@ impl DockerPort { body: &ContainerCreateBody, deadline: Instant, ) -> Result { - let query = form_urlencoded::Serializer::new(String::new()) - .append_pair("name", name.as_str()) - .finish(); + self.create_container_inner(name, body, None, deadline) + .await + } + + /// Create from an immutable image ID using an explicit captured platform. + /// + /// # Errors + /// Returns bounded request, status, deadline or transport failure; the owner + /// must reconcile uncertain creation under the preallocated name/labels. + /// + /// ```no_run + /// # async fn example(port: &rustack_lambda_core::docker::DockerPort, key: &rustack_lambda_core::docker::DockerKey, body: &rustack_lambda_core::docker::models::ContainerCreateBody) { + /// let result = port.create_container_on_platform(key,body, + /// rustack_lambda_core::artifact::Architecture::Arm64, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_container_on_platform( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + architecture: Architecture, + deadline: Instant, + ) -> Result { + self.create_container_inner(name, body, Some(architecture), deadline) + .await + } + + async fn create_container_inner( + &self, + name: &DockerKey, + body: &ContainerCreateBody, + architecture: Option, + deadline: Instant, + ) -> Result { + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("name", name.as_str()); + if let Some(architecture) = architecture { + query.append_pair("platform", architecture.docker_platform()); + } + let query = query.finish(); decode( self.call( Operation::Create, diff --git a/crates/rustack-lambda-core/src/docker/tests.rs b/crates/rustack-lambda-core/src/docker/tests.rs index b57c7058..7fedef33 100644 --- a/crates/rustack-lambda-core/src/docker/tests.rs +++ b/crates/rustack-lambda-core/src/docker/tests.rs @@ -66,6 +66,9 @@ struct Daemon { } impl Daemon { fn new() -> Self { + Self::with_version("1.53") + } + fn with_version(version: &'static str) -> Self { let directory = tempfile::tempdir().unwrap(); let listener = UnixListener::bind(directory.path().join("docker.sock")).unwrap(); let (requests, received) = mpsc::channel(32); @@ -85,7 +88,7 @@ impl Daemon { // normal. Panics still fail the joined server task. if let Ok(incoming) = read_request(stream).await { if incoming.head.starts_with("GET /version ") { - incoming.reply(200, br#"{"ApiVersion":"1.53","MinAPIVersion":"1.44"}"#).await; + incoming.reply(200, format!(r#"{{"ApiVersion":"{version}","MinAPIVersion":"1.44"}}"#).as_bytes()).await; } else { let _ = requests.send(incoming).await; } } }); @@ -747,3 +750,95 @@ async fn test_should_reject_expired_and_excessive_deadlines_before_metadata_publ let decoded: Result = decode(response, Instant::now()); assert!(matches!(decoded, Err(DockerError::Deadline))); } + +#[rstest::rstest] +#[case("1.44", false)] +#[case("1.48", false)] +#[case("1.49", true)] +#[case("1.53", true)] +#[tokio::test] +async fn test_should_resolve_preloaded_images_with_versioned_platform_selection( + #[case] version: &'static str, + #[case] platform: bool, +) { + use crate::artifact::Architecture; + let mut daemon = Daemon::with_version(version); + let mut wire = daemon.wire().await; + let resolver = ImageResolver::new(wire.port()); + let reference = "registry.test:5000/team/worker:v1".parse().unwrap(); + let invalid_override = rustack_lambda_model::types::ImageConfig { + working_directory: Some("relative".into()), + ..Default::default() + }; + assert!(matches!( + resolver + .resolve_local( + &reference, + Architecture::Arm64, + Some(&invalid_override), + deadline() + ) + .await, + Err(DockerError::Configuration) + )); + assert!(daemon.requests.try_recv().is_err()); + let (image, ()) = tokio::join!( + resolver.resolve_local(&reference, Architecture::Arm64, None, deadline()), + async { + let request = daemon.next().await; + let target = request.head.split_whitespace().nth(1).unwrap(); + assert_eq!( + target, + format!("/v{version}/images/registry%2Etest%3A5000%2Fteam%2Fworker%3Av1/json") + ); + let id = format!("sha256:{}", "a".repeat(64)); + let body = serde_json::json!({"Id":id,"Os":"linux","Architecture":"arm64","Size":1024,"Config":{"Cmd":["/start"]},"GraphDriver":{"Name":"overlay2","Data":{}}}); + request + .reply(200, &serde_json::to_vec(&body).unwrap()) + .await; + if platform { + let request = daemon.next().await; + let target = request.head.split_whitespace().nth(1).unwrap(); + let (path, query) = target.split_once('?').unwrap(); + assert_eq!( + path, + format!("/v{version}/images/sha256%3A{}/json", "a".repeat(64)) + ); + let pairs: Vec<_> = form_urlencoded::parse(query.as_bytes()).collect(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs.first().unwrap().0, "platform"); + assert_eq!( + serde_json::from_str::(&pairs.first().unwrap().1).unwrap(), + serde_json::json!({"os":"linux","architecture":"arm64"}) + ); + let mut selected = body; + selected["Id"] = serde_json::json!(format!("sha256:{}", "b".repeat(64))); + selected["Descriptor"] = serde_json::json!({"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":format!("sha256:{}","b".repeat(64))}); + request + .reply(200, &serde_json::to_vec(&selected).unwrap()) + .await; + } else { + let request = daemon.next().await; + assert!(request.head.starts_with(&format!("GET /v{version}/info "))); + request + .reply( + 200, + br#"{"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"]]}"#, + ) + .await; + } + } + ); + let image = image.unwrap(); + assert_eq!(image.metadata().architecture, Architecture::Arm64); + assert_eq!( + image.metadata().image_id.as_str(), + format!("sha256:{}", "a".repeat(64)) + ); + assert_eq!( + image.metadata().execution.plan(None).unwrap().entrypoint(), + [""] + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/types.rs b/crates/rustack-lambda-core/src/docker/types.rs index 4ada90d0..fe3b0751 100644 --- a/crates/rustack-lambda-core/src/docker/types.rs +++ b/crates/rustack-lambda-core/src/docker/types.rs @@ -7,6 +7,7 @@ use rustack_core::json::JsonError; use tokio::time::{Instant, timeout_at}; use super::check_deadline; +use crate::artifact::ArtifactError; /// Docker boundary failure classes; no peer response body is displayed. #[derive(Clone, thiserror::Error)] @@ -57,6 +58,9 @@ pub enum DockerError { /// Bounded metadata validation failed. #[error(transparent)] Json(#[from] JsonError), + /// Image inspection cannot establish a valid executable artifact. + #[error(transparent)] + Image(#[from] ArtifactError), } impl fmt::Debug for DockerError { diff --git a/crates/rustack-lambda-core/src/function_config.rs b/crates/rustack-lambda-core/src/function_config.rs index 692c5fc9..d580bea6 100644 --- a/crates/rustack-lambda-core/src/function_config.rs +++ b/crates/rustack-lambda-core/src/function_config.rs @@ -201,7 +201,7 @@ fn validate_layout( Ok(()) } -fn validate_image_config(config: &ImageConfig) -> Result<(), LambdaServiceError> { +pub(crate) fn validate_image_config(config: &ImageConfig) -> Result<(), LambdaServiceError> { let mut bytes = 0usize; for args in [config.entry_point.as_deref(), config.command.as_deref()] .into_iter() diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs index f260183c..1976e3b8 100644 --- a/crates/rustack-lambda-core/tests/docker_wire.rs +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -9,9 +9,10 @@ use anyhow::{Context, Result, bail, ensure}; use bytes::Bytes; use http::StatusCode; use rustack_lambda_core::{ - artifact::{ImageReference, Sha256Digest}, + artifact::{Architecture as ImageArchitecture, ImageReference, ResolvedImage}, docker::{ - DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, OutputChannel, OwnerFilter, + DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, ImageResolver, OutputChannel, + OwnerFilter, models::{ContainerCreateBody, HostConfig}, }, }; @@ -125,17 +126,20 @@ impl Environment { }) } - async fn image(port: &DockerPort, reference: &str, architecture: &str) -> Result { + async fn image( + port: &DockerPort, + reference: &str, + architecture: &str, + ) -> Result { let reference: ImageReference = reference.parse()?; - let image = port.inspect_image(&reference, deadline()).await?; - ensure!( - image.os.as_deref() == Some("linux") - && image.architecture.as_deref() == Some(architecture), - "fixture image platform mismatch" - ); - let id = image.id.context("image content id")?; - let _: Sha256Digest = id.parse()?; - Ok(id) + let architecture = match architecture { + "arm64" => ImageArchitecture::Arm64, + "amd64" => ImageArchitecture::X86_64, + _ => bail!("unknown fixture architecture"), + }; + Ok(ImageResolver::new(port.clone()) + .resolve_local(&reference, architecture, None, deadline()) + .await?) } async fn start(&mut self) -> Result<()> { @@ -155,10 +159,15 @@ impl Environment { ); let relay = Self::image(&port, &relay, &arch).await?; let function = Self::image(&port, &format!("rustack-p0-handler:{arch}"), &arch).await?; - port.create_container( + let relay_plan = relay.metadata().execution.plan(None)?; + let function_plan = function.metadata().execution.plan(None)?; + port.create_container_on_platform( &self.names.relay, &ContainerCreateBody { - image: Some(relay), + image: Some(relay.metadata().image_id.as_str().into()), + entrypoint: Some(relay_plan.entrypoint().to_vec()), + cmd: Some(relay_plan.command().to_vec()), + working_dir: Some(relay_plan.working_directory().into()), user: Some("1000:1000".into()), env: Some(vec![ format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={}", self.names.environment), @@ -175,6 +184,7 @@ impl Environment { host_config: Some(limits("none".into(), 64 * 1024 * 1024, 100_000_000, 64)), ..Default::default() }, + relay.metadata().architecture, deadline(), ) .await?; @@ -185,10 +195,13 @@ impl Environment { client.next(deadline()).await? == ClientEvent::RelayReady, "relay handshake mismatch" ); - port.create_container( + port.create_container_on_platform( &self.names.function, &ContainerCreateBody { - image: Some(function), + image: Some(function.metadata().image_id.as_str().into()), + entrypoint: Some(function_plan.entrypoint().to_vec()), + cmd: Some(function_plan.command().to_vec()), + working_dir: Some(function_plan.working_directory().into()), user: Some("1000:1000".into()), labels: Some(self.names.labels()), env: Some(vec![ @@ -207,6 +220,7 @@ impl Environment { )), ..Default::default() }, + function.metadata().architecture, deadline(), ) .await?; @@ -379,3 +393,77 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra let cleanup = environment.cleanup().await; result.and(cleanup) } + +#[tokio::test] +#[ignore = "requires explicit Docker wire qualification images/socket"] +async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() -> Result<()> { + let mut environment = Environment::new().await?; + let result = async { + let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; + let port = environment.wire.port(); + let image = Environment::image(&port, &format!("rustack-p0-handler:{arch}"), &arch).await?; + for reset in [false, true] { + let overrides = rustack_lambda_model::types::ImageConfig { + entry_point: reset.then(Vec::new), + command: Some(if reset { + vec!["/bin/true".into()] + } else { + vec![] + }), + ..Default::default() + }; + let plan = image.metadata().execution.plan(Some(&overrides))?; + port.create_container_on_platform( + &environment.names.function, + &ContainerCreateBody { + image: Some(image.metadata().image_id.as_str().into()), + entrypoint: Some(plan.entrypoint().to_vec()), + cmd: Some(plan.command().to_vec()), + working_dir: Some(plan.working_directory().into()), + labels: Some(environment.names.labels()), + user: Some("1000:1000".into()), + host_config: Some(limits("none".into(), 192 * 1024 * 1024, 1_000_000_000, 256)), + ..Default::default() + }, + image.metadata().architecture, + deadline(), + ) + .await?; + let actual = port + .inspect_container(&environment.names.function, deadline()) + .await? + .config + .context("actual create config")?; + if reset { + ensure!( + actual.entrypoint.as_deref().unwrap_or_default().is_empty(), + "daemon restored a cleared entrypoint" + ); + ensure!( + actual + .cmd + .as_deref() + .is_some_and(|command| command == ["/bin/true"]), + "reset command mismatch" + ); + } else { + ensure!( + actual + .entrypoint + .as_deref() + .is_some_and(|entrypoint| entrypoint == plan.entrypoint()), + "inherited entrypoint mismatch" + ); + ensure!( + actual.cmd.as_deref().unwrap_or_default().is_empty(), + "daemon restored a cleared command" + ); + } + remove(&port, &environment.names.function).await?; + } + Ok(()) + } + .await; + let cleanup = environment.cleanup().await; + result.and(cleanup) +} diff --git a/docs/index.md b/docs/index.md index 35becd5c..7d76b092 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ ## Research -- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation and actual RIC/diagnostic qualification. +- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation, platform identity/default merge validation and actual RIC/diagnostic qualification; Bollard and historical Moby sources are pinned. - [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. - [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. - [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. diff --git a/docs/research/study-bollard-docker-transport-boundaries.md b/docs/research/study-bollard-docker-transport-boundaries.md index 0ec735d5..14df184c 100644 --- a/docs/research/study-bollard-docker-transport-boundaries.md +++ b/docs/research/study-bollard-docker-transport-boundaries.md @@ -1,6 +1,6 @@ # Study: Bollard's Docker Response and Attach Boundaries -Status: pinned source review complete; bounded wire implementation and qualification recorded below · Date:2026-10-25 · Vendor pin: `vendors/bollard` @ `f9ec79e7546edfb674b1066c44a68815cb52251c` (`v0.21.1`) +Status: pinned source review complete; bounded wire implementation and qualification recorded below · Date:2026-10-25 · Vendor pins: `vendors/bollard` @ `f9ec79e7546edfb674b1066c44a68815cb52251c` (`v0.21.1`); `vendors/moby` @ `af898abe44662d9554fb15ee4d4a7307f1b8e315` (`v28.0.0`, historical compatibility/merge source) ## Question @@ -81,3 +81,25 @@ The current workspace and standalone probe align on [Tokio1.54.1](https://docs.r A retained compatibility regression calls the actual pinned Bollard0.21.1 create and logs APIs against a fake local Unix daemon under TRACE with LogTracer enabled. It covers synchronous stream construction, asynchronous request/response polls and the decoder's truncated-EOF Console fallback. Test-only markers in request environment, response warnings, container target and trailing bytes are absent from infrastructure capture while an application event remains present. The production boundary independently captures create requests, response bytes/headers and fragmented trailers. These are executed call-site tests, not only synthetic logging-target tests. The explicit `make test-lambda-docker-wire` target builds the real relay and unchanged official Python RIC image and uses DockerWire for daemon IO without the Docker CLI. It sends and receives6MiB-minus4096-byte-class data through actual multiplexed attach, checks computed warm results and a handler error, closes/joins the relay stream and verifies the run-label scope is empty after removal. Stress qualification uses60s IO/120s attempt budgets; it does not establish default5s latency or throughput. The existing five-test `test-lambda-runtime-docker` suite remains a separate relay-contract regression. Exact platform results and current review state are recorded in [verification](../../specs/rustack-workflows-70-verification-plan.md) and [implementation review](../../specs/rustack-workflows-93-review.md). + +## Platform-specific image identity follow-up + +The2026-10-25 Engine29.5.2/containerd-store inspection of the local official-RIC fixture disproved an overly broad reading of the generated Id field's historical config-ID description. Unqualified inspect returned an OCI index descriptor and the same digest as Id. Platform-qualified inspect returned a different Id and a single image-manifest descriptor for Linux arm64, while RepoDigests still named the parent index. The manifests view also included a separate attestation descriptor. Neither the parent Id nor the first RepoDigests string alone identifies the selected executable child. + +The [official version history](https://docs.docker.com/reference/api/engine/version-history/#v149-api-changes) and [CLI inspect documentation](https://docs.docker.com/reference/cli/docker/image/inspect/) pin platform-qualified inspection to API1.49+. The platform argument is JSON-encoded OCI platform data and cannot be combined with manifests=true. Generated contracts distinguish descriptor/manifest/platform data at `vendors/bollard/codegen/swagger/src/models.rs:4006`, `:4129` (manifest summary), and the typed image inspect query in `vendors/bollard/codegen/swagger/src/query_parameters.rs`. Descriptor fields and RepoDigests are daemon metadata, not evidence of a successful registry push: BuildKit load can populate RepoDigests even for a short, unpushed local name. + +Downstream resolution must request the declared platform when available, validate Linux architecture and an executable effective entry configuration, and reject an unresolved index or attestation as executable metadata. Older compatible API lanes may inspect legacy single-platform images but cannot silently accept an unresolved index. A registry URI must distinguish its repository/index evidence from the selected child descriptor; short local references retain the explicit local URI extension. The implementation contract and tests belong to runtime §2.8/D21. This is identity correctness evidence, not qualification of arbitrary cross-architecture emulation or registry pulls. + +Independent preflight also identified the Docker create merge boundary: an empty request entrypoint can cause an empty command to be filled from the image. The future create adapter must consume fully materialized validated argv/workdir, with the empty-entrypoint reset sentinel, rather than pass only optional function overrides. Capturing only a metadata hash would lose the defaults needed to do this without per-invoke inspection. Repository evidence comparison also needs familiar Docker Hub name normalization while retaining the original short-local provenance decision. These refinements are recorded as W33/W34 in the review backlog and pinned in §2.8/D21. + +The first consumer qualification deliberately tried create by the platform-inspect child Id and failed404 on both retained daemons. A read-only lookup of that still-current child digest also returns404, while the current unqualified parent Id is addressable. Resolution must therefore capture the addressable parent before selecting a platform and create by that immutable parent plus an explicit platform parameter. The selected manifest remains independent fingerprint/registry metadata; it must not be discarded. Rebuilding a mutable tag can also replace its index descriptor, so a later deployment owner must retain artifact availability under its own lifetime policy; missing externally removed content must fail rather than resolve a newer tag. + +Independent implementation review additionally requires recomputing saved fingerprints after valid-string mutations, and rejects descriptor-free fallback without positive classic-store /info evidence. A downlevel containerd reply can omit Descriptor while still exposing a nonempty GraphDriver. Classic names must match between inspect and info, manifest summaries must not contradict the fallback, and containerd's driver-type status marker must reject it. These qualifications are explicit compatibility checks, not a claim that an old containerd/GraphDriver spelling proves classic behavior. + +### Pinned engine source for merge and legacy-store guards + +The newly vendored Moby v28.0.0 is a historical API1.48 source pin, not the source version of the live Engine29.5.2 daemons. Its config merge fills an empty command when the request entrypoint has length zero at `vendors/moby/daemon/commit.go:72`; `vendors/moby/daemon/create.go:302` resets the single-empty-string entrypoint only after merge. This explains why materialized nonempty/reset entrypoints preserve an intentionally empty command. Actual modern-daemon create/inspect regressions verify both clear-command and reset-entrypoint cases. + +The same historical source strips Descriptor below API1.48 at `vendors/moby/api/server/router/image/image_routes.go:374`. Containerd nevertheless supplies a GraphDriver name from its snapshotter at `vendors/moby/daemon/containerd/image_inspect.go:117`; a nonempty GraphDriver cannot alone prove classic storage. The info status marker is emitted at `vendors/moby/daemon/containerd/service.go:112`, with the exact io.containerd.snapshotter.v1 value pinned at `vendors/moby/vendor/github.com/containerd/containerd/v2/plugins/types.go:39`. Classic VFS returns nil status at `vendors/moby/daemon/graphdriver/vfs/driver.go:87`. Normalize absent/null DriverStatus after matching classic driver evidence, while rejecting the explicit containerd marker and malformed rows. Fake-daemon/pure metadata tests cover these paths; no execution qualification of an old daemon is claimed. + +The final local resolver retains validated execution defaults in Arc storage, recomputes a canonical fingerprint on restore, binds platform inspection to the captured addressable parent Id and passes that Id with explicit platform to actual create. Its61 additional tests cover metadata/defaults/legacy API behavior and valid-value stale-hash mutation. Both real two-case Docker wire suites pass on arm64 VZ and amd64/QEMU. Full workspace1964 tests and standard build/fmt/Clippy/rustdoc pass. Deployment actors, availability leases, allowed pulls, unpacked-image quota proof and pool/provider integration still belong to the remaining P1 work. diff --git a/specs/index.md b/specs/index.md index 8c84b3af..96dd3f88 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO is also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO and platform-aware local image resolution are also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 0e4d7d49..9e80346f 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -123,6 +123,39 @@ Caller / HTTP WorkSupervisor actor Backend supervisor │◀── CleanupFailed ────│ prevent false successful quiescence │ ``` +### 2.8 Local image resolution and selected platform identity + +ImageResolver::new(DockerPort) exposes resolve_local(reference, architecture, ImageConfig overrides, deadline) → ResolvedImage. ImageMetadata also owns validated ImageExecutionDefaults (normalized inherited argv/workdir/environment/user); captures and snapshots retain these values, not only their hash. Debug redacts them and deserialization revalidates their bounds. It performs inspection only, never pulls, builds, reads credentials or mutates daemon resources. The deployment actor owns its bounded work slot and publication/CAS; later allowed-pull resolution must reuse this validation path. Cancellation or expiry of this read-only operation cannot publish a late result. Recheck the absolute clock before and after metadata validation/fingerprinting. + +First inspect the requested reference once and capture its canonical, addressable daemon Id. On negotiated API≥1.49, inspect that captured immutable Id with a percent-encoded JSON OCI platform containing exactly os=linux and architecture=amd64/arm64, avoiding a second lookup of the mutable tag. Retain the addressable image_id and optional selected_manifest digest separately: a selected child Id returned by containerd inspection is not necessarily independently addressable for create. Every environment create supplies the captured image_id plus the explicit linux/amd64 or linux/arm64 platform parameter; it never substitutes an unqualified tag or assumes the child digest is a local image object. Older APIs retain single-platform inspection only with verified classic-store evidence. Validate the resulting OS/architecture, positive size≤10GiB and canonical SHA-256 Id. If a Descriptor is present, accept only OCI image-manifest or Docker schema2 image-manifest media types, canonical digest and a compatible optional platform. Reject index/manifest-list/attestation/unknown descriptors. A platform-selected containerd Id may be a child manifest digest rather than the legacy config digest. Validate that returned Id, but use the captured addressable parent Id for create and retain the selected descriptor digest for identity/hash checks. Never create from a parent index without the declared platform. Reject explicit nonempty Manifests in the descriptor-free fallback. Require matching classic GraphDriver and /info driver names from overlay2/aufs/btrfs/devicemapper/vfs/zfs/fuse-overlayfs, well-formed bounded DriverStatus rows and no io.containerd.snapshotter.v1 driver-type marker. Normalize missing/null DriverStatus to empty after matching classic evidence, since classic VFS legitimately reports no status rows. A missing Descriptor or nonempty GraphDriver alone is insufficient. Ambiguous/downlevel containerd resolution fails explicitly rather than choosing the daemon default platform. + +RepoDigests has≤128 validated digest references. Repository identity comparison applies Docker familiar-name normalization: absent registry defaults to docker.io, index.docker.io aliases docker.io, and a one-component Docker Hub repository gains library/. Registry host comparison is ASCII case-insensitive; repository paths remain case-sensitive. The original request spelling determines the short-local URI extension before normalization. A requested digest must match the selected descriptor digest or an advertised parent RepoDigest for the same repository; otherwise fail inspection. For explicit registry-domain references with matching repository evidence, use that repository plus the selected descriptor digest as registry_manifest; a legacy descriptor-free image may use a matching manifest RepoDigest, choosing the requested digest or otherwise a deterministic lexical digest. Do not use a digest from an unrelated alias/repository. A short local reference always uses registry_manifest=None and local://Id, even if BuildKit has populated RepoDigests. The returned registry URI reflects daemon metadata, not a network publication/existence check. The captured addressable Id must remain available through the deployment/version/artifact lifetime. The deployment actor still needs an owned availability lease/pin policy; tag movement must never cause it to execute a newer image. Missing externally removed content fails explicitly. CodeSize/size_bytes uses the selected daemon-reported Size, whose containerd representation can be compressed content rather than unpacked size. The deployment actor must separately prove the10GiB unpacked-image quota before Active; this remains a task1.3 gate and is not established by the read-only local resolver. + +Validate inherited ENTRYPOINT/CMD with the same argv limits as function ImageConfig, preserving omitted versus explicitly empty overrides. Docker's single-empty-entrypoint reset means no entrypoint. Compose the effective entrypoint+command, require its first executable argument to be nonempty, and reject NULs/bounds before publication; do not require a shell or bootstrap name. An absent/empty inherited working directory means Docker's root default; an explicit override still must be absolute. Inherited image Env is≤128 entries,≤4KiB per entry and≤64KiB total, with no NUL; image Env is distinct from the function's stricter4KiB environment budget. Validate a bounded ASCII environment key of1..256 bytes before `=` (or a key-only deletion marker); preserve ordered values for Docker semantics. Image USER is bounded≤256 printable ASCII bytes. Reject image-declared VOLUMEs in this local profile: automatic anonymous writable mounts would bypass rootfs/tmpfs policy. The eventual container builder must also disable image healthchecks and apply the specified nonroot-user policy. + +ImageExecutionDefaults::plan(overrides) validates the complete composed argv, including combined inherited/overridden byte budgets, and returns an immutable redacted ImageExecutionPlan. The plan materializes ENTRYPOINT, CMD and WORKDIR; it never relies on daemon fallback. Encode a semantic empty ENTRYPOINT as the Docker reset sentinel [""], preserve a nonempty entrypoint, and always send the full command array even when empty. This prevents Docker from re-inheriting CMD when the request entrypoint is empty. No per-invoke inspect is permitted. Empty inherited argv can be supplied by valid function overrides, but malformed inherited strings/directories remain invalid even when no default executable exists. Reject saved image metadata without validated execution defaults; older metadata-only records require deployment resolution rather than pretending to be executable. + +```text +┌─ read-only image resolution ─────────────────────────────────────────┐ +│ bounded platform inspect → child identity + validated defaults │ +│ │ stable base fingerprint │ +└──────────────────────────────────────┼───────────────────────────────┘ + ▼ +┌─ immutable ResolvedImage / captured revision / snapshot ─────────────┐ +│ Id + platform + URI evidence + defaults (argv/env/user redacted) │ +│ │ captured function overrides │ +│ ▼ │ +│ validated materialized plan │ +└────────────────────────────────────┼────────────────────────────────┘ + ▼ +┌─ future environment owner ──────────────────────────────────────────┐ +│ create by selected Id; full argv/workdir; empty entrypoint=[""] │ +│ security/resource policy + fixed environment injection separately │ +└────────────────────────────────────────────────────────────────────┘ +``` + +Fingerprint a versioned, explicitly ordered serialization of captured addressable Id, retained selected_manifest digest, Linux architecture, inspected size and normalized inherited execution fields (entrypoint, command, root-default workdir, ordered environment, user). Exclude tags, unrelated RepoDigests, labels, creation timestamps and map iteration order. ImageMetadata::execution_fingerprint centralizes bounded SHA-256 serialization; ResolvedImage construction/deserialization recomputes it and rejects inconsistent metadata, including valid-string execution-plan mutations. No raw argv/environment or metadata is logged. Function overrides remain captured separately in ExecutionRevision and do not change the base inspection fingerprint. Invalid overrides fail before daemon IO. Fake-daemon tests cover platform query/version fallback, every rejection, deterministic identity under tag/label/RepoDigest ordering changes and explicit-empty overrides. Real wire qualification must create/run the function and relay using identities returned by this resolver. + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 77d12ec9..5641ee08 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -79,6 +79,8 @@ The shared runtime contract suite is available as `make test-lambda-runtime`; it Current wire qualification on2026-10-25 passes on Engine29.5.2 Linux arm64 in native VZ and Linux amd64 under QEMU. Both near6MiB cases pass with60s IO/120s attempt stress budgets. These do not establish the default5s IO latency budget, native x86 performance, Docker Desktop/rootless support or full Lambda provider/pool/deployment acceptance. Independent daemon queries find no remaining containers bearing the wire harness ownership label after the runs. +Local image resolution now extends `test-lambda-docker-wire` with a second real create/inspect case: clear inherited CMD and reset ENTRYPOINT without daemon re-inheritance. The execution case uses the captured addressable parent Id plus explicit platform, and preserves the selected child manifest independently.61 unit additions cover validated inherited defaults/plans, deterministic base fingerprints, ten valid-value stale-hash snapshot mutations, familiar registry identities, actual captured-parent query encoding and legacy API1.44/1.48 classic evidence. Null/missing VFS status remains valid; explicit containerd markers and contradictory manifest summaries reject. Historical engine code is pinned at vendors/moby v28.0.0; fake older-version cases are not old-daemon execution qualification. Current two-case suites pass on both retained daemons. Local resolution does not establish deployment status/CAS, availability leases, allowed registry pulls or unpacked-image quota enforcement. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index d592a652..1496cfb1 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,7 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Deployment/image actors, remaining stream/network/probe consumers and environment/pool/provider integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Deployment/image actors (including status/CAS, availability leases, allowed pulls and unpacked quota proof), remaining stream/network/probe consumers and environment/pool/provider integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement task1.3 deployment/image and environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts, together with remaining stream/network/probe operations, followed by startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 92f9752b..ab078842 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -159,6 +159,25 @@ W01's implemented filter now covers Hyper/hyper_util/Bollard independent of appl Deployment/image actors, remaining stream/network/probe consumers, warm pooling, startup/resource checks, full SDK/provider acceptance and B01/B03 remain required before P1 closes. W30 is closed for implemented wire admission/handoff; future log/pull owners remain subject to its existing-reservation rule. W31's source citations remain pinned and checked. +## P1 task1.3 local image resolution review + +Independent design and implementation review found the issues below. All are fixed, with a final read-only recheck reporting no remaining concrete in-slice finding. The [source study](../docs/research/study-bollard-docker-transport-boundaries.md#platform-specific-image-identity-follow-up) pins the actual observations and historical Moby merge/store source. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W33 | P2; runtime §2.8 / `artifact/execution.rs:38` | A hash alone discards the defaults needed to preserve empty overrides without per-invoke inspection. ResolvedImage now retains validated Arc defaults and returns a fully materialized redacted Docker plan with the empty-entrypoint reset sentinel | Default/override/composed-budget tests, snapshot round trips and real create/inspect merge checks | +| W34 | P3; `docker/image.rs`, repository matching | Raw repository equality loses familiar Docker Hub digest evidence. Normalize default docker.io/library and index.docker.io, while classifying short-local provenance from original spelling | Familiar-name, registry-case, port/IPv6, unrelated-alias and short-local cases | +| W35 | P2; `artifact/image.rs:443` | Valid saved execution changes could retain a stale inspection hash; the selected descriptor digest was not retained. Metadata now stores selected_manifest and the constructor/deserializer recomputes centralized canonical SHA-256 over parent/child/platform/size/defaults | Ten valid-value mutation cases reject stale fingerprints; immutable defaults and Debug remain redacted | +| W36 | P2; `docker/image.rs:206` | Missing Descriptor alone can misclassify downlevel containerd as classic, including contradictory manifest summaries. Require matching classic inspect/info driver evidence, reject the containerd status marker, and reject explicit manifest summaries | API1.44/1.48 classic fallback, modern captured-parent selection, ambiguous summaries, malformed rows and btrfs/zfs marker negatives | +| W37 | P2; `docker/image.rs:108` | Requiring nonnull DriverStatus rejects legitimate classic VFS. Normalize absent/null rows after classic evidence, retaining marker/shape rejection | Missing/null VFS positives and containerd negatives; historical source pinned | +| W38 | P2; actual Docker qualification | Platform-qualified child Id is not independently addressable by lookup/create on the retained containerd store. Bind the requested tag once, platform-inspect its captured parent Id, retain child manifest separately and create from immutable parent plus explicit platform | Initial404 reproduced on both daemons; corrected two-case suites pass on each. Fake parent-a/child-b cases prove the second lookup cannot re-resolve a moved tag | + +The resolver performs bounded read-only local inspection only. It validates selected Linux image metadata, inherited argv/environment/user/working-directory bounds, effective entry overrides and local/registry URI representation. No registry pull, credential discovery or image build occurs.61 new tests pass within288 Lambda core tests;58 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,964 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. + +Actual official-RIC wire execution now consumes resolver identities and materialized plans. Near6MiB computed warm calls/errors and a separate clear-command/reset-entrypoint create test pass on both arm64 VZ and amd64/QEMU, with independent stream/container cleanup. Default IO latency, old-daemon execution, native x86/Desktop/rootless and full provider/executor acceptance remain outside this evidence. Strict production Clippy has the same65 existing B03 findings, with none in the new resolver/defaults paths. Hex reuses an existing workspace version/package; audit still reports the16 B01 advisories and deny bans/licenses/sources pass without new ignores. + +Deployment/status/CAS actors, image availability leases, allowed pulls, unpacked-image quota proof, environment/pool owners, remaining network/log/probe consumers and B01/B03 remain required before P1 closes. This slice does not advertise Docker invocation in Rustack. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 28c0b291..1d74e8b5 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -159,6 +159,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [Docker request ownership and quotas](./rustack-workflows-20-lambda-runtime-design.md#36-bounded-docker-api-boundary); executable held-wait and business-saturation regressions in the Docker wire tests. - Date:2026-10-25. The wire lane does not itself own container lifetime or prove successful cleanup. +## D21 — Resolve the selected executable image, not its parent index + +- Context: the live Engine29.5.2/containerd image store returns an index Id for unqualified inspection and a child-manifest Id for platform-qualified inspection; RepoDigests may continue naming the parent index, including for unpushed BuildKit-loaded local images. +- Alternatives: always use inspect Id without a platform; take the first RepoDigest; require a registry round trip for every local deployment; request/validate the selected platform and keep local/registry representation explicit. +- Decision: platform-qualified read-only local resolution validates executable child identity, retains validated inherited execution defaults and derives an explicit Docker-create plan preserving empty overrides, rejects unresolved index descriptors, and derives registry child references only from matching repository evidence. Capture the addressable parent Id before platform inspection, retain the selected child descriptor independently, and create by immutable parent Id plus explicit platform; the selected child Id is not necessarily a locally addressable image object. Recompute the canonical execution fingerprint during restore. Short local names retain local://Id. Legacy descriptor-free images remain supported on older compatible APIs. +- Why: deployment pinning must survive tag movement and daemon default-platform changes without inventing registry provenance. Inspection hashes must also be deterministic across irrelevant metadata/map ordering. +- Pinned by: [local resolution](./rustack-workflows-20-lambda-runtime-design.md#28-local-image-resolution-and-selected-platform-identity), [live/source evidence](../docs/research/study-bollard-docker-transport-boundaries.md#platform-specific-image-identity-follow-up). +- Date:2026-10-25. This refines D19's generated Id representation; it does not promise registry existence, cross-platform execution or completed deployment/provider integration. + ## Glossary | Term | Meaning in this specification | diff --git a/vendors/moby b/vendors/moby new file mode 160000 index 00000000..af898abe --- /dev/null +++ b/vendors/moby @@ -0,0 +1 @@ +Subproject commit af898abe44662d9554fb15ee4d4a7307f1b8e315 From 25a1199ac180d5c919d7fd110e886ba2eb2b0d5c Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 21:38:32 -0700 Subject: [PATCH 014/106] phase 1: retain owned image pins and validate unpacked deployment artifacts Own bounded image preparation, live artifact leases and phase-aware cleanup through cancellation and supervision failure. Preserve uncertain nested Docker mutations until a commit is observed; validate selected-platform unpacked image quotas without running code. Verified 1,997 workspace tests, 321 Lambda core cases, 76 core doctests, build/fmt/Clippy/rustdoc and two actual Docker cases on each arm64 VZ and amd64 QEMU. Independent review fixes W39-W42. B01 advisories and B03 strict baseline findings remain mandatory P1 work; provider deployment, pool/executor and later workflow phases remain incomplete. --- crates/rustack-lambda-core/src/artifact.rs | 4 + .../rustack-lambda-core/src/artifact/image.rs | 6 +- .../src/artifact/pinned.rs | 175 ++++ .../rustack-lambda-core/src/docker/image.rs | 183 +++- .../rustack-lambda-core/src/docker/lease.rs | 907 ++++++++++++++++++ .../src/docker/lease_tests.rs | 826 ++++++++++++++++ crates/rustack-lambda-core/src/docker/mod.rs | 17 +- crates/rustack-lambda-core/src/docker/pin.rs | 212 ++++ .../rustack-lambda-core/src/docker/tests.rs | 22 +- .../rustack-lambda-core/src/docker/types.rs | 6 + .../rustack-lambda-core/tests/docker_wire.rs | 85 +- docs/index.md | 2 +- ...udy-bollard-docker-transport-boundaries.md | 14 + specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 32 + .../rustack-workflows-60-operations-design.md | 4 +- .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 6 +- specs/rustack-workflows-93-review.md | 19 + specs/rustack-workflows-99-decisions.md | 9 + 20 files changed, 2474 insertions(+), 59 deletions(-) create mode 100644 crates/rustack-lambda-core/src/artifact/pinned.rs create mode 100644 crates/rustack-lambda-core/src/docker/lease.rs create mode 100644 crates/rustack-lambda-core/src/docker/lease_tests.rs create mode 100644 crates/rustack-lambda-core/src/docker/pin.rs diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs index d762d08c..9e6ef2a2 100644 --- a/crates/rustack-lambda-core/src/artifact.rs +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -5,6 +5,7 @@ mod execution; mod image; +mod pinned; use std::{ fmt::{Debug, Formatter, Result as FmtResult}, @@ -18,9 +19,12 @@ pub use execution::{ImageExecutionDefaults, ImageExecutionMetadata, ImageExecuti pub use image::{ Architecture, ArtifactError, ImageMetadata, ImageReference, ResolvedImage, Sha256Digest, }; +pub use pinned::{PinnedImage, UnpackedImageSize}; use tempfile::TempDir; use crate::storage::compute_sha256; + +pub(crate) const MAX_IMAGE_BYTES: u64 = 10 * 1024 * 1024 * 1024; #[cfg(test)] use crate::{storage::stage_zip, test_zip::minimal_zip}; diff --git a/crates/rustack-lambda-core/src/artifact/image.rs b/crates/rustack-lambda-core/src/artifact/image.rs index 6d4d1fa1..b343ebf5 100644 --- a/crates/rustack-lambda-core/src/artifact/image.rs +++ b/crates/rustack-lambda-core/src/artifact/image.rs @@ -11,7 +11,7 @@ use winnow::{ token::take_while, }; -use super::ImageExecutionDefaults; +use super::{ImageExecutionDefaults, MAX_IMAGE_BYTES}; /// Closed Lambda architecture, also used to select a Docker platform. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -416,7 +416,7 @@ impl ImageMetadata { size: u64, defaults: &'a ImageExecutionDefaults, } - if !(1..=10 * 1024 * 1024 * 1024).contains(&size) { + if !(1..=MAX_IMAGE_BYTES).contains(&size) { return Err(ArtifactError::Inspection); } // Validated defaults contain <=128KiB of argv/environment, whose worst @@ -445,7 +445,7 @@ impl TryFrom for ResolvedImage { fn try_from(metadata: ImageMetadata) -> Result { if metadata.os != "linux" || metadata.size_bytes == 0 - || metadata.size_bytes > 10 * 1024 * 1024 * 1024 + || metadata.size_bytes > MAX_IMAGE_BYTES || metadata .registry_manifest .as_ref() diff --git a/crates/rustack-lambda-core/src/artifact/pinned.rs b/crates/rustack-lambda-core/src/artifact/pinned.rs new file mode 100644 index 00000000..e81a63fb --- /dev/null +++ b/crates/rustack-lambda-core/src/artifact/pinned.rs @@ -0,0 +1,175 @@ +//! Live image artifacts cannot be reconstructed from snapshot metadata alone. + +use std::{ + fmt, + num::NonZeroU64, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::watch; + +use super::{ArtifactError, MAX_IMAGE_BYTES, ResolvedImage}; + +/// Positive daemon-reported unpacked image storage, within the local10GiB quota. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "u64", into = "u64")] +#[non_exhaustive] +pub struct UnpackedImageSize(NonZeroU64); +impl TryFrom for UnpackedImageSize { + type Error = ArtifactError; + fn try_from(value: u64) -> Result { + let value = NonZeroU64::new(value) + .filter(|value| value.get() <= MAX_IMAGE_BYTES) + .ok_or(ArtifactError::Inspection)?; + Ok(Self(value)) + } +} +impl From for u64 { + fn from(size: UnpackedImageSize) -> Self { + size.0.get() + } +} +impl UnpackedImageSize { + /// Number of validated unpacked bytes, independent of compressed size. + /// + /// ``` + /// let size = rustack_lambda_core::artifact::UnpackedImageSize::try_from(4096)?; + /// assert_eq!(size.get(),4096); + /// # Ok::<(),rustack_lambda_core::artifact::ArtifactError>(()) + /// ``` + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } +} + +/// Shared live pin lease; cloning retains availability until joined retirement. +#[derive(Clone)] +#[non_exhaustive] +pub struct PinnedImage(Arc); + +struct Lease { + resolved: ResolvedImage, + size: UnpackedImageSize, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, +} +impl Drop for Lease { + fn drop(&mut self) { + self.release.send_replace(()); + } +} +impl fmt::Debug for PinnedImage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("PinnedImage") + .field("image_id", &self.0.resolved.metadata().image_id) + .field("unpacked_bytes", &self.0.size) + .field("live", &self.is_live()) + .finish_non_exhaustive() + } +} +impl PinnedImage { + pub(crate) fn new( + resolved: ResolvedImage, + size: UnpackedImageSize, + release: watch::Sender<()>, + active: Arc, + stop: watch::Receiver, + ) -> Self { + Self(Arc::new(Lease { + resolved, + size, + release, + active, + stop, + })) + } + + /// Immutable metadata captured by preparation; never resolve a mutable tag. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let id = &image.resolved().metadata().image_id; + /// # } + /// ``` + #[must_use] + pub fn resolved(&self) -> &ResolvedImage { + &self.0.resolved + } + + /// Proven unpacked size used by deployment quota enforcement. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let bytes = image.unpacked_size().get(); + /// # } + /// ``` + #[must_use] + pub fn unpacked_size(&self) -> UnpackedImageSize { + self.0.size + } + + /// Whether the owning manager still permits consumers to use this lease. + /// + /// Shutdown revokes leases after the runtime has joined their consumers. + /// + /// ```no_run + /// # fn inspect(image:&rustack_lambda_core::artifact::PinnedImage) { + /// let available = image.is_live(); + /// # } + /// ``` + #[must_use] + pub fn is_live(&self) -> bool { + self.0.active.load(Ordering::Acquire) && !*self.0.stop.borrow() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::artifact::Architecture; + + #[test] + fn test_should_validate_unpacked_storage_limits_including_deserialization() { + assert!(UnpackedImageSize::try_from(0).is_err()); + assert!(UnpackedImageSize::try_from(MAX_IMAGE_BYTES + 1).is_err()); + assert_eq!( + UnpackedImageSize::try_from(MAX_IMAGE_BYTES).unwrap().get(), + MAX_IMAGE_BYTES + ); + assert!(serde_json::from_str::("0").is_err()); + let size: UnpackedImageSize = serde_json::from_str("4096").unwrap(); + assert_eq!(size.get(), 4096); + } + + #[tokio::test] + async fn test_should_keep_release_channel_until_last_clone_and_revoke_on_stop() { + let (release, mut released) = watch::channel(()); + let (stop, stopping) = watch::channel(false); + let active = Arc::new(AtomicBool::new(true)); + let image = PinnedImage::new( + ResolvedImage::fixture(Architecture::Arm64), + 4096.try_into().unwrap(), + release, + active.clone(), + stopping, + ); + let captured = image.clone(); + drop(image); + assert!(!released.has_changed().unwrap()); + assert!(captured.is_live()); + assert_eq!(captured.unpacked_size().get(), 4096); + assert!(!format!("{captured:?}").contains("entrypoint")); + stop.send_replace(true); + assert!(!captured.is_live()); + drop(captured); + let _ = released.changed().await; + assert!(released.has_changed().is_err()); + } +} diff --git a/crates/rustack-lambda-core/src/docker/image.rs b/crates/rustack-lambda-core/src/docker/image.rs index ce47bddd..558dfab5 100644 --- a/crates/rustack-lambda-core/src/docker/image.rs +++ b/crates/rustack-lambda-core/src/docker/image.rs @@ -7,12 +7,12 @@ use tokio::time::Instant; use super::{ DockerError, DockerPort, Operation, decode, escaped, - models::{ImageInspect, SystemInfo}, + models::{ImageInspect, ImageManifestSummaryKindEnum, OciDescriptor, SystemInfo}, }; use crate::{ artifact::{ Architecture, ArtifactError, ImageExecutionDefaults, ImageExecutionMetadata, ImageMetadata, - ImageReference, ResolvedImage, Sha256Digest, + ImageReference, ResolvedImage, Sha256Digest, UnpackedImageSize, }, function_config::validate_image_config, }; @@ -103,14 +103,100 @@ impl ImageResolver { } result } + + /// Prove the selected image's unpacked-storage quota without running code. + /// + /// # Errors + /// Rejects missing, ambiguous, incompatible or out-of-quota unpacked proof, + /// classic-store ambiguity, expiry or bounded Docker metadata/IO failure. + /// + /// ```no_run + /// # async fn size(port:rustack_lambda_core::docker::DockerPort,image:&rustack_lambda_core::artifact::ResolvedImage) { + /// let bytes = rustack_lambda_core::docker::ImageResolver::new(port).unpacked_size(image, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn unpacked_size( + &self, + image: &ResolvedImage, + deadline: Instant, + ) -> Result { + super::check_deadline(deadline)?; + let root = self + .port + .image_manifests(&image.metadata().image_id, deadline) + .await?; + if image.metadata().selected_manifest.is_none() { + verify_classic_store(&root, &self.port.info(deadline).await?)?; + } + let size = unpacked_proof(image, &root)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + Ok(size) + } +} + +fn unpacked_proof( + image: &ResolvedImage, + root: &ImageInspect, +) -> Result { + let metadata = image.metadata(); + if root.id.as_deref() != Some(metadata.image_id.as_str()) { + return Err(invalid()); + } + let Some(manifest) = &metadata.selected_manifest else { + if root.os.as_deref() != Some("linux") + || root.architecture.as_deref() != Some(docker_arch(metadata.architecture)) + { + return Err(invalid()); + } + let size = root + .size + .and_then(|size| u64::try_from(size).ok()) + .ok_or_else(invalid)?; + return size.try_into().map_err(DockerError::from); + }; + let summaries = root.manifests.as_deref().ok_or_else(invalid)?; + if summaries.len() > 128 { + return Err(invalid()); + } + let mut matching = summaries + .iter() + .filter(|summary| summary.id == manifest.as_str()); + let summary = matching.next().ok_or_else(invalid)?; + if matching.next().is_some() + || !summary.available + || summary.kind != Some(ImageManifestSummaryKindEnum::IMAGE) + || image_descriptor(&summary.descriptor, docker_arch(metadata.architecture))? != *manifest + { + return Err(invalid()); + } + let data = summary.image_data.as_ref().ok_or_else(invalid)?; + if data.platform.os.as_deref() != Some("linux") + || data.platform.architecture.as_deref() != Some(docker_arch(metadata.architecture)) + { + return Err(invalid()); + } + let bytes = u64::try_from(data.size.unpacked).map_err(|_| invalid())?; + bytes.try_into().map_err(DockerError::from) +} + +fn docker_arch(architecture: Architecture) -> &'static str { + match architecture { + Architecture::X86_64 => "amd64", + Architecture::Arm64 => "arm64", + } } fn verify_classic_store(inspected: &ImageInspect, info: &SystemInfo) -> Result<(), DockerError> { let driver = inspected.graph_driver.as_ref().ok_or_else(invalid)?; - if !matches!( - driver.name.as_str(), - "overlay2" | "aufs" | "btrfs" | "devicemapper" | "vfs" | "zfs" | "fuse-overlayfs" - ) || info.driver.as_deref() != Some(driver.name.as_str()) + if inspected.descriptor.is_some() + || !matches!( + driver.name.as_str(), + "overlay2" | "aufs" | "btrfs" | "devicemapper" | "vfs" | "zfs" | "fuse-overlayfs" + ) + || info.driver.as_deref() != Some(driver.name.as_str()) || inspected .manifests .as_ref() @@ -214,9 +300,17 @@ fn selected_descriptor( { return Err(invalid()); } - let Some(descriptor) = &inspected.descriptor else { - return Ok(None); - }; + inspected + .descriptor + .as_ref() + .map(|descriptor| image_descriptor(descriptor, architecture)) + .transpose() +} + +fn image_descriptor( + descriptor: &OciDescriptor, + architecture: &str, +) -> Result { if !matches!( descriptor.media_type.as_deref(), Some( @@ -239,9 +333,12 @@ fn selected_descriptor( { return Err(invalid()); } - Ok(Some( - descriptor.digest.as_deref().ok_or_else(invalid)?.parse()?, - )) + descriptor + .digest + .as_deref() + .ok_or_else(invalid)? + .parse() + .map_err(DockerError::from) } fn repository_identity(reference: &ImageReference) -> String { @@ -598,4 +695,66 @@ mod tests { assert!(verify_classic_store(&inspected, &classic).is_err()); assert!(selected_descriptor(&inspected, "arm64").is_err()); } + + fn storage_proof() -> Value { + json!({"Id":digest('a'),"Manifests":[{"ID":digest('b'), + "Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":digest('b'),"platform":{"os":"linux","architecture":"arm64"}}, + "Available":true,"Kind":"image","Size":{"Content":64,"Total":99999}, + "ImageData":{"Containers":[],"Platform":{"os":"linux","architecture":"arm64"},"Size":{"Unpacked":4096}}}]}) + } + + #[test] + fn test_should_measure_selected_unpacked_storage_instead_of_compressed_or_total_size() { + let image = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let proof: ImageInspect = serde_json::from_value(storage_proof()).unwrap(); + assert_eq!(unpacked_proof(&image, &proof).unwrap().get(), 4096); + let legacy = ResolvedImage::fixture(Architecture::Arm64); + let proof: ImageInspect = serde_json::from_value( + json!({"Id":digest('a'),"Size":8192,"Os":"linux","Architecture":"arm64"}), + ) + .unwrap(); + assert_eq!(unpacked_proof(&legacy, &proof).unwrap().get(), 8192); + } + + #[rstest::rstest] + #[case("parent")] + #[case("missing")] + #[case("duplicate")] + #[case("unavailable")] + #[case("attestation")] + #[case("id")] + #[case("descriptor")] + #[case("platform")] + #[case("data")] + #[case("zero")] + #[case("negative")] + #[case("large")] + fn test_should_reject_missing_or_incompatible_unpacked_storage_evidence(#[case] fault: &str) { + let image = selected("registry.test/team/worker:v1", inspection()).unwrap(); + let mut proof = storage_proof(); + match fault { + "parent" => proof["Id"] = json!(digest('e')), + "missing" => proof["Manifests"] = json!([]), + "duplicate" => { + let duplicate = proof["Manifests"][0].clone(); + proof["Manifests"].as_array_mut().unwrap().push(duplicate); + } + "unavailable" => proof["Manifests"][0]["Available"] = json!(false), + "attestation" => proof["Manifests"][0]["Kind"] = json!("attestation"), + "id" => proof["Manifests"][0]["ID"] = json!(digest('e')), + "descriptor" => proof["Manifests"][0]["Descriptor"]["digest"] = json!(digest('e')), + "platform" => { + proof["Manifests"][0]["ImageData"]["Platform"]["architecture"] = json!("amd64"); + } + "data" => proof["Manifests"][0]["ImageData"] = Value::Null, + "zero" => proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = json!(0), + "negative" => proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = json!(-1), + "large" => { + proof["Manifests"][0]["ImageData"]["Size"]["Unpacked"] = + json!(10_u64 * 1024 * 1024 * 1024 + 1); + } + _ => panic!("unknown test fault"), + } + assert!(unpacked_proof(&image, &serde_json::from_value(proof).unwrap()).is_err()); + } } diff --git a/crates/rustack-lambda-core/src/docker/lease.rs b/crates/rustack-lambda-core/src/docker/lease.rs new file mode 100644 index 00000000..8e380e1b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease.rs @@ -0,0 +1,907 @@ +//! Retained pin preparation, shared live leases and phase-aware cleanup recovery. + +use std::{ + collections::HashMap, + fmt, mem, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use http::StatusCode; +use rustack_lambda_model::types::ImageConfig; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + task::{Id, JoinError, JoinHandle, JoinSet}, + time::{Instant, sleep, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + DockerError, DockerPort, ImagePin, ImagePinScope, ImageResolver, check_deadline, + connection::stopped, +}; +use crate::{ + artifact::{ + Architecture, ImageReference, PinnedImage, ResolvedImage, Sha256Digest, UnpackedImageSize, + }, + executor::ExecutorError, + function_config::validate_image_config, + invocation::CleanupTask, +}; + +/// Validated image-preparation and live-pin admission/IO budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ImageLeaseConfig { + pins: u16, + jobs: u8, + io: Duration, +} +impl Default for ImageLeaseConfig { + fn default() -> Self { + Self { + pins: 1024, + jobs: 8, + io: Duration::from_secs(5), + } + } +} +impl ImageLeaseConfig { + /// Validate bounded pin/job counts and the independent mutation IO budget. + /// + /// # Errors + /// Rejects pins outside1..8192, jobs outside1..32 or exceeding pins, and + /// zero/over60s IO budgets. + /// + /// ``` + /// let config = rustack_lambda_core::docker::ImageLeaseConfig::new(1024,8,std::time::Duration::from_secs(5))?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(pins: u16, jobs: u8, io: Duration) -> Result { + if !(1..=8192).contains(&pins) + || !(1..=32).contains(&jobs) + || u16::from(jobs) > pins + || io < Duration::from_secs(1) + || io > Duration::from_secs(60) + { + return Err(DockerError::Configuration); + } + Ok(Self { pins, jobs, io }) + } + fn deadline(self) -> Instant { + Instant::now() + self.io + } +} + +/// Diagnostic counts; successful cleanup is established by joined ownership. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ImageLeaseStatus { + /// Slots retained by queued, preparing, live or quarantined pins. + pub pins: usize, + /// Accepted preparations, including their failed-preparation cleanup. + pub preparing: usize, + /// Workers retaining uncertain/mismatching or failed cleanup. + pub quarantined: usize, + /// Whether new preparations can be accepted. + pub accepting: bool, + /// A worker/root supervision failure permanently closes admission. + pub worker_failed: bool, +} +#[derive(Debug, Default)] +struct Health { + failed: AtomicBool, + quarantined: AtomicUsize, +} + +#[derive(Debug)] +struct PinBudget { + capacity: Arc, + count: AtomicUsize, + changed: watch::Sender<()>, +} +impl PinBudget { + fn new(max: u16) -> Self { + let (changed, _) = watch::channel(()); + Self { + capacity: Arc::new(Semaphore::new(usize::from(max))), + count: AtomicUsize::new(0), + changed, + } + } + fn acquire(self: &Arc) -> Result { + let permit = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + self.count.fetch_add(permit.num_permits(), Ordering::AcqRel); + self.changed.send_replace(()); + Ok(PinPermit { + permit, + budget: self.clone(), + }) + } +} +#[derive(Debug)] +struct PinPermit { + permit: OwnedSemaphorePermit, + budget: Arc, +} +impl Drop for PinPermit { + fn drop(&mut self) { + self.budget + .count + .fetch_sub(self.permit.num_permits(), Ordering::AcqRel); + self.budget.changed.send_replace(()); + } +} +impl Health { + fn check(&self) -> Result<(), DockerError> { + if self.failed.load(Ordering::Acquire) { + return Err(DockerError::Worker); + } + if self.quarantined.load(Ordering::Acquire) > 0 { + return Err(DockerError::Quarantined); + } + Ok(()) + } +} + +#[derive(TypedBuilder)] +struct Request { + reference: ImageReference, + architecture: Architecture, + overrides: Option, + deadline: Instant, + cancelled: watch::Receiver<()>, + reply: oneshot::Sender>, + slot: OwnedSemaphorePermit, +} +impl fmt::Debug for Request { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ImagePreparation") + .field("deadline", &self.deadline) + .finish_non_exhaustive() + } +} +#[derive(Debug)] +struct Envelope { + request: Request, + pin: ImagePin, + permit: PinPermit, +} + +/// Cloneable bounded preparation port; it does not own manager shutdown. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct ImageLeasePort { + sender: mpsc::Sender, + slots: Arc, + pins: Arc, + stop: watch::Sender, + health: Arc, + scope: ImagePinScope, + config: ImageLeaseConfig, +} +impl ImageLeasePort { + fn available(&self) -> Result<(), DockerError> { + self.health.check()?; + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + Ok(()) + } + /// Prepare a live pinned artifact; dropping this future signals cancellation. + /// + /// # Errors + /// Returns invalid settings, bounded admission, deadline, metadata, mutation + /// or cleanup failure. In-flight mutations and cleanup remain supervised. + /// + /// ```no_run + /// # async fn prepare(port:&rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::docker::DockerError> { + /// let image = port.prepare(&"worker:v1".parse()?,rustack_lambda_core::artifact::Architecture::Arm64,None, + /// tokio::time::Instant::now()+std::time::Duration::from_secs(30)).await?; + /// assert!(image.is_live()); + /// # Ok(()) } + /// ``` + pub async fn prepare( + &self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + ) -> Result { + self.available()?; + check_deadline(deadline)?; + if let Some(config) = overrides { + validate_image_config(config).map_err(|_| DockerError::Configuration)?; + } + let slot = self + .slots + .clone() + .try_acquire_owned() + .map_err(|_| DockerError::Busy)?; + let permit = self.pins.acquire()?; + let pin = self.scope.allocate()?; + let (cancel, cancelled) = watch::channel(()); + let (reply, response) = oneshot::channel(); + let request = Request::builder() + .reference(reference.clone()) + .architecture(architecture) + .overrides(overrides.cloned()) + .deadline(deadline) + .cancelled(cancelled) + .reply(reply) + .slot(slot) + .build(); + self.sender + .try_send(Envelope { + request, + pin, + permit, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Stopped, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + drop(cancel); + self.available()?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + /// Observe release of every pin reservation without stopping admission. + /// + /// # Errors + /// Returns expiry or supervision failure. A timeout never drops cleanup. + /// Concurrent new preparations can delay this observation. + /// + /// ```no_run + /// # async fn empty(port:&rustack_lambda_core::docker::ImageLeasePort) { + /// let clean=port.wait_empty(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn wait_empty(&self, deadline: Instant) -> Result<(), DockerError> { + check_deadline(deadline)?; + let mut changed = self.pins.changed.subscribe(); + while self.pins.count.load(Ordering::Acquire) > 0 { + timeout_at(deadline, changed.changed()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + } + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + self.health.check() + } + /// Inspect bounded resource counts without logging source image defaults. + /// + /// ```no_run + /// # fn status(port:&rustack_lambda_core::docker::ImageLeasePort) { + /// let status=port.status(); + /// # } + /// ``` + #[must_use] + pub fn status(&self) -> ImageLeaseStatus { + let worker_failed = self.health.failed.load(Ordering::Acquire); + let quarantined = self.health.quarantined.load(Ordering::Acquire); + ImageLeaseStatus { + pins: self.pins.count.load(Ordering::Acquire), + preparing: usize::from(self.config.jobs).saturating_sub(self.slots.available_permits()), + quarantined, + accepting: !worker_failed && quarantined == 0 && !*self.stop.borrow(), + worker_failed, + } + } +} + +/// Owner of every preparation, live alias and retained cleanup worker. +#[derive(Debug)] +#[non_exhaustive] +pub struct ImageLeaseManager { + port: ImageLeasePort, + task: Option>>, + primary: Option>, + settlement: Option>, + emergency: Arc, +} +impl ImageLeaseManager { + /// Start under the current Tokio runtime with an explicit scoped Docker port. + /// + /// # Errors + /// Rejects absence of a Tokio runtime. Directory locking belongs to the + /// runtime coordinator; the scope is never taken from a function request. + /// + /// ```no_run + /// # fn start(port:rustack_lambda_core::docker::DockerPort,scope:rustack_lambda_core::docker::ImagePinScope) -> Result<(),rustack_lambda_core::docker::DockerError> { + /// let owner=rustack_lambda_core::docker::ImageLeaseManager::start(port,scope,Default::default())?; + /// # Ok(()) } + /// ``` + pub fn start( + docker: DockerPort, + scope: ImagePinScope, + config: ImageLeaseConfig, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| DockerError::Worker)?; + let (sender, requests) = mpsc::channel(32); + let (stop, _) = watch::channel(false); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let port = ImageLeasePort::builder() + .sender(sender) + .slots(Arc::new(Semaphore::new(usize::from(config.jobs)))) + .pins(Arc::new(PinBudget::new(config.pins))) + .stop(stop.clone()) + .health(health.clone()) + .scope(scope) + .config(config) + .build(); + let actor = Actor::builder() + .docker(docker) + .config(config) + .inbox(requests) + .control(stop) + .health(health) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + let task = runtime.spawn(actor.run()); + Ok(Self { + port, + task: Some(task), + primary: None, + settlement: None, + emergency, + }) + } + /// Clone a preparation port; retain this owner through consumer shutdown. + /// + /// ```no_run + /// # fn port(owner:&rustack_lambda_core::docker::ImageLeaseManager) { + /// let port=owner.port(); + /// # } + /// ``` + #[must_use] + pub fn port(&self) -> ImageLeasePort { + self.port.clone() + } + + /// Revoke leases, stop admission and join pin cleanup before `DockerWire` stops. + /// + /// # Errors + /// Returns timeout while retaining cleanup, or cached supervision failure. + /// The runtime must first join invocation/environment consumers of the leases. + /// + /// ```no_run + /// # async fn stop(owner:&mut rustack_lambda_core::docker::ImageLeaseManager) { + /// let result=owner.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), DockerError> { + self.port.stop.send_replace(true); + if let Some(result) = &self.settlement { + return result.clone(); + } + check_deadline(deadline)?; + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Err(DockerError::Worker), + }; + self.primary = Some(result); + self.task.take(); + } + let emergency = timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker); + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + let result = self + .primary + .clone() + .ok_or(DockerError::Worker)? + .and(emergency); + self.settlement = Some(result.clone()); + result + } +} +impl Drop for ImageLeaseManager { + fn drop(&mut self) { + self.port.stop.send_replace(true); + } +} + +#[derive(Debug, Clone)] +enum Phase { + Unissued, + Uncertain(Sha256Digest), + Known(Sha256Digest), + Clean, +} +#[derive(Debug)] +struct Progress { + phase: watch::Sender, + released: watch::Receiver<()>, + active: Arc, +} +#[derive(Debug)] +struct Record { + pin: ImagePin, + progress: Arc, + permit: PinPermit, +} + +type PinJob = (Uuid, Result<(), DockerError>); +type PinJoin = Result<(Id, PinJob), JoinError>; + +#[derive(Debug, TypedBuilder)] +struct Actor { + docker: DockerPort, + config: ImageLeaseConfig, + inbox: mpsc::Receiver, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default = Arc::new(Semaphore::new(2)))] + cleanup: Arc, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + records: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + async fn run(mut self) -> Result<(), DockerError> { + let mut stop = self.control.subscribe(); + let mut wire_stop = self.docker.raw.stop.subscribe(); + loop { + if *stop.borrow() { + self.inbox.close(); + while let Ok(envelope) = self.inbox.try_recv() { + let _ = envelope.request.reply.send(Err(DockerError::Stopped)); + } + for record in self.records.values() { + record.progress.active.store(false, Ordering::Release); + } + if self.jobs.is_empty() { + break; + } + if let Some(result) = self.jobs.join_next_with_id().await { + self.finished(&result); + } + continue; + } + tokio::select! { + biased; + ()=stopped(&mut wire_stop)=>{ self.health.failed.store(true,Ordering::Release);self.control.send_replace(true); }, + ()=stopped(&mut stop)=>{}, + result=self.jobs.join_next_with_id(),if !self.jobs.is_empty()=>if let Some(result)=result {self.finished(&result);}, + envelope=self.inbox.recv()=>match envelope {Some(envelope)=>self.start(envelope),None=>{self.control.send_replace(true);}}, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(DockerError::Worker) + } else { + Ok(()) + } + } + fn start(&mut self, envelope: Envelope) { + if self.health.check().is_err() + || *self.control.borrow() + || envelope.request.reply.is_closed() + || envelope.request.cancelled.has_changed().is_err() + || Instant::now() >= envelope.request.deadline + { + let _ = envelope.request.reply.send(Err(DockerError::Stopped)); + return; + } + let (phase, _) = watch::channel(Phase::Unissued); + let (release, released) = watch::channel(()); + let progress = Arc::new(Progress { + phase, + released, + active: Arc::new(AtomicBool::new(false)), + }); + let id = envelope.pin.id; + let worker = Worker::builder() + .docker(self.docker.clone()) + .config(self.config) + .pin(envelope.pin.clone()) + .progress(progress.clone()) + .control(self.control.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .build(); + self.records.insert( + id, + Record { + pin: envelope.pin, + progress, + permit: envelope.permit, + }, + ); + let health = self.health.clone(); + let control = self.control.clone(); + let job = self.jobs.spawn(async move { + let mut guard = WorkerGuard { + health, + control, + finished: false, + }; + let result = worker.run(envelope.request, release).await; + guard.finished = true; + (id, result) + }); + self.index.insert(job.id(), id); + } + fn finished(&mut self, result: &PinJoin) { + let task = match result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(id) = self.index.remove(&task) else { + self.fail(); + return; + }; + let clean = self.records.get(&id).is_some_and(|record| { + matches!( + *record.progress.phase.borrow(), + Phase::Unissued | Phase::Clean + ) + }); + if matches!(result, Ok((_, (returned, Ok(())))) if *returned==id) && clean { + if let Some(record) = self.records.remove(&id) { + drop(record.permit); + } + return; + } + self.fail(); + let Some(record) = self.records.get(&id) else { + return; + }; + record.progress.active.store(false, Ordering::Release); + let worker = Worker::builder() + .docker(self.docker.clone()) + .config(self.config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(self.control.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .build(); + let job = self.jobs.spawn(async move { + worker.retire().await; + (id, Ok(())) + }); + self.index.insert(job.id(), id); + } + fn fail(&self) { + self.health.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + if self.records.is_empty() { + return; + } + self.fail(); + for record in self.records.values() { + record.progress.active.store(false, Ordering::Release); + } + let records = mem::take(&mut self.records); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + let docker = self.docker.clone(); + let config = self.config; + let control = self.control.clone(); + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover_all(records, docker, config, control, health, cleanup).await + }); + } +} + +async fn recover_all( + mut records: HashMap, + docker: DockerPort, + config: ImageLeaseConfig, + control: watch::Sender, + health: Arc, + cleanup: Arc, +) -> Result<(), ExecutorError> { + let mut jobs = JoinSet::new(); + let mut index = HashMap::new(); + for (id, record) in &records { + let worker = Worker::builder() + .docker(docker.clone()) + .config(config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(control.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .build(); + let id = *id; + let job = jobs.spawn(async move { + worker.retire().await; + id + }); + index.insert(job.id(), id); + } + while let Some(result) = jobs.join_next_with_id().await { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some(id) = index.remove(&task) else { + continue; + }; + let Some(record) = records.get(&id) else { + continue; + }; + if matches!( + *record.progress.phase.borrow(), + Phase::Unissued | Phase::Clean + ) { + if let Some(record) = records.remove(&id) { + drop(record.permit); + } + } else { + let worker = Worker::builder() + .docker(docker.clone()) + .config(config) + .pin(record.pin.clone()) + .progress(record.progress.clone()) + .control(control.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .build(); + let job = jobs.spawn(async move { + worker.retire().await; + id + }); + index.insert(job.id(), id); + } + } + if records.is_empty() { + Ok(()) + } else { + Err(ExecutorError::CleanupFailed( + "Image pin recovery lost supervision".into(), + )) + } +} + +struct WorkerGuard { + health: Arc, + control: watch::Sender, + finished: bool, +} +impl Drop for WorkerGuard { + fn drop(&mut self) { + if !self.finished { + self.health.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } + } +} +struct Dirty(Arc); +impl Dirty { + fn new(health: Arc) -> Self { + health.quarantined.fetch_add(1, Ordering::AcqRel); + Self(health) + } +} +impl Drop for Dirty { + fn drop(&mut self) { + self.0.quarantined.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Debug, TypedBuilder)] +struct Worker { + docker: DockerPort, + config: ImageLeaseConfig, + pin: ImagePin, + progress: Arc, + control: watch::Sender, + health: Arc, + cleanup: Arc, +} +impl Worker { + fn check(&self, deadline: Instant, cancel: &watch::Receiver<()>) -> Result<(), DockerError> { + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + if *self.control.borrow() { + return Err(DockerError::Stopped); + } + if cancel.has_changed().is_err() { + return Err(DockerError::Cancelled); + } + Ok(()) + } + async fn run( + mut self, + request: Request, + release: watch::Sender<()>, + ) -> Result<(), DockerError> { + let Request { + reference, + architecture, + overrides, + deadline, + mut cancelled, + reply, + slot, + } = request; + let staged = self + .stage( + &reference, + architecture, + overrides.as_ref(), + deadline, + &mut cancelled, + ) + .await; + drop(reference); + drop(overrides); + match staged { + Ok((resolved, size)) => { + self.progress.active.store(true, Ordering::Release); + let image = PinnedImage::new( + resolved, + size, + release, + self.progress.active.clone(), + self.control.subscribe(), + ); + match self.check(deadline, &cancelled) { + Ok(()) => { + let _ = reply.send(Ok(image)); + } + Err(error) => { + self.progress.active.store(false, Ordering::Release); + drop(image); + let _ = reply.send(Err(error)); + } + } + drop(cancelled); + drop(slot); + let mut released = self.progress.released.clone(); + let mut stop = self.control.subscribe(); + tokio::select! {biased;()=stopped(&mut stop)=>{},_ = released.changed()=>{}} + self.retire().await; + } + Err(error) => { + let _ = reply.send(Err(error)); + self.retire().await; + drop(slot); + } + } + Ok(()) + } + async fn stage( + &mut self, + reference: &ImageReference, + architecture: Architecture, + overrides: Option<&ImageConfig>, + deadline: Instant, + cancel: &mut watch::Receiver<()>, + ) -> Result<(ResolvedImage, UnpackedImageSize), DockerError> { + self.check(deadline, cancel)?; + let resolver = ImageResolver::new(self.docker.clone()); + let mut stop = self.control.subscribe(); + let image = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.resolve_local(reference,architecture,overrides,deadline)=>result?,}; + self.check(deadline, cancel)?; + let expected = image.metadata().image_id.clone(); + self.progress + .phase + .send_replace(Phase::Uncertain(expected.clone())); + let tagged = self + .docker + .tag_image_pin(&expected, &self.pin, self.config.deadline()) + .await; + if matches!( + &tagged, + Err(DockerError::Busy + | DockerError::Status(StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND)) + ) { + self.progress.phase.send_replace(Phase::Clean); + } + tagged?; + self.progress + .phase + .send_replace(Phase::Known(expected.clone())); + self.check(deadline, cancel)?; + let actual = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(self.pin.reference(),deadline)=>result?,}; + if actual.id.as_deref() != Some(expected.as_str()) { + return Err(DockerError::Quarantined); + } + let size = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.unpacked_size(&image,deadline)=>result?,}; + self.check(deadline, cancel)?; + Ok((image, size)) + } + async fn retire(self) { + self.progress.active.store(false, Ordering::Release); + let mut dirty = matches!(*self.progress.phase.borrow(), Phase::Uncertain(_)) + .then(|| Dirty::new(self.health.clone())); + let mut delay = Duration::from_secs(1); + loop { + if self.cleanup_step().await { + self.progress.phase.send_replace(Phase::Clean); + return; + } + if dirty.is_none() { + dirty = Some(Dirty::new(self.health.clone())); + } + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + async fn cleanup_step(&self) -> bool { + let phase = self.progress.phase.borrow().clone(); + let (expected, known) = match phase { + Phase::Unissued | Phase::Clean => return true, + Phase::Known(id) => (id, true), + Phase::Uncertain(id) => (id, false), + }; + let Ok(_cleanup) = self.cleanup.acquire().await else { + return false; + }; + let inspected = self + .docker + .inspect_image_pin(&self.pin, self.config.deadline()) + .await; + match inspected { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => known, + Ok(image) if image.id.as_deref() == Some(expected.as_str()) => { + self.progress.phase.send_replace(Phase::Known(expected)); + let _ = self + .docker + .remove_image_pin(&self.pin, self.config.deadline()) + .await; + matches!( + self.docker + .inspect_image_pin(&self.pin, self.config.deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ) + } + _ => false, + } + } +} + +#[cfg(test)] +#[path = "lease_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/lease_tests.rs b/crates/rustack-lambda-core/src/docker/lease_tests.rs new file mode 100644 index 00000000..3d0479b6 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/lease_tests.rs @@ -0,0 +1,826 @@ +use std::{ + future::Future, + task::{Context, Waker}, +}; + +use serde_json::{Value, json}; +use tokio::{ + sync::Notify, + time::{sleep_until, timeout}, +}; + +use super::*; +use crate::docker::{ + DockerWire, + tests::{Daemon, IncomingRequest}, +}; + +fn digest(letter: char) -> String { + format!("sha256:{}", letter.to_string().repeat(64)) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn scope() -> ImagePinScope { + ImagePinScope::new(Uuid::new_v4(), Uuid::new_v4()).unwrap() +} +fn config(pins: u16, jobs: u8) -> ImageLeaseConfig { + ImageLeaseConfig::new(pins, jobs, Duration::from_secs(1)).unwrap() +} +fn metadata(parent: &str, platform: bool) -> Value { + let child = if parent == digest('a') { + digest('c') + } else { + digest('d') + }; + json!({"Id":if platform {&child} else {parent},"Os":"linux","Architecture":"arm64","Size":64, + "Descriptor":{"mediaType":if platform {"application/vnd.oci.image.manifest.v1+json"} else {"application/vnd.oci.image.index.v1+json"}, + "digest":if platform {&child} else {parent},"platform":{"os":"linux","architecture":"arm64"}}, + "Config":{"Entrypoint":["/entry"],"Cmd":[if parent==digest('a') {"old_handler"} else {"new_handler"}],"Env":["SECRET=private_value"],"WorkingDir":"/task","User":"1000"}}) +} +fn manifests(parent: &str, zero: bool) -> Value { + let mut value = metadata(parent, false); + let child = if parent == digest('a') { + digest('c') + } else { + digest('d') + }; + value["Manifests"] = json!([{"ID":child,"Descriptor":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":child,"platform":{"os":"linux","architecture":"arm64"}}, + "Available":true,"Kind":"image","Size":{"Content":64,"Total":99999}, + "ImageData":{"Containers":[],"Platform":{"os":"linux","architecture":"arm64"},"Size":{"Unpacked":if zero {0} else {4096}}}}]); + value +} + +#[derive(Debug, Default)] +struct Stats { + tags: AtomicUsize, + deletes: AtomicUsize, + absent: AtomicUsize, + aliases: AtomicUsize, + changed: Notify, +} +impl Stats { + async fn until(&self, condition: impl Fn() -> bool) { + timeout(Duration::from_secs(10), async { + loop { + let changed = self.changed.notified(); + if condition() { + break; + } + changed.await; + } + }) + .await + .unwrap(); + } + fn notify(&self) { + self.changed.notify_waiters(); + } +} +#[derive(Debug, Clone, Copy)] +enum Mode { + Normal, + Reject400, + Reject404, + Lost, + PostCommit500, + Late, + Late500, + Never, + Wrong, + ZeroSize, + LostDelete, + HoldDelete, +} +enum Command { + Retarget, + Repair, + Commit, + AllowDeletes, +} +struct Pending { + alias: String, + parent: String, + at: Option, + request: Option, +} + +#[derive(TypedBuilder)] +struct Script { + daemon: Daemon, + mode: Mode, + stats: Arc, + commands: mpsc::Receiver, + stop: watch::Receiver, + #[builder(default)] + aliases: HashMap, + #[builder(default)] + expected: HashMap, + #[builder(default)] + pending: Option, + #[builder(default)] + retargeted: bool, + #[builder(default)] + allow_deletes: bool, +} +impl Script { + async fn run(mut self) { + loop { + let at = self.pending.as_ref().and_then(|pending| pending.at); + tokio::select! { + biased; + _ = self.stop.changed()=>break, + () = sleep_until(at.unwrap_or_else(Instant::now)),if at.is_some()=>self.commit(), + command=self.commands.recv()=>match command {Some(Command::Retarget)=>self.retargeted=true,Some(Command::Repair)=>self.aliases=self.expected.clone(),Some(Command::Commit)=>self.commit(),Some(Command::AllowDeletes)=>self.allow_deletes=true,None=>break}, + request=self.daemon.requests.recv()=>match request {Some(request)=>self.request(request).await,None=>break}, + } + self.stats + .aliases + .store(self.aliases.len(), Ordering::Release); + self.stats.notify(); + } + assert_eq!(self.aliases.len(), 0, "fake daemon retains image aliases"); + assert!(self.pending.is_none()); + self.daemon.close().await; + } + fn commit(&mut self) { + if let Some(pending) = self.pending.take() { + self.aliases.insert(pending.alias, pending.parent); + drop(pending.request); + } + } + async fn tag(&mut self, request: IncomingRequest, image: &str, query: &str) { + assert!(image.ends_with("/tag")); + let parent = image.strip_suffix("/tag").unwrap().to_owned(); + let fields: HashMap<_, _> = form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect(); + let alias = format!( + "{}:{}", + fields.get("repo").unwrap(), + fields.get("tag").unwrap() + ); + assert!(alias.starts_with("rustack-internal/lambda-")); + self.stats.tags.fetch_add(1, Ordering::AcqRel); + self.expected.insert(alias.clone(), parent.clone()); + match self.mode { + Mode::Reject400 => request.reply(400, b"{}").await, + Mode::Reject404 => request.reply(404, b"{}").await, + Mode::Late | Mode::Never => { + assert!(self.pending.is_none()); + self.pending = Some(Pending { + alias, + parent, + at: matches!(self.mode, Mode::Late) + .then(|| Instant::now() + Duration::from_secs(2)), + request: Some(request), + }); + } + Mode::Late500 => { + assert!(self.pending.is_none()); + self.pending = Some(Pending { + alias, + parent, + at: Some(Instant::now() + Duration::from_secs(2)), + request: None, + }); + request.reply(500, b"{}").await; + } + _ => { + self.aliases.insert( + alias, + if matches!(self.mode, Mode::Wrong) { + digest('b') + } else { + parent + }, + ); + match self.mode { + Mode::Lost => drop(request), + Mode::PostCommit500 => request.reply(500, b"{}").await, + _ => request.reply(201, b"").await, + } + } + } + } + + async fn request(&mut self, request: IncomingRequest) { + let line = request.head.lines().next().unwrap().to_owned(); + let fields: Vec<_> = line.split_whitespace().collect(); + let method = fields.first().unwrap(); + let target = fields.get(1).unwrap(); + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let path = percent_encoding::percent_decode_str(path) + .decode_utf8() + .unwrap() + .into_owned(); + let image = path.strip_prefix("/v1.53/images/").unwrap(); + if *method == "POST" { + self.tag(request, image, query).await; + } else if *method == "DELETE" { + assert_eq!(query, "force=false&noprune=true"); + assert!(image.starts_with("rustack-internal/lambda-")); + self.stats.deletes.fetch_add(1, Ordering::AcqRel); + if matches!(self.mode, Mode::HoldDelete) && !self.allow_deletes { + request.reply(500, b"{}").await; + return; + } + let removed = self.aliases.remove(image); + assert!(removed.is_some()); + if matches!(self.mode, Mode::LostDelete) { + drop(request); + return; + } + request + .reply( + 200, + &serde_json::to_vec(&json!([{"Untagged":image}])).unwrap(), + ) + .await; + } else { + assert_eq!(*method, "GET"); + let reference = image.strip_suffix("/json").unwrap(); + let parent = if reference == "worker:v1" { + Some(if self.retargeted { + digest('b') + } else { + digest('a') + }) + } else if let Some(id) = self.aliases.get(reference) { + Some(id.clone()) + } else if reference == digest('a') || reference == digest('b') { + let current = if self.retargeted { + digest('b') + } else { + digest('a') + }; + (current == reference || self.aliases.values().any(|id| id == reference)) + .then(|| reference.to_owned()) + } else { + None + }; + if let Some(parent) = parent { + let value = if query == "manifests=true" { + manifests(&parent, matches!(self.mode, Mode::ZeroSize)) + } else { + metadata(&parent, !query.is_empty()) + }; + request + .reply(200, &serde_json::to_vec(&value).unwrap()) + .await; + } else { + self.stats.absent.fetch_add(1, Ordering::AcqRel); + request.reply(404, b"{}").await; + } + } + } +} + +struct FakePins { + wire: DockerWire, + stop: watch::Sender, + commands: mpsc::Sender, + task: Option>, + stats: Arc, +} +impl FakePins { + async fn new(mode: Mode) -> Self { + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let (stop, stopping) = watch::channel(false); + let (commands, receiver) = mpsc::channel(8); + let stats = Arc::new(Stats::default()); + let script = Script::builder() + .daemon(daemon) + .mode(mode) + .stats(stats.clone()) + .commands(receiver) + .stop(stopping) + .build(); + Self { + wire, + stop, + commands, + task: Some(tokio::spawn(script.run())), + stats, + } + } + fn owner(&self, pins: u16, jobs: u8) -> ImageLeaseManager { + ImageLeaseManager::start(self.wire.port(), scope(), config(pins, jobs)).unwrap() + } + async fn close(mut self) { + self.wire.shutdown().await.unwrap(); + self.stop.send_replace(true); + self.task.take().unwrap().await.unwrap(); + } +} +impl Drop for FakePins { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[tokio::test] +async fn test_should_pin_old_code_across_tag_retargeting_and_keep_cloned_artifacts_live() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + assert!(image.is_live()); + assert_eq!(image.unpacked_size().get(), 4096); + assert_eq!(port.status().pins, 1); + assert!(!format!("{image:?}").contains("private_value")); + let mut observer = Box::pin(port.wait_empty(deadline())); + let mut context = Context::from_waker(Waker::noop()); + assert!(observer.as_mut().poll(&mut context).is_pending()); + assert_eq!(port.status().pins, 1); + fake.commands.send(Command::Retarget).await.unwrap(); + let newer = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(observer); + assert_eq!(image.resolved().metadata().image_id.as_str(), digest('a')); + assert_eq!(newer.resolved().metadata().image_id.as_str(), digest('b')); + assert_eq!( + image + .resolved() + .metadata() + .execution + .plan(None) + .unwrap() + .command(), + ["old_handler"] + ); + let captured = image.clone(); + drop(image); + drop(newer); + assert!(matches!( + port.wait_empty(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(captured.is_live()); + drop(captured); + port.wait_empty(deadline()).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 2); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 2); + fake.close().await; +} + +#[rstest::rstest] +#[case(Mode::Reject400)] +#[case(Mode::Reject404)] +#[case(Mode::Lost)] +#[case(Mode::PostCommit500)] +#[case(Mode::ZeroSize)] +#[tokio::test] +async fn test_should_reconcile_rejected_and_lost_preparation_outcomes(#[case] mode: Mode) { + let fake = FakePins::new(mode).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(port.status().quarantined, 0); + assert!(port.status().accepting); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!( + fake.stats.deletes.load(Ordering::Acquire), + usize::from(!matches!(mode, Mode::Reject400 | Mode::Reject404)) + ); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_retain_uncertain_tag_through_cancelled_waiter_and_initial_absence() { + let fake = FakePins::new(Mode::Late).await; + let mut owner = fake.owner(1, 1); + let port = owner.port(); + let call = port.clone(); + let caller = tokio::spawn(async move { + call.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + fake.stats + .until(|| fake.stats.absent.load(Ordering::Acquire) > 0) + .await; + assert_eq!(port.status().pins, 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_bound_preparation_and_pin_admission_and_retain_shutdown_on_timeout() { + let fake = FakePins::new(Mode::Never).await; + let mut owner = fake.owner(1, 1); + let port = owner.port(); + let call = port.clone(); + let caller = tokio::spawn(async move { + call.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(matches!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await, + Err(DockerError::Busy) + )); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(owner.task.is_some()); + assert_eq!(port.status().pins, 1); + fake.commands.send(Command::Commit).await.unwrap(); + assert!(caller.await.unwrap().is_err()); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + fake.close().await; +} + +#[tokio::test] +async fn test_should_quarantine_observed_alias_conflicts_until_exclusive_owner_repairs_them() { + let fake = FakePins::new(Mode::Wrong).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + fake.stats.until(|| port.status().quarantined > 0).await; + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + assert!(matches!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await, + Err(DockerError::Quarantined) + )); + fake.commands.send(Command::Repair).await.unwrap(); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(port.status().quarantined, 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_join_after_a_shutdown_waiter_is_dropped_and_cache_root_failure() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + let mut shutdown = Box::pin(owner.shutdown(deadline())); + let mut context = Context::from_waker(Waker::noop()); + assert!(shutdown.as_mut().poll(&mut context).is_pending()); + drop(shutdown); + owner.shutdown(deadline()).await.unwrap(); + assert!(!image.is_live()); + drop(image); + fake.close().await; + + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + owner.task.as_ref().unwrap().abort(); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert!(!image.is_live()); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert_eq!(port.status().pins, 0); + drop(image); + fake.close().await; +} + +#[test] +fn test_should_reject_invalid_pin_scope_and_preparation_configuration() { + assert!(ImagePinScope::new(Uuid::nil(), Uuid::new_v4()).is_err()); + let pin = scope().allocate().unwrap(); + assert!( + pin.reference() + .as_str() + .starts_with("rustack-internal/lambda-") + ); + for (pins, jobs, io) in [ + (0, 1, Duration::from_secs(1)), + (8193, 1, Duration::from_secs(1)), + (1, 2, Duration::from_secs(1)), + (1, 1, Duration::from_nanos(1)), + (1, 1, Duration::from_secs(61)), + ] { + assert!(ImageLeaseConfig::new(pins, jobs, io).is_err()); + } +} + +#[tokio::test] +async fn test_should_retire_an_image_sent_to_a_response_channel_with_no_consuming_waiter() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let reference = "worker:v1".parse().unwrap(); + let mut request = Box::pin(port.prepare(&reference, Architecture::Arm64, None, deadline())); + let mut context = Context::from_waker(Waker::noop()); + assert!(request.as_mut().poll(&mut context).is_pending()); + let accepted = timeout_at(deadline(), port.slots.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + assert_eq!(port.status().pins, 1); + drop(request); + drop(accepted); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_report_root_failure_even_when_no_pin_has_started() { + let fake = FakePins::new(Mode::Normal).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + owner.task.as_ref().unwrap().abort(); + timeout_at(deadline(), async { + while !owner.task.as_ref().unwrap().is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(matches!( + port.wait_empty(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!( + owner.shutdown(deadline()).await, + Err(DockerError::Worker) + )); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 0); + fake.close().await; +} + +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_recover_owned_aliases_after_worker_or_off_runtime_root_failure( + #[case] off_runtime: bool, +) { + let fake = FakePins::new(Mode::Normal).await; + let config = config(1, 1); + let pin = scope().allocate().unwrap(); + let resolved = ImageResolver::new(fake.wire.port()) + .resolve_local( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + fake.wire + .port() + .tag_image_pin(&resolved.metadata().image_id, &pin, deadline()) + .await + .unwrap(); + let budget = Arc::new(PinBudget::new(1)); + let permit = budget.acquire().unwrap(); + let (control, stop) = watch::channel(false); + let (release, released) = watch::channel(()); + let (phase, _) = watch::channel(Phase::Known(resolved.metadata().image_id.clone())); + let active = Arc::new(AtomicBool::new(true)); + let progress = Arc::new(Progress { + phase, + released, + active: active.clone(), + }); + let image = PinnedImage::new(resolved, 4096.try_into().unwrap(), release, active, stop); + let (sender, inbox) = mpsc::channel(32); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let id = pin.id; + let mut jobs: JoinSet = JoinSet::new(); + let job = jobs.spawn(async { panic!("controlled image worker failure") }); + let actor = Actor::builder() + .docker(fake.wire.port()) + .config(config) + .inbox(inbox) + .control(control) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(emergency.clone()) + .jobs(jobs) + .records(HashMap::from([( + id, + Record { + pin, + progress, + permit, + }, + )])) + .index(HashMap::from([(job.id(), id)])) + .build(); + if off_runtime { + let thread = std::thread::spawn(move || drop(actor)); + timeout_at( + deadline(), + tokio::task::spawn_blocking(move || thread.join()), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + timeout_at(deadline(), emergency.wait()) + .await + .unwrap() + .unwrap(); + } else { + let result = timeout_at(deadline(), tokio::spawn(actor.run())) + .await + .unwrap() + .unwrap(); + assert!(matches!(result, Err(DockerError::Worker))); + } + assert!(health.failed.load(Ordering::Acquire)); + assert!(!image.is_live()); + assert_eq!(budget.count.load(Ordering::Acquire), 0); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + emergency.wait().await.unwrap(); + drop(sender); + drop(image); + fake.close().await; +} + +#[tokio::test] +async fn test_should_retain_a_late_nested_mutation_after_a_complete_http_500() { + let fake = FakePins::new(Mode::Late500).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + assert!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await + .is_err() + ); + fake.stats + .until(|| fake.stats.absent.load(Ordering::Acquire) > 0) + .await; + assert_eq!(port.status().pins, 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 0); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_confirm_absence_after_an_acknowledged_tag_and_lost_delete_response() { + let fake = FakePins::new(Mode::LostDelete).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + port.wait_empty(deadline()).await.unwrap(); + assert_eq!(fake.stats.deletes.load(Ordering::Acquire), 1); + assert_eq!(port.status().quarantined, 0); + owner.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_retain_cleanup_capacity_and_shutdown_join_until_delete_recovers() { + let fake = FakePins::new(Mode::HoldDelete).await; + let mut owner = fake.owner(2, 1); + let port = owner.port(); + let image = port + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + fake.stats.until(|| port.status().quarantined > 0).await; + assert_eq!(port.status().pins, 1); + assert!(matches!( + port.prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline() + ) + .await, + Err(DockerError::Quarantined) + )); + assert!(matches!( + owner + .shutdown(Instant::now() + Duration::from_millis(50)) + .await, + Err(DockerError::Deadline) + )); + assert!(owner.task.is_some()); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + owner.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().pins, 0); + assert_eq!(port.status().quarantined, 0); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert!(fake.stats.deletes.load(Ordering::Acquire) >= 2); + fake.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 07da1b24..c2c055ca 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -6,6 +6,8 @@ mod connection; mod framing; mod image; +mod lease; +mod pin; mod types; use std::{ @@ -25,11 +27,13 @@ use http::{Method, StatusCode}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; pub use image::ImageResolver; +pub use lease::{ImageLeaseConfig, ImageLeaseManager, ImageLeasePort, ImageLeaseStatus}; use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, }; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +pub use pin::{ImagePin, ImagePinScope}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; @@ -53,6 +57,9 @@ enum Operation { Version, Info, Image, + TagPin, + InspectPin, + RemovePin, Create, Start, Inspect, @@ -64,7 +71,15 @@ enum Operation { } impl Operation { fn cleanup(self) -> bool { - matches!(self, Self::Inspect | Self::Kill | Self::Remove | Self::List) + matches!( + self, + Self::Inspect + | Self::Kill + | Self::Remove + | Self::List + | Self::InspectPin + | Self::RemovePin + ) } } diff --git a/crates/rustack-lambda-core/src/docker/pin.rs b/crates/rustack-lambda-core/src/docker/pin.rs new file mode 100644 index 00000000..bf111d1b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pin.rs @@ -0,0 +1,212 @@ +//! Only generated private alias identities can reach image mutation endpoints. + +use std::fmt; + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + DockerError, DockerPort, Operation, decode, escaped, + models::{ImageDeleteResponseItem, ImageInspect}, + unit, +}; +use crate::artifact::{ImageReference, Sha256Digest}; + +/// Trusted runtime owner/run identity; never populated from a Lambda request. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ImagePinScope { + owner: Uuid, + run: Uuid, +} +impl ImagePinScope { + /// Validate the scope supplied by the directory-lock owner. + /// + /// # Errors + /// Rejects nil UUIDs. This does not acquire the runtime's directory lock. + /// + /// ``` + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(owner: Uuid, run: Uuid) -> Result { + if owner.is_nil() || run.is_nil() { + return Err(DockerError::Configuration); + } + Ok(Self { owner, run }) + } + /// Allocate a fresh private alias before issuing any Docker mutation. + /// + /// # Errors + /// Returns validation failure if the generated reference cannot validate. + /// + /// ``` + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; + /// let pin = scope.allocate()?; + /// assert!(pin.reference().as_str().starts_with("rustack-internal/")); + /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn allocate(self) -> Result { + let id = Uuid::new_v4(); + let reference = format!( + "rustack-internal/lambda-{}/{}:{}", + self.owner.simple(), + self.run.simple(), + id.simple() + ) + .parse()?; + Ok(ImagePin { id, reference }) + } +} + +/// An allocated alias; arbitrary source references cannot become cleanup keys. +#[derive(Clone)] +#[non_exhaustive] +pub struct ImagePin { + pub(super) id: Uuid, + reference: ImageReference, +} +impl fmt::Debug for ImagePin { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ImagePin") + .field("id", &self.id) + .finish_non_exhaustive() + } +} +impl ImagePin { + /// Exact private alias; its namespace is reserved for the owning runtime. + /// + /// ```no_run + /// # fn reference(pin:&rustack_lambda_core::docker::ImagePin) { + /// let reference = pin.reference(); + /// # } + /// ``` + #[must_use] + pub fn reference(&self) -> &ImageReference { + &self.reference + } +} + +impl DockerPort { + /// Create only the allocated private alias, targeting a captured image ID. + /// + /// # Errors + /// Returns bounded wire failure. Any uncertain outcome retains pin ownership. + /// + /// ```no_run + /// # async fn tag(port:&rustack_lambda_core::docker::DockerPort,id:&rustack_lambda_core::artifact::Sha256Digest,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.tag_image_pin(id,pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn tag_image_pin( + &self, + id: &Sha256Digest, + pin: &ImagePin, + deadline: Instant, + ) -> Result<(), DockerError> { + let query = form_urlencoded::Serializer::new(String::new()) + .append_pair("repo", pin.reference.repository()) + .append_pair("tag", &pin.id.simple().to_string()) + .finish(); + let reply = self + .call( + Operation::TagPin, + Method::POST, + format!("/images/{}/tag?{query}", escaped(id.as_str())), + Bytes::new(), + StatusCode::CREATED, + deadline, + ) + .await?; + unit(reply, deadline) + } + + /// Inspect a known private alias using reserved cleanup capacity. + /// + /// # Errors + /// Returns missing alias status, bounded metadata or transport failure. + /// + /// ```no_run + /// # async fn inspect(port:&rustack_lambda_core::docker::DockerPort,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.inspect_image_pin(pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_image_pin( + &self, + pin: &ImagePin, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::InspectPin, + Method::GET, + format!("/images/{}/json", escaped(pin.reference.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + /// Remove only an allocated private alias, without force or ancestor pruning. + /// + /// # Errors + /// Returns status, bounded response or transport failure. The owner verifies + /// identity under its exclusive namespace before removal and confirms absence. + /// + /// ```no_run + /// # async fn remove(port:&rustack_lambda_core::docker::DockerPort,pin:&rustack_lambda_core::docker::ImagePin) { + /// let result = port.remove_image_pin(pin,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn remove_image_pin( + &self, + pin: &ImagePin, + deadline: Instant, + ) -> Result, DockerError> { + decode( + self.call( + Operation::RemovePin, + Method::DELETE, + format!( + "/images/{}?force=false&noprune=true", + escaped(pin.reference.as_str()) + ), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } + + pub(super) async fn image_manifests( + &self, + id: &Sha256Digest, + deadline: Instant, + ) -> Result { + let suffix = if self.api_version().minor() >= 48 { + "?manifests=true" + } else { + "" + }; + decode( + self.call( + Operation::Image, + Method::GET, + format!("/images/{}/json{suffix}", escaped(id.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } +} diff --git a/crates/rustack-lambda-core/src/docker/tests.rs b/crates/rustack-lambda-core/src/docker/tests.rs index 7fedef33..863dcd4f 100644 --- a/crates/rustack-lambda-core/src/docker/tests.rs +++ b/crates/rustack-lambda-core/src/docker/tests.rs @@ -31,13 +31,13 @@ fn deadline() -> Instant { Instant::now() + Duration::from_secs(3) } -struct IncomingRequest { - head: String, +pub(super) struct IncomingRequest { + pub(super) head: String, body: Vec, - stream: UnixStream, + pub(super) stream: UnixStream, } impl IncomingRequest { - async fn reply(mut self, status: u16, body: &[u8]) { + pub(super) async fn reply(mut self, status: u16, body: &[u8]) { let head = format!( "HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() @@ -50,7 +50,7 @@ impl IncomingRequest { .unwrap() .unwrap(); } - async fn raw(mut self, response: &[u8]) { + pub(super) async fn raw(mut self, response: &[u8]) { timeout_at(deadline(), self.stream.write_all(response)) .await .unwrap() @@ -58,14 +58,14 @@ impl IncomingRequest { } } -struct Daemon { +pub(super) struct Daemon { directory: TempDir, - requests: mpsc::Receiver, + pub(super) requests: mpsc::Receiver, stop: watch::Sender, task: JoinHandle<()>, } impl Daemon { - fn new() -> Self { + pub(super) fn new() -> Self { Self::with_version("1.53") } fn with_version(version: &'static str) -> Self { @@ -117,20 +117,20 @@ impl Daemon { .parse() .unwrap() } - async fn wire(&self) -> DockerWire { + pub(super) async fn wire(&self) -> DockerWire { match DockerWire::connect(self.socket(), deadline()).await { Ok(wire) => wire, Err(DockerError::Http(error)) => panic!("fixture HTTP startup failed: {error:?}"), Err(error) => panic!("fixture startup failed: {error}"), } } - async fn next(&mut self) -> IncomingRequest { + pub(super) async fn next(&mut self) -> IncomingRequest { timeout_at(deadline(), self.requests.recv()) .await .unwrap() .unwrap() } - async fn close(self) { + pub(super) async fn close(self) { self.stop.send_replace(true); self.task.await.unwrap(); } diff --git a/crates/rustack-lambda-core/src/docker/types.rs b/crates/rustack-lambda-core/src/docker/types.rs index fe3b0751..e68d856f 100644 --- a/crates/rustack-lambda-core/src/docker/types.rs +++ b/crates/rustack-lambda-core/src/docker/types.rs @@ -37,6 +37,12 @@ pub enum DockerError { /// A bounded mailbox has no capacity. #[error("Docker wire is busy")] Busy, + /// An attached preparation caller cancelled before receiving an artifact. + #[error("Docker preparation cancelled")] + Cancelled, + /// An owned image mutation/cleanup is not yet proven settled. + #[error("Docker image cleanup is quarantined")] + Quarantined, /// An owned task panicked or failed to join. #[error("Docker supervision failed")] Worker, diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs index 1976e3b8..5877770b 100644 --- a/crates/rustack-lambda-core/tests/docker_wire.rs +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -9,10 +9,10 @@ use anyhow::{Context, Result, bail, ensure}; use bytes::Bytes; use http::StatusCode; use rustack_lambda_core::{ - artifact::{Architecture as ImageArchitecture, ImageReference, ResolvedImage}, + artifact::{Architecture as ImageArchitecture, ImageReference, PinnedImage}, docker::{ - DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, ImageResolver, OutputChannel, - OwnerFilter, + DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, ImageLeaseConfig, + ImageLeaseManager, ImagePinScope, OutputChannel, OwnerFilter, models::{ContainerCreateBody, HostConfig}, }, }; @@ -105,6 +105,8 @@ struct Environment { wire: DockerWire, names: Names, runtime: Option, + images: ImageLeaseManager, + artifacts: Vec, } impl Environment { async fn new() -> Result { @@ -113,33 +115,41 @@ impl Environment { let socket: DockerSocket = format!("unix://{socket}").parse()?; let wire = DockerWire::connect(socket, deadline()).await?; let run = Uuid::new_v4(); + let owner = Uuid::new_v4(); + let images = ImageLeaseManager::start( + wire.port(), + ImagePinScope::new(owner, run)?, + ImageLeaseConfig::default(), + )?; Ok(Self { wire, names: Names { relay: format!("rustack-wire-relay-{run}").parse()?, function: format!("rustack-wire-function-{run}").parse()?, - owner: Uuid::new_v4(), + owner, run, environment: EnvironmentId::default(), }, runtime: None, + images, + artifacts: Vec::new(), }) } - async fn image( - port: &DockerPort, - reference: &str, - architecture: &str, - ) -> Result { + async fn image(&mut self, reference: &str, architecture: &str) -> Result { let reference: ImageReference = reference.parse()?; let architecture = match architecture { "arm64" => ImageArchitecture::Arm64, "amd64" => ImageArchitecture::X86_64, _ => bail!("unknown fixture architecture"), }; - Ok(ImageResolver::new(port.clone()) - .resolve_local(&reference, architecture, None, deadline()) - .await?) + let image = self + .images + .port() + .prepare(&reference, architecture, None, deadline()) + .await?; + self.artifacts.push(image.clone()); + Ok(image) } async fn start(&mut self) -> Result<()> { @@ -157,14 +167,16 @@ impl Environment { && info.architecture.as_deref() == Some(daemon_arch), "daemon platform mismatch" ); - let relay = Self::image(&port, &relay, &arch).await?; - let function = Self::image(&port, &format!("rustack-p0-handler:{arch}"), &arch).await?; - let relay_plan = relay.metadata().execution.plan(None)?; - let function_plan = function.metadata().execution.plan(None)?; + let relay = self.image(&relay, &arch).await?; + let function = self + .image(&format!("rustack-p0-handler:{arch}"), &arch) + .await?; + let relay_plan = relay.resolved().metadata().execution.plan(None)?; + let function_plan = function.resolved().metadata().execution.plan(None)?; port.create_container_on_platform( &self.names.relay, &ContainerCreateBody { - image: Some(relay.metadata().image_id.as_str().into()), + image: Some(relay.resolved().metadata().image_id.as_str().into()), entrypoint: Some(relay_plan.entrypoint().to_vec()), cmd: Some(relay_plan.command().to_vec()), working_dir: Some(relay_plan.working_directory().into()), @@ -184,7 +196,7 @@ impl Environment { host_config: Some(limits("none".into(), 64 * 1024 * 1024, 100_000_000, 64)), ..Default::default() }, - relay.metadata().architecture, + relay.resolved().metadata().architecture, deadline(), ) .await?; @@ -198,7 +210,7 @@ impl Environment { port.create_container_on_platform( &self.names.function, &ContainerCreateBody { - image: Some(function.metadata().image_id.as_str().into()), + image: Some(function.resolved().metadata().image_id.as_str().into()), entrypoint: Some(function_plan.entrypoint().to_vec()), cmd: Some(function_plan.command().to_vec()), working_dir: Some(function_plan.working_directory().into()), @@ -220,7 +232,7 @@ impl Environment { )), ..Default::default() }, - function.metadata().architecture, + function.resolved().metadata().architecture, deadline(), ) .await?; @@ -320,8 +332,23 @@ impl Environment { ); Ok(()) }); - let joined = self.wire.shutdown().await.map_err(anyhow::Error::from); - function.and(runtime).and(relay).and(listed).and(joined) + self.artifacts.clear(); + let images = self + .images + .shutdown(deadline()) + .await + .map_err(anyhow::Error::from); + let joined = if images.is_ok() { + self.wire.shutdown().await.map_err(anyhow::Error::from) + } else { + Ok(()) + }; + function + .and(runtime) + .and(relay) + .and(listed) + .and(images) + .and(joined) } } @@ -401,7 +428,9 @@ async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() let result = async { let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; let port = environment.wire.port(); - let image = Environment::image(&port, &format!("rustack-p0-handler:{arch}"), &arch).await?; + let image = environment + .image(&format!("rustack-p0-handler:{arch}"), &arch) + .await?; for reset in [false, true] { let overrides = rustack_lambda_model::types::ImageConfig { entry_point: reset.then(Vec::new), @@ -412,11 +441,15 @@ async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() }), ..Default::default() }; - let plan = image.metadata().execution.plan(Some(&overrides))?; + let plan = image + .resolved() + .metadata() + .execution + .plan(Some(&overrides))?; port.create_container_on_platform( &environment.names.function, &ContainerCreateBody { - image: Some(image.metadata().image_id.as_str().into()), + image: Some(image.resolved().metadata().image_id.as_str().into()), entrypoint: Some(plan.entrypoint().to_vec()), cmd: Some(plan.command().to_vec()), working_dir: Some(plan.working_directory().into()), @@ -425,7 +458,7 @@ async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() host_config: Some(limits("none".into(), 192 * 1024 * 1024, 1_000_000_000, 256)), ..Default::default() }, - image.metadata().architecture, + image.resolved().metadata().architecture, deadline(), ) .await?; diff --git a/docs/index.md b/docs/index.md index 7d76b092..8767ee7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ ## Research -- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation, platform identity/default merge validation and actual RIC/diagnostic qualification; Bollard and historical Moby sources are pinned. +- [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation, platform identity/default merge validation, owned image pin/quota/cleanup preparation and actual RIC/diagnostic qualification; Bollard and historical Moby sources are pinned. - [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. - [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. - [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. diff --git a/docs/research/study-bollard-docker-transport-boundaries.md b/docs/research/study-bollard-docker-transport-boundaries.md index 14df184c..f9e02df6 100644 --- a/docs/research/study-bollard-docker-transport-boundaries.md +++ b/docs/research/study-bollard-docker-transport-boundaries.md @@ -103,3 +103,17 @@ The newly vendored Moby v28.0.0 is a historical API1.48 source pin, not the sour The same historical source strips Descriptor below API1.48 at `vendors/moby/api/server/router/image/image_routes.go:374`. Containerd nevertheless supplies a GraphDriver name from its snapshotter at `vendors/moby/daemon/containerd/image_inspect.go:117`; a nonempty GraphDriver cannot alone prove classic storage. The info status marker is emitted at `vendors/moby/daemon/containerd/service.go:112`, with the exact io.containerd.snapshotter.v1 value pinned at `vendors/moby/vendor/github.com/containerd/containerd/v2/plugins/types.go:39`. Classic VFS returns nil status at `vendors/moby/daemon/graphdriver/vfs/driver.go:87`. Normalize absent/null DriverStatus after matching classic driver evidence, while rejecting the explicit containerd marker and malformed rows. Fake-daemon/pure metadata tests cover these paths; no execution qualification of an old daemon is claimed. The final local resolver retains validated execution defaults in Arc storage, recomputes a canonical fingerprint on restore, binds platform inspection to the captured addressable parent Id and passes that Id with explicit platform to actual create. Its61 additional tests cover metadata/defaults/legacy API behavior and valid-value stale-hash mutation. Both real two-case Docker wire suites pass on arm64 VZ and amd64/QEMU. Full workspace1964 tests and standard build/fmt/Clippy/rustdoc pass. Deployment actors, availability leases, allowed pulls, unpacked-image quota proof and pool/provider integration still belong to the remaining P1 work. + +## Image pin and unpacked-size consumer preflight + +Pinned Moby's containerd tag path targets a resolved descriptor at `vendors/moby/daemon/containerd/image_tag.go:18`; the create/replace helper at:37 establishes the alias before later event/dangling-name work. This permits visible matching alias evidence to establish a tag commit after a lost response, while absence cannot establish that an unobserved tag will never commit. Cleanup must never replay Tag as a barrier. + +Image delete by repository reference removes that reference, and force/prune flags govern additional deletion at `vendors/moby/daemon/containerd/image_delete.go:24`. Rustack's adapter therefore accepts only allocated private-pin identities, never a caller source tag or content ID, and disables force/ancestor pruning. Published metadata still uses the original requested URI. + +The generated manifest summary explicitly separates compressed/content, unpacked image data and total storage at `vendors/bollard/codegen/swagger/src/models.rs:4236`. The real fixture's selected content size and unpacked size differ substantially. Modern quota proof must match the selected image summary by digest/platform and read ImageData.Size.Unpacked; a compressed Size or Total is not the same measurement. Legacy fallback remains restricted to verified classic storage. No image handler needs to run to obtain this metadata for already unpacked images; unavailable proof is an explicit preparation failure. + +Fresh-alias Tag preflight returns400/404 before mutation in the pinned Engine router at `vendors/moby/api/server/router/image/image_routes.go:472`; success201 follows backend return at:491. Classic reference validation precedes mutation at `vendors/moby/reference/store.go:125`; post-mutation storage failures map to500. The manager therefore treats only exact400/404 and local wire Busy as known no-effect rejection. + +A full500 response is still not a commit barrier. The containerd client invokes a remote Create at `vendors/moby/vendor/github.com/containerd/containerd/v2/client/image_store.go:73`; Moby wraps Create errors as System at `vendors/moby/daemon/containerd/image_tag.go:45`. The remote metadata update enters a database transaction at `vendors/moby/vendor/github.com/containerd/containerd/v2/core/metadata/bolt.go:54`, which does not establish that a cancelled/disconnected client's observed error stopped server commit. Keep HTTP500 in the uncertain phase and test a complete outer error followed by initial404 and a later matching alias commit. Do not replace retained quarantine with an HTTP-status-only completion assumption. + +The owned preparation consumer now implements that contract with bounded job/pin budgets, private-only tag/removal keys, shared release channels, phase registry/retained joins, a separate two-slot cleanup semaphore and emergency phase recovery on the captured runtime handle. The real harness keeps acquired artifacts through actual RIC calls and container retirement before joining pin cleanup. Both arm64 VZ and amd64/QEMU two-case suites pass; independent scoped image queries are empty afterward.33 new unit/lifecycle cases pass within321 Lambda core tests, plus76 core doctests; the workspace has1,997 passing tests and standard build/fmt/Clippy/rustdoc gates. Provider status/CAS, leased invocation capture, previous-run coordination, allowed pulls and the production executor remain required P1 consumers. diff --git a/specs/index.md b/specs/index.md index 96dd3f88..2c818173 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO and platform-aware local image resolution are also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 9e80346f..d2a5179f 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -156,6 +156,38 @@ ImageExecutionDefaults::plan(overrides) validates the complete composed argv, in Fingerprint a versioned, explicitly ordered serialization of captured addressable Id, retained selected_manifest digest, Linux architecture, inspected size and normalized inherited execution fields (entrypoint, command, root-default workdir, ordered environment, user). Exclude tags, unrelated RepoDigests, labels, creation timestamps and map iteration order. ImageMetadata::execution_fingerprint centralizes bounded SHA-256 serialization; ResolvedImage construction/deserialization recomputes it and rejects inconsistent metadata, including valid-string execution-plan mutations. No raw argv/environment or metadata is logged. Function overrides remain captured separately in ExecutionRevision and do not change the base inspection fingerprint. Invalid overrides fail before daemon IO. Fake-daemon tests cover platform query/version fallback, every rejection, deterministic identity under tag/label/RepoDigest ordering changes and explicit-empty overrides. Real wire qualification must create/run the function and relay using identities returned by this resolver. +### 2.9 Owned image pins and deployment preparation + +ImageLeaseManager is a retained actor above DockerWire. ImageLeaseConfig validates1..8192 total pins (default1024),1..32 accepted preparation jobs (default8,≤pin capacity) and1..60s independent Docker IO budget (default5s). These map to validated operator budgets; no process environment is mutated. ImagePinScope contains nonnil owner/run UUIDs supplied by the runtime's directory-lock owner. Each pin allocates a fresh UUID before any mutation; its only possible alias is rustack-internal/lambda-/:, with lowercase simple UUIDs. ImagePin has no arbitrary-string constructor/deserializer. Wire tag/remove APIs accept this type; removal never targets an operator source reference, bare content ID or a generic image filter. Request removal uses force=false/noprune=true. Daemon collection of content with no remaining references is separate from Rustack deleting operator tags or pruning other images. + +The cloneable preparation port accepts a validated reference, architecture, optional ImageConfig and absolute deadline. Reserve both a preparation slot and a pin slot before bounded mailbox admission; accepted-plus-running preparations cannot exceed their configured budget. A cancelled/expired/closed request before mutation releases those reservations. Workers resolve read-only metadata, record the expected addressable parent ID, then issue tag under an independent bounded IO budget retained across caller cancellation. Validate the private alias resolves to that exact parent. Measure unpacked size before publishing a PinnedImage. Recheck caller deadline/cancellation before success. This is preparation only: the deployment actor still owns Pending/Active/update status, resource/revision CAS, allowed pulls and rollback on stale results. + +PinnedImage is a closed live artifact containing immutable ResolvedImage, validated positive UnpackedImageSize≤10GiB and a shared lease. Its Debug excludes image defaults/argv/environment. Clones retain the private alias; the last lease closes a release channel. ImageLeaseManager retains every worker and pin reservation through actual cleanup. A cancelled response waiter cannot orphan a completed image sitting in its response channel. Capture must eventually use this leased artifact; serialized metadata alone is insufficient to authorize Docker execution. Snapshots serialize metadata and must reacquire/revalidate image availability during restore, not deserialize a live lease. + +For modern stores, inspect manifests=true by the captured immutable parent ID (never the user tag). Find exactly one available image-kind summary whose canonical ID and descriptor digest equal selected_manifest and whose image platform matches Linux/architecture. Require a positive ImageData.Size.Unpacked≤10GiB. Reject attestations, duplicates, incomplete content, incompatible platforms and missing/zero/negative/oversized unpacked proof. This reported unpacked-storage measure is the local quota; do not add compressed Content to it or confuse Total with it. For descriptor-free images verified as classic by §2.8, use the classic inspect Size. Recheck deadlines before returning the validated size. No function code or probe container is run by preparation. + +Cleanup invalidates the lease, verifies alias identity and removes only that private alias, then confirms absence after all possible create effects are settled. A fully acknowledged tag is known created. Exact fresh-alias HTTP400/404 from the trusted Engine and wire Busy admission failure prove no tag was committed and release that reservation. Every other failed outcome remains uncertain. A complete HTTP500 is not a completion barrier: Docker can return an internal containerd gRPC error while the nested server still commits. For uncertain tag completion, one404 does not prove cleanliness: retain quarantine/reservation until a matching visible alias establishes the tag commit, then remove/confirm absence. Never reissue a tag as a cleanup barrier. A mismatching alias is not deleted. The rustack-internal/lambda-/ namespace is reserved for this owner and must not be concurrently retargeted by operators or other clients. Docker provides no expected-target CAS for alias deletion, so identity verification plus removal assumes that exclusive namespace; observed mismatches quarantine rather than delete. Failed cleanup retries1/2/4 seconds capped30s, retains ownership and temporarily rejects new preparations; verified recovery clears quarantine. Caller cleanup deadlines bound waiters, not the retained cleanup future. Worker panic permanently stops admission, revokes leases and retains phase/identity through supervised recovery; successful quiescence must not hide it. + +Owner shutdown is a separate priority signal. The runtime first joins deployment/invocation/environment consumers, then stops this owner; shutdown revokes all leases and joins known pin cleanup before DockerWire shutdown. A cancelled shutdown waiter retains the join handle; failed settlement is cached. Owner Drop signals the same path even outside an entered runtime. A root-actor failure transfers recorded pin phases/reservations and aborted worker joins to retained emergency cleanup on its captured runtime handle. Startup reconciliation of previous runs remains with the directory-lock coordinator and may act only on the same persistent owner; this manager never scans/deletes generic user images. Required tests include bounds/admission, dropped waiters, clone lifetime, tag retargeting, partial/late tag results, identity conflicts, malformed unpacked proofs, worker/root supervision failure and real private-alias cleanup on both retained daemons. + +```text +┌─ deployment actor / future provider CAS ──────────────────────────────┐ +│ reserve job → prepare(ref, architecture, overrides, deadline) │ +└───────────────────────────────┬──────────────────────────────────────┘ + ▼ bounded preparation mailbox +┌─ ImageLeaseManager (owns pin slots and worker joins) ─────────────────┐ +│ resolve parent → register phase → tag unique private alias │ +│ → verify alias → prove unpacked quota → publish shared lease │ +│ │ │ +│ last lease / priority stop ──┴─▶ verify/remove/confirm absent │ +│ IO error / uncertain create ─▶ quarantine + retained retry │ +└───────────────────────────────┬──────────────────────────────────────┘ + ▼ typed pin operations, bounded drivers +┌─ DockerWire ─────────────────────────────────────────────────────────┐ +│ tag and manifest queries: business lane │ alias cleanup: cleanup lane│ +└──────────────────────────────────────────────────────────────────────┘ +``` + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 0866a5ee..b17acf4b 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -69,7 +69,9 @@ All budgets are validated once; limits below are local defaults and safe configu | workflowCallbackTokens / workflowCallbackTombstones |4,096 /100,000|1..65,536 /1..1,000,000; closed-token TTL follows execution retention | | lambdaDockerEnvironments |32|1..32 initially, including busy/idle/quarantined environments | | lambdaDockerMemoryBytes |8 GiB|256 MiB..1 TiB; includes relay caps and all allocated environments | -| lambdaDockerDeploymentJobs |8|1..32; queued plus running deployments | +| lambdaDockerDeploymentJobs |8|1..32; queued plus running deployments;≤image pin budget | +| lambdaDockerImagePins |1024|1..8,192; preparations, live revisions/captures and quarantined aliases | +| lambdaDockerIoSeconds |5|1..60; independent bounded management/mutation and partial-control IO budget | | lambdaDockerPullSeconds / lambdaDockerStartSeconds |120 /15|1..600 /1..60 | | lambdaDockerCleanupSeconds |10|1..60; bounded by global shutdown remaining time | | workflowDrainSeconds |5|0..60; bounded by global shutdown remaining time | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 5641ee08..6a21357d 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -81,6 +81,8 @@ Current wire qualification on2026-10-25 passes on Engine29.5.2 Linux arm64 in na Local image resolution now extends `test-lambda-docker-wire` with a second real create/inspect case: clear inherited CMD and reset ENTRYPOINT without daemon re-inheritance. The execution case uses the captured addressable parent Id plus explicit platform, and preserves the selected child manifest independently.61 unit additions cover validated inherited defaults/plans, deterministic base fingerprints, ten valid-value stale-hash snapshot mutations, familiar registry identities, actual captured-parent query encoding and legacy API1.44/1.48 classic evidence. Null/missing VFS status remains valid; explicit containerd markers and contradictory manifest summaries reject. Historical engine code is pinned at vendors/moby v28.0.0; fake older-version cases are not old-daemon execution qualification. Current two-case suites pass on both retained daemons. Local resolution does not establish deployment status/CAS, availability leases, allowed registry pulls or unpacked-image quota enforcement. +Owned image preparation now extends the same real wire suite: acquire private aliases/live PinnedImages, prove selected unpacked size, retain leases through handler/container lifetime, remove containers, release/join pins, then stop DockerWire. Both retained architecture lanes pass and subsequent scoped image queries find no private aliases. The33 additional cases cover size proof, shared lease release, precommit rejection, complete outer500 with delayed nested commit, initial404 uncertainty, discarded successful responses, conflicting aliases, lost/failed DELETE, capacity observation and retained shutdown/worker/root recovery including off-runtime Drop. Source-pinned exceptions treat only exact fresh-alias400/404 and wire Busy as no-effect; HTTP500 remains uncertain. These gates do not establish provider publication/CAS, snapshot live-lease reacquisition, previous-run reconciliation or a production executor. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 1496cfb1..a613a0b7 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, host Runtime API, immutable artifact hardening, invocation admission, binary-owned cross-service adapters, validated YAML settings, runtime worker ownership and service snapshots. Missing: Docker executor/relay and image resolution, full ImageConfig propagation, Step Functions protocol/model/core, ASL engine, integration handles, workflow snapshots and test fixtures. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, existing gateway/YAML/runtime/snapshot infrastructure. Missing: provider asynchronous image publication/status/CAS, leased invocation capture, Docker environment/pool executor, startup ownership/reconciliation/probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -142,7 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Deployment/image actors (including status/CAS, availability leases, allowed pulls and unpacked quota proof), remaining stream/network/probe consumers and environment/pool/provider integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Provider deployment/status/CAS and leased invocation capture, directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement task1.3 deployment/image and environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts, together with remaining stream/network/probe operations, followed by startup/resource probes and task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement provider deployment/status/CAS and leased invocation capture using ImageLeaseManager, then task1.3 environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe operations, followed by task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index ab078842..46c69b9a 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -178,6 +178,25 @@ Actual official-RIC wire execution now consumes resolver identities and material Deployment/status/CAS actors, image availability leases, allowed pulls, unpacked-image quota proof, environment/pool owners, remaining network/log/probe consumers and B01/B03 remain required before P1 closes. This slice does not advertise Docker invocation in Rustack. +## P1 task1.3 owned image preparation review + +The independent read-only design and implementation review verified typed private-only mutation endpoints, shared lease lifetime, retained cancellation/cleanup, digest/platform unpacked proof, and worker/root recovery. The final recheck found no remaining concrete in-slice issue. Source review preserves the HTTP500 uncertainty rule: a returned nested containerd gRPC error does not prove its server stopped committing. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W39 | P2; runtime §2.9, private alias removal | Verify-ID and DELETE are separate operations; Docker has no expected-target CAS. Reserve the private namespace exclusively and state that concurrent external retargeting invalidates that guarantee. Observed mismatches quarantine without deletion | Wrong-target/repair regression; generated-only pin identities and force=false/noprune=true endpoints; pinned classic/containerd source | +| W40 | P3; `docker/lease.rs:65` | IO budget accepted subsecond positive durations despite the1..60s contract. Constructor now enforces the actual lower bound |1ns and over60s rejection plus typed config checks | +| W41 | P2; `docker/lease.rs:606`, self-review with independent recheck | Root failure with no registered rows could drop queued work without recording failure. Actor now records settled completion explicitly; abnormal Drop closes admission/marks failure even with zero rows | Empty-root failure, aborted-root cached failure, worker panic and off-runtime root Drop tests | +| W42 | P2; `docker/lease.rs:275`, self-review with independent recheck | Acquiring every semaphore permit to observe emptiness consumed other preparation capacity and distorted pin counts. Dedicated bounded count/watch observation now leaves admission capacity untouched | Wait-empty observer remains pending while a second legitimate preparation succeeds; clone lifetime and cancelled observer cases | + +ImageLeaseManager now owns accepted preparations, live aliases and uncertain cleanup through retained workers/registry reservations. PinnedImage cannot be deserialized from metadata; clones keep the release channel alive, and shutdown revokes further use after consumers settle. Successful or discarded response delivery retains artifact ownership correctly. UnpackedImageSize validates1..10GiB; modern proof selects exactly one available image summary by digest/platform and reads Unpacked, with verified classic fallback. Preparation does not run user code or pull/build images. + +Thirty-three new cases pass within321 Lambda core tests, including13 unpacked proof cases, two scalar/shared-lease cases and18 preparation/lifecycle cases. The latter cover exact precommit400/404 rejection, lost/postcommit/late nested500 outcomes, initial404 quarantine, no Tag replay, caller/response abandonment, bounded admission, alias conflict repair, lost/failed DELETE recovery, cancelled/timed shutdown, cached supervision failure and cleanup outside an entered runtime.76 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,997 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. + +The real Docker wire harness now acquires PinnedImages, keeps them through actual handler calls/container cleanup, and joins the image manager before DockerWire shutdown. Both two-case suites pass on arm64 VZ and amd64/QEMU, retaining near6MiB computed warm/error and empty-override behavior. Independent daemon queries find no private pin aliases. These are preparation/control-path qualifications, not provider/executor or default-latency acceptance. No dependencies/versions changed. Strict boundary checks report the same65 preexisting B03 findings, with no new pin/preparation/defaults-path finding; B01 remains an open dependency gate. + +Next consume these artifacts in provider deployment/status/CAS and invocation capture, followed by pool/environment actors, startup directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and complete P1/M0 acceptance. No Docker execution capability is advertised by the current binary yet. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 1d74e8b5..9edeaf2c 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -168,6 +168,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [local resolution](./rustack-workflows-20-lambda-runtime-design.md#28-local-image-resolution-and-selected-platform-identity), [live/source evidence](../docs/research/study-bollard-docker-transport-boundaries.md#platform-specific-image-identity-follow-up). - Date:2026-10-25. This refines D19's generated Id representation; it does not promise registry existence, cross-platform execution or completed deployment/provider integration. +## D22 — Own private image aliases through deployment and invocation lifetime + +- Context: immutable IDs prevent tag drift but do not retain addressability when mutable image references are replaced. A prepared revision also needs unpacked-size proof rather than containerd compressed Size. +- Alternatives: rely on an operator tag/ID remaining present; hold never-started containers per revision; create private aliases under validated owner/run/pin identities with retained lease supervision. +- Decision: a bounded image lease actor prepares a private alias, verifies immutable identity and unpacked quota, and publishes a shared live artifact; only known private aliases can be removed. Record mutation phase before IO and retain uncertain cleanup/reservations across caller cancellation, deadlines and supervised recovery. +- Why: published versions and captured calls must own artifact availability. Observing one absent alias after an uncertain tag does not prove that a late tag cannot appear; visible matching commit followed by removal can establish cleanliness. A root/worker failure must preserve these identities rather than merely release permits. +- Pinned by: [owned image preparation](./rustack-workflows-20-lambda-runtime-design.md#29-owned-image-pins-and-deployment-preparation), [transport/source study](../docs/research/study-bollard-docker-transport-boundaries.md). +- Date:2026-10-25. Provider status/CAS, allowed pulls, directory ownership and previous-run reconciliation remain separate required consumers. + ## Glossary | Term | Meaning in this specification | From cfc894fa3c14efae0695a10c5f04c42a789db998 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 22:01:04 -0700 Subject: [PATCH 015/106] phase 1: guard image deployment settlement and capture live artifact leases Add typed readiness/update states and closed store-issued candidates. Atomically settle by resource UUID, original revision and generation; retain previous revisions during failed or pending updates. Capture live PinnedImages, validate canonical image hashes and unpacked sizes, migrate snapshot states without inventing leases, and reject unsupported image publication before mutation. Verified 2,022 workspace tests, 346 Lambda core cases and 92 core doctests; workspace build, nightly formatting, Clippy, rustdoc and Native SDK regressions pass. Independent review findings W43-W44 and self-review fixes W45-W46 are addressed, including deterministic HTTP timer tests. Strict baseline B03 and advisory B01 remain open. Retained provider deployment ownership/API wiring, Docker pool/executor and P2-P5 remain required; no M0 completion is claimed. --- crates/rustack-core/Cargo.toml | 2 +- crates/rustack-core/src/http.rs | 4 +- crates/rustack-lambda-core/src/artifact.rs | 45 +- .../rustack-lambda-core/src/artifact/image.rs | 23 + .../src/artifact/pinned.rs | 15 + crates/rustack-lambda-core/src/deployment.rs | 493 +++++++++++++ .../src/deployment_tests.rs | 689 ++++++++++++++++++ .../rustack-lambda-core/src/executor/auto.rs | 5 +- .../src/function_config.rs | 37 +- crates/rustack-lambda-core/src/lib.rs | 1 + crates/rustack-lambda-core/src/provider.rs | 519 ++++++++----- crates/rustack-lambda-core/src/resolver.rs | 4 +- crates/rustack-lambda-core/src/storage.rs | 65 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 42 +- .../rustack-workflows-60-operations-design.md | 3 +- .../rustack-workflows-70-verification-plan.md | 3 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 15 + specs/rustack-workflows-99-decisions.md | 10 + 20 files changed, 1784 insertions(+), 197 deletions(-) create mode 100644 crates/rustack-lambda-core/src/deployment.rs create mode 100644 crates/rustack-lambda-core/src/deployment_tests.rs diff --git a/crates/rustack-core/Cargo.toml b/crates/rustack-core/Cargo.toml index d4728ffe..32890ce2 100644 --- a/crates/rustack-core/Cargo.toml +++ b/crates/rustack-core/Cargo.toml @@ -31,4 +31,4 @@ uuid = { workspace = true } [dev-dependencies] proptest = { workspace = true } rstest = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-core/src/http.rs b/crates/rustack-core/src/http.rs index 6debd1f0..4223bd84 100644 --- a/crates/rustack-core/src/http.rs +++ b/crates/rustack-core/src/http.rs @@ -320,7 +320,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_should_expire_pending_body_without_eof() { let budget = BodyBudget::new(3, Duration::from_secs(1), Duration::from_millis(5)).unwrap(); assert!(matches!( @@ -372,7 +372,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_should_not_reset_total_deadline_on_progress() { let budget = BodyBudget::new(1000, Duration::from_millis(20), Duration::from_millis(10)).unwrap(); diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs index 9e6ef2a2..cd263b86 100644 --- a/crates/rustack-lambda-core/src/artifact.rs +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -34,8 +34,49 @@ use crate::{storage::stage_zip, test_zip::minimal_zip}; pub enum DeploymentArtifact { /// Successfully staged ZIP, with shared lifetime ownership. Zip(ValidatedArtifact), - /// Resolved Linux image, pinned to immutable daemon content. - Image(ResolvedImage), + /// Live Linux image lease, retained through backend cleanup. + Image(PinnedImage), +} + +/// Image deployment availability; snapshot metadata never grants a live lease. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub enum ImageArtifact { + /// No image has been prepared. + #[default] + Unresolved, + /// Immutable restored metadata requiring availability reacquisition. + Restored(ResolvedImage), + /// Prepared image with shared availability ownership. + Pinned(PinnedImage), +} +impl ImageArtifact { + /// Captured metadata, whether or not runtime availability has been acquired. + /// + /// ``` + /// assert!(rustack_lambda_core::artifact::ImageArtifact::Unresolved.resolved().is_none()); + /// ``` + #[must_use] + pub fn resolved(&self) -> Option<&ResolvedImage> { + match self { + Self::Unresolved => None, + Self::Restored(image) => Some(image), + Self::Pinned(image) => Some(image.resolved()), + } + } + + /// Live artifact ownership, absent for restored metadata alone. + /// + /// ``` + /// assert!(rustack_lambda_core::artifact::ImageArtifact::Unresolved.pinned().is_none()); + /// ``` + #[must_use] + pub fn pinned(&self) -> Option<&PinnedImage> { + match self { + Self::Pinned(image) => Some(image), + Self::Unresolved | Self::Restored(_) => None, + } + } } /// Shared ownership of a ZIP archive and the extracted code it produced. diff --git a/crates/rustack-lambda-core/src/artifact/image.rs b/crates/rustack-lambda-core/src/artifact/image.rs index b343ebf5..692a3e25 100644 --- a/crates/rustack-lambda-core/src/artifact/image.rs +++ b/crates/rustack-lambda-core/src/artifact/image.rs @@ -2,6 +2,7 @@ use std::{fmt, net::Ipv6Addr, str::FromStr}; +use base64::{Engine, engine::general_purpose::STANDARD}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use typed_builder::TypedBuilder; @@ -516,6 +517,28 @@ impl ResolvedImage { .try_into() .expect("valid unit image metadata") } + /// Base64 SHA-256 of canonical inspected execution metadata, independent of a tag. + /// + /// # Errors + /// Rejects a malformed canonical fingerprint before decoding it. + /// + /// ```no_run + /// # fn hash(image: &rustack_lambda_core::artifact::ResolvedImage) -> Result<(), rustack_lambda_core::artifact::ArtifactError> { + /// let code_hash = image.code_sha256()?; + /// assert_eq!(code_hash.len(), 44); + /// # Ok(()) } + /// ``` + pub fn code_sha256(&self) -> Result { + let digest = self + .0 + .inspection_hash + .as_str() + .strip_prefix("sha256:") + .ok_or(ArtifactError::Inspection)?; + let bytes = hex::decode(digest).map_err(|_| ArtifactError::Inspection)?; + Ok(STANDARD.encode(bytes)) + } + /// Immutable inspected metadata; Invoke must use `image_id` rather than a tag. /// /// ``` diff --git a/crates/rustack-lambda-core/src/artifact/pinned.rs b/crates/rustack-lambda-core/src/artifact/pinned.rs index e81a63fb..4fd386d7 100644 --- a/crates/rustack-lambda-core/src/artifact/pinned.rs +++ b/crates/rustack-lambda-core/src/artifact/pinned.rs @@ -75,6 +75,21 @@ impl fmt::Debug for PinnedImage { } } impl PinnedImage { + #[cfg(test)] + pub(crate) fn fixture(resolved: ResolvedImage) -> Self { + let (release, _) = watch::channel(()); + let (_, stop) = watch::channel(false); + let size = UnpackedImageSize::try_from(resolved.metadata().size_bytes) + .expect("validated unit image size"); + Self::new( + resolved, + size, + release, + Arc::new(AtomicBool::new(true)), + stop, + ) + } + pub(crate) fn new( resolved: ResolvedImage, size: UnpackedImageSize, diff --git a/crates/rustack-lambda-core/src/deployment.rs b/crates/rustack-lambda-core/src/deployment.rs new file mode 100644 index 00000000..ef722a6a --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment.rs @@ -0,0 +1,493 @@ +//! Typed deployment transitions and stale-result protection at the store boundary. +//! +//! A retained worker owns each candidate until settlement. Store keys bind resource +//! lifetime, original revision and deployment generation; metadata is never a live lease. + +use std::fmt; + +use rustack_lambda_model::types::ImageConfig; +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::{ + artifact::{Architecture, ImageArtifact, ImageReference, PinnedImage}, + error::LambdaServiceError, + identity::FunctionResourceId, + storage::{FunctionRecord, FunctionStore, VersionRecord}, +}; + +/// A nonnil identity for one accepted deployment attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +#[non_exhaustive] +pub struct DeploymentGeneration(Uuid); + +impl TryFrom for DeploymentGeneration { + type Error = LambdaServiceError; + fn try_from(value: Uuid) -> Result { + if value.is_nil() { + return Err(invalid("Deployment generation cannot be nil")); + } + Ok(Self(value)) + } +} +impl TryFrom for DeploymentGeneration { + type Error = LambdaServiceError; + fn try_from(value: String) -> Result { + if value.len() != 36 { + return Err(invalid("Invalid deployment generation")); + } + let id = Uuid::parse_str(&value).map_err(|_| invalid("Invalid deployment generation"))?; + Self::try_from(id) + } +} +impl From for String { + fn from(value: DeploymentGeneration) -> Self { + value.0.to_string() + } +} + +/// Bounded public failure vocabulary; no backend response or secret is retained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub enum ImageDeploymentFailure { + /// The requested content is unavailable. + ImageNotFound, + /// Image metadata or its effective entry configuration is invalid. + InvalidImage, + /// The selected architecture cannot run under the configured policy. + UnsupportedPlatform, + /// Docker is unavailable or its ownership health is degraded. + RuntimeUnavailable, + /// Deployment exceeded its absolute deadline. + DeadlineExceeded, + /// Deployment was cancelled before settlement. + Cancelled, + /// Its retained supervisor failed. + SupervisorFailed, +} +impl ImageDeploymentFailure { + /// Static, bounded API reason with no untrusted backend text. + /// + /// ``` + /// use rustack_lambda_core::deployment::ImageDeploymentFailure; + /// assert!(ImageDeploymentFailure::InvalidImage.reason().len() < 256); + /// ``` + #[must_use] + pub const fn reason(self) -> &'static str { + match self { + Self::ImageNotFound => "Requested image content is unavailable", + Self::InvalidImage => "Image metadata or entry configuration is invalid", + Self::UnsupportedPlatform => "Image platform is unsupported by this runtime", + Self::RuntimeUnavailable => "Image runtime is unavailable", + Self::DeadlineExceeded => "Image deployment deadline exceeded", + Self::Cancelled => "Image deployment was cancelled", + Self::SupervisorFailed => "Image deployment supervisor failed", + } + } + + /// AWS-compatible reason category for the local failure. + /// + /// ``` + /// use rustack_lambda_core::deployment::ImageDeploymentFailure; + /// assert_eq!(ImageDeploymentFailure::InvalidImage.code(), "InvalidImage"); + /// ``` + #[must_use] + pub const fn code(self) -> &'static str { + match self { + Self::ImageNotFound => "ImageDeleted", + Self::InvalidImage | Self::UnsupportedPlatform => "InvalidImage", + Self::RuntimeUnavailable + | Self::DeadlineExceeded + | Self::Cancelled + | Self::SupervisorFailed => "InternalError", + } + } +} + +/// One authoritative state for both function readiness and update progress. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + content = "detail", + rename_all = "camelCase", + deny_unknown_fields +)] +#[non_exhaustive] +pub enum DeploymentState { + /// Usable revision with no failed or pending update. + #[default] + Ready, + /// Initial deployment; no previous executable revision exists. + Creating(DeploymentGeneration), + /// Initial deployment failed. + CreateFailed(ImageDeploymentFailure), + /// Previous revision remains usable while a candidate prepares. + Updating(DeploymentGeneration), + /// Previous revision remains usable after a failed update. + UpdateFailed(ImageDeploymentFailure), +} +impl DeploymentState { + /// Public Lambda function state. + /// + /// ``` + /// assert_eq!(rustack_lambda_core::deployment::DeploymentState::Ready.state(), "Active"); + /// ``` + #[must_use] + pub const fn state(self) -> &'static str { + match self { + Self::Ready | Self::Updating(_) | Self::UpdateFailed(_) => "Active", + Self::Creating(_) => "Pending", + Self::CreateFailed(_) => "Failed", + } + } + + /// Public Lambda update status. + /// + /// ``` + /// assert_eq!(rustack_lambda_core::deployment::DeploymentState::Ready.update_status(), "Successful"); + /// ``` + #[must_use] + pub const fn update_status(self) -> &'static str { + match self { + Self::Ready => "Successful", + Self::Creating(_) | Self::Updating(_) => "InProgress", + Self::CreateFailed(_) | Self::UpdateFailed(_) => "Failed", + } + } + + /// Public failure reason, if a deployment failed. + /// + /// ``` + /// assert!(rustack_lambda_core::deployment::DeploymentState::Ready.failure().is_none()); + /// ``` + #[must_use] + pub const fn failure(self) -> Option { + match self { + Self::CreateFailed(reason) | Self::UpdateFailed(reason) => Some(reason), + _ => None, + } + } + + /// `StateReason` describes initial readiness failure, not a failed update. + pub(crate) const fn creation_failure(self) -> Option { + match self { + Self::CreateFailed(failure) => Some(failure), + _ => None, + } + } + + pub(crate) fn ensure_mutable(self) -> Result<(), LambdaServiceError> { + if matches!(self, Self::Creating(_) | Self::Updating(_)) { + return Err(conflict("A function deployment is already in progress")); + } + Ok(()) + } + + pub(crate) fn restore(legacy: &str, saved: Option) -> Result { + let state = match saved { + Some(state) if state.state() == legacy => state, + Some(_) => return Err(invalid("Snapshot deployment states contradict each other")), + None => match legacy { + "Active" => Self::Ready, + "Pending" => Self::CreateFailed(ImageDeploymentFailure::Cancelled), + "Failed" => Self::CreateFailed(ImageDeploymentFailure::InvalidImage), + _ => return Err(invalid("Unsupported snapshot function state")), + }, + }; + Ok(match state { + Self::Creating(_) => Self::CreateFailed(ImageDeploymentFailure::Cancelled), + Self::Updating(_) => Self::UpdateFailed(ImageDeploymentFailure::Cancelled), + state => state, + }) + } +} + +/// Closed candidate issued by a store transaction, owned by a deployment worker. +/// +/// Debug omits its image reference, entry configuration and environment. +/// +/// Construction is restricted to validated store transactions: +/// +/// ```compile_fail +/// use rustack_lambda_core::deployment::ImageDeployment; +/// let forged = ImageDeployment::builder(); +/// ``` +#[derive(Clone, TypedBuilder)] +#[builder( + builder_method(vis = ""), + build_method(vis = ""), + builder_type(vis = "") +)] +#[non_exhaustive] +pub struct ImageDeployment { + name: String, + resource: FunctionResourceId, + original_revision: String, + generation: DeploymentGeneration, + initial: bool, + candidate: VersionRecord, + architecture: Architecture, + reference: ImageReference, +} +impl fmt::Debug for ImageDeployment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ImageDeployment") + .field("resource", &self.resource) + .field("generation", &self.generation) + .field("initial", &self.initial) + .finish_non_exhaustive() + } +} +impl ImageDeployment { + fn new( + record: &FunctionRecord, + candidate: VersionRecord, + initial: bool, + ) -> Result { + crate::resolver::FunctionName::parse(&record.name)?; + if record.latest.revision_id.len() != 36 + || Uuid::parse_str(&record.latest.revision_id).is_err() + { + return Err(invalid("Invalid deployment revision identity")); + } + let architecture = crate::function_config::validate_execution_settings(&candidate)?; + let reference = candidate + .image_uri + .as_deref() + .ok_or_else(|| invalid("Image reference is required"))? + .parse() + .map_err(|_| invalid("Invalid image reference"))?; + Ok(Self::builder() + .name(record.name.clone()) + .resource(record.resource_id) + .original_revision(record.latest.revision_id.clone()) + .generation(DeploymentGeneration(Uuid::new_v4())) + .initial(initial) + .candidate(candidate) + .architecture(architecture) + .reference(reference) + .build()) + } + + /// Reference captured at acceptance for bounded image preparation. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let reference = job.reference(); + /// # } + /// ``` + #[must_use] + pub fn reference(&self) -> &ImageReference { + &self.reference + } + + /// Validated requested architecture for this candidate. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let architecture = job.architecture(); + /// # } + /// ``` + #[must_use] + pub const fn architecture(&self) -> Architecture { + self.architecture + } + + /// Complete entry overrides; omitted and empty arrays remain distinct. + /// + /// ```no_run + /// # fn inspect(job: &rustack_lambda_core::deployment::ImageDeployment) { + /// let overrides = job.image_config(); + /// # } + /// ``` + #[must_use] + pub fn image_config(&self) -> Option<&ImageConfig> { + self.candidate.image_config.as_ref() + } + + fn matches(&self, record: &FunctionRecord) -> bool { + let state = if self.initial { + DeploymentState::Creating(self.generation) + } else { + DeploymentState::Updating(self.generation) + }; + record.resource_id == self.resource + && record.latest.revision_id == self.original_revision + && record.latest.deployment == state + } +} + +impl FunctionStore { + /// Insert an initial Pending image deployment after the caller reserves work. + /// + /// # Errors + /// Rejects duplicate/invalid records and non-image or already resolved candidates. + /// + /// ```no_run + /// # fn accept(store: &rustack_lambda_core::storage::FunctionStore, record: rustack_lambda_core::storage::FunctionRecord) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let candidate = store.insert_image_deployment(record)?; + /// # Ok(()) } + /// ``` + pub fn insert_image_deployment( + &self, + mut record: FunctionRecord, + ) -> Result { + if record.latest.package_type != "Image" + || !matches!(record.latest.image, ImageArtifact::Unresolved) + || record.latest.deployment != DeploymentState::Ready + || !record.versions.is_empty() + { + return Err(invalid( + "Initial image candidate must be unresolved and unpublished", + )); + } + record.latest.code_sha256.clear(); + record.latest.code_size = 0; + let job = ImageDeployment::new(&record, record.latest.clone(), true)?; + record.latest.deployment = DeploymentState::Creating(job.generation); + self.insert(record)?; + Ok(job) + } + + /// Reserve an image update while preserving the previous usable revision. + /// + /// # Errors + /// Rejects stale resource/revision keys, pending work, package changes or invalid settings. + /// + /// ```no_run + /// # fn accept(store: &rustack_lambda_core::storage::FunctionStore, record: &rustack_lambda_core::storage::FunctionRecord) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// let job = store.begin_image_update(&record.name, record.resource_id, + /// &record.latest.revision_id, "worker:v2".parse().map_err(|_| rustack_lambda_core::error::LambdaServiceError::DockerNotAvailable)?, + /// rustack_lambda_core::artifact::Architecture::Arm64)?; + /// # Ok(()) } + /// ``` + pub fn begin_image_update( + &self, + name: &str, + resource: FunctionResourceId, + expected: &str, + reference: ImageReference, + architecture: Architecture, + ) -> Result { + self.try_update_if_revision(name, Some(expected), |record| { + if record.resource_id != resource { + return Err(conflict("Function resource changed")); + } + record.latest.deployment.ensure_mutable()?; + if record.latest.package_type != "Image" { + return Err(invalid("Deployment package type cannot change")); + } + let mut candidate = record.latest.clone(); + candidate.image_uri = Some(reference.into()); + candidate.architectures = vec![architecture.as_str().into()]; + candidate.image = ImageArtifact::Unresolved; + candidate.code_sha256.clear(); + candidate.code_size = 0; + candidate.deployment = DeploymentState::Ready; + let initial = matches!(record.latest.deployment, DeploymentState::CreateFailed(_)); + let job = ImageDeployment::new(record, candidate.clone(), initial)?; + if initial { + record.latest = candidate; + } + record.latest.deployment = if initial { + DeploymentState::Creating(job.generation) + } else { + DeploymentState::Updating(job.generation) + }; + Ok(job) + }) + } + + /// Atomically settle a candidate only while all accepted identity keys match. + /// + /// Stale/duplicate/deleted results return false and cannot mutate live metadata. + /// The caller must retain and join the worker independently of this transaction. + /// + /// # Errors + /// Rejects a revoked/mismatching lease or an invalid candidate without committing it. + /// + /// ```no_run + /// # fn finish(store: &rustack_lambda_core::storage::FunctionStore, job: &rustack_lambda_core::deployment::ImageDeployment) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// store.settle_image_deployment(job, Err(rustack_lambda_core::deployment::ImageDeploymentFailure::Cancelled))?; + /// # Ok(()) } + /// ``` + pub fn settle_image_deployment( + &self, + job: &ImageDeployment, + outcome: Result, + ) -> Result { + let Some(before) = self.get(&job.name)? else { + return Ok(false); + }; + if !job.matches(&before) { + return Ok(false); + } + let prepared = match outcome { + Ok(image) => { + if !image.is_live() + || image.resolved().metadata().requested_reference != job.reference + || image.resolved().metadata().architecture != job.architecture + { + return Err(invalid( + "Prepared image does not match deployment candidate", + )); + } + image + .resolved() + .metadata() + .execution + .plan(job.image_config()) + .map_err(|_| invalid("Invalid effective image entry configuration"))?; + let mut candidate = job.candidate.clone(); + candidate.code_sha256 = image + .resolved() + .code_sha256() + .map_err(|_| invalid("Invalid image fingerprint"))?; + candidate.code_size = image.unpacked_size().get(); + candidate.image = ImageArtifact::Pinned(image); + candidate.deployment = DeploymentState::Ready; + candidate.revision_id = Uuid::new_v4().to_string(); + candidate.last_modified = crate::provider::now_iso8601(); + Ok(candidate) + } + Err(failure) => Err(failure), + }; + match self.update(&job.name, |record| { + if !job.matches(record) { + return false; + } + match prepared { + Ok(candidate) => record.latest = candidate, + Err(failure) => { + record.latest.deployment = if job.initial { + DeploymentState::CreateFailed(failure) + } else { + DeploymentState::UpdateFailed(failure) + } + } + } + true + }) { + Err(LambdaServiceError::FunctionNotFound { .. }) => Ok(false), + result => result, + } + } +} + +fn invalid(message: &str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} +fn conflict(message: &str) -> LambdaServiceError { + LambdaServiceError::ResourceConflict { + message: message.into(), + } +} + +#[cfg(test)] +#[path = "deployment_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/deployment_tests.rs b/crates/rustack-lambda-core/src/deployment_tests.rs new file mode 100644 index 00000000..510716df --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_tests.rs @@ -0,0 +1,689 @@ +use std::{ + collections::{BTreeMap, HashMap}, + sync::{Arc, atomic::AtomicBool}, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use rustack_lambda_model::{ + input::{PublishVersionInput, UpdateFunctionCodeInput, UpdateFunctionConfigurationInput}, + types::ImageConfig, +}; +use tokio::sync::watch; + +use super::*; +use crate::{ + artifact::{ImageMetadata, ResolvedImage}, + config::LambdaConfig, + provider::RustackLambda, + storage::{FunctionStoreSnapshot, PolicyDocument}, +}; + +fn image( + reference: &str, + letter: char, + architecture: Architecture, +) -> (PinnedImage, watch::Receiver<()>, watch::Sender) { + let mut metadata: ImageMetadata = ResolvedImage::fixture(architecture).into(); + metadata.requested_reference = reference.parse().unwrap(); + metadata.image_id = format!("sha256:{}", letter.to_string().repeat(64)) + .parse() + .unwrap(); + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + None, + architecture, + metadata.size_bytes, + &metadata.execution, + ) + .unwrap(); + let (release, released) = watch::channel(()); + let (stop, stopped) = watch::channel(false); + let image = PinnedImage::new( + metadata.try_into().unwrap(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped, + ); + (image, released, stop) +} +fn record() -> FunctionRecord { + FunctionRecord { + resource_id: FunctionResourceId::new(), + name: "worker".into(), + arn: "arn:aws:lambda:us-east-1:000000000000:function:worker".into(), + latest: VersionRecord { + version: "$LATEST".into(), + runtime: None, + handler: None, + role: "arn:aws:iam::000000000000:role/test".into(), + description: "old description".into(), + timeout: 3, + memory_size: 128, + environment: HashMap::from([("SECRET".into(), "private value".into())]), + package_type: "Image".into(), + code_path: None, + image_uri: Some("worker:v1".into()), + image: ImageArtifact::Unresolved, + zip_bytes: None, + deployment: DeploymentState::Ready, + last_modified: "2026-10-25T00:00:00.000+0000".into(), + architectures: vec!["arm64".into()], + ephemeral_storage_size: 512, + code_sha256: String::new(), + code_size: 0, + revision_id: Uuid::new_v4().to_string(), + layers: Vec::new(), + vpc_config: None, + dead_letter_config: None, + tracing_config: None, + image_config: None, + logging_config: None, + snap_start: None, + }, + versions: BTreeMap::new(), + next_version: 1, + aliases: HashMap::new(), + policy: PolicyDocument::default(), + tags: HashMap::new(), + code_signing_config_arn: None, + url_config: None, + reserved_concurrent_executions: None, + event_invoke_configs: HashMap::new(), + created_at: "2026-10-25T00:00:00.000+0000".into(), + } +} +fn store() -> (tempfile::TempDir, FunctionStore) { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + (root, store) +} +fn ready(store: &FunctionStore) -> FunctionRecord { + let job = store.insert_image_deployment(record()).unwrap(); + store + .settle_image_deployment(&job, Ok(image("worker:v1", 'a', Architecture::Arm64).0)) + .unwrap(); + store.get("worker").unwrap().unwrap() +} +fn update(store: &FunctionStore, reference: &str) -> ImageDeployment { + let before = store.get("worker").unwrap().unwrap(); + store + .begin_image_update( + "worker", + before.resource_id, + &before.latest.revision_id, + reference.parse().unwrap(), + Architecture::Arm64, + ) + .unwrap() +} +fn current(store: &FunctionStore) -> VersionRecord { + store.get("worker").unwrap().unwrap().latest +} + +#[test] +fn test_should_reserve_pending_and_settle_with_content_hash_and_unpacked_size() { + let (_root, store) = store(); + let mut before = record(); + before.latest.code_sha256 = "unverified tag hash".into(); + before.latest.code_size = 999; + let old_revision = before.latest.revision_id.clone(); + let resource = before.resource_id; + let job = store.insert_image_deployment(before).unwrap(); + assert_eq!(current(&store).deployment.state(), "Pending"); + assert_eq!(current(&store).code_sha256, ""); + assert_eq!(current(&store).code_size, 0); + assert_eq!(current(&store).deployment.update_status(), "InProgress"); + let (image, _, _) = image("worker:v1", 'a', Architecture::Arm64); + let hash = image + .resolved() + .metadata() + .inspection_hash + .as_str() + .strip_prefix("sha256:") + .unwrap(); + let expected = STANDARD.encode(hex::decode(hash).unwrap()); + assert!(!format!("{job:?}").contains("private value")); + assert!(!format!("{job:?}").contains("worker:v1")); + assert!(store.settle_image_deployment(&job, Ok(image)).unwrap()); + let after = store.get("worker").unwrap().unwrap(); + assert_eq!(after.resource_id, resource); + assert_eq!(after.latest.deployment, DeploymentState::Ready); + assert_ne!(after.latest.revision_id, old_revision); + assert_eq!(after.latest.code_sha256, expected); + assert_eq!(after.latest.code_size, 4096); // Metadata Size is only1024. + assert!(after.latest.image.pinned().unwrap().is_live()); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::SupervisorFailed)) + .unwrap() + ); +} + +#[test] +fn test_should_preserve_previous_revision_while_updating_and_after_failure() { + let (_root, store) = store(); + let before = ready(&store).latest; + let job = update(&store, "worker:v2"); + let pending = current(&store); + assert_eq!(pending.deployment.state(), "Active"); + assert_eq!(pending.deployment.update_status(), "InProgress"); + assert_eq!(pending.revision_id, before.revision_id); + assert_eq!(pending.image_uri, before.image_uri); + assert_eq!(pending.code_sha256, before.code_sha256); + assert_eq!( + pending.image.resolved().unwrap().metadata().image_id, + before.image.resolved().unwrap().metadata().image_id + ); + assert!( + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::InvalidImage)) + .unwrap() + ); + let failed = current(&store); + assert_eq!( + failed.deployment, + DeploymentState::UpdateFailed(ImageDeploymentFailure::InvalidImage) + ); + assert_eq!(failed.revision_id, before.revision_id); + assert_eq!(failed.image_uri, before.image_uri); + assert_eq!(failed.code_sha256, before.code_sha256); + assert_eq!(failed.environment, before.environment); + let retry = update(&store, "worker:v2"); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); + assert!( + store + .settle_image_deployment(&retry, Ok(image("worker:v2", 'b', Architecture::Arm64).0)) + .unwrap() + ); + assert_eq!(current(&store).image_uri.as_deref(), Some("worker:v2")); + assert_ne!(current(&store).code_sha256, before.code_sha256); +} + +#[test] +fn test_should_retry_failed_creation_without_claiming_a_previous_active_revision() { + let (_root, store) = store(); + let initial = store.insert_image_deployment(record()).unwrap(); + assert!( + store + .settle_image_deployment(&initial, Err(ImageDeploymentFailure::ImageNotFound)) + .unwrap() + ); + assert_eq!(current(&store).deployment.state(), "Failed"); + let retry = update(&store, "worker:fixed"); + assert_eq!(current(&store).deployment.state(), "Pending"); + assert_eq!(current(&store).image_uri.as_deref(), Some("worker:fixed")); + assert!( + store + .settle_image_deployment( + &retry, + Ok(image("worker:fixed", 'b', Architecture::Arm64).0) + ) + .unwrap() + ); + assert_eq!(current(&store).deployment, DeploymentState::Ready); +} + +#[test] +fn test_should_reject_stale_resource_revision_and_generation_independently() { + let (_root, store) = store(); + let before = ready(&store); + assert!( + store + .begin_image_update( + "worker", + FunctionResourceId::new(), + &before.latest.revision_id, + "worker:v2".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + assert!( + store + .begin_image_update( + "worker", + before.resource_id, + "stale", + "worker:v2".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + let job = update(&store, "worker:v2"); + assert!( + store + .begin_image_update( + "worker", + before.resource_id, + &before.latest.revision_id, + "worker:v3".parse().unwrap(), + Architecture::Arm64 + ) + .is_err() + ); + store + .update("worker", |record| { + record.latest.revision_id = Uuid::new_v4().to_string(); + }) + .unwrap(); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); + store.remove("worker").unwrap(); + let mut recreated = before; + recreated.resource_id = FunctionResourceId::new(); + recreated.latest.deployment = DeploymentState::Updating(job.generation); + store.insert(recreated).unwrap(); + assert!( + !store + .settle_image_deployment(&job, Ok(image("worker:v2", 'b', Architecture::Arm64).0)) + .unwrap() + ); + store.remove("worker").unwrap(); + assert!( + !store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap() + ); +} + +#[rstest] +#[case("other:v1", Architecture::Arm64, false)] +#[case("worker:v1", Architecture::X86_64, false)] +#[case("worker:v1", Architecture::Arm64, true)] +fn test_should_reject_mismatching_or_revoked_preparation_without_partial_commit( + #[case] reference: &str, + #[case] architecture: Architecture, + #[case] revoke: bool, +) { + let (_root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let pending = current(&store).deployment; + let (image, _, stop) = image(reference, 'a', architecture); + if revoke { + stop.send_replace(true); + } + assert!(store.settle_image_deployment(&job, Ok(image)).is_err()); + assert_eq!(current(&store).deployment, pending); + assert!(current(&store).image.pinned().is_none()); + assert!( + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::InvalidImage)) + .unwrap() + ); +} + +#[test] +fn test_should_drop_stale_result_lease_without_modifying_a_new_generation() { + let (_root, store) = store(); + ready(&store); + let job = update(&store, "worker:v2"); + store + .settle_image_deployment(&job, Err(ImageDeploymentFailure::Cancelled)) + .unwrap(); + let replacement = update(&store, "worker:v3"); + let (image, released, _) = image("worker:v2", 'b', Architecture::Arm64); + assert!(!store.settle_image_deployment(&job, Ok(image)).unwrap()); + assert!(released.has_changed().is_err()); + assert_eq!( + current(&store).deployment, + DeploymentState::Updating(replacement.generation) + ); +} + +#[test] +fn test_should_rollback_every_change_when_a_fallible_store_mutation_fails() { + let (_root, store) = store(); + let before = ready(&store); + let result: Result<(), _> = + store.try_update_if_revision("worker", Some(&before.latest.revision_id), |record| { + record.latest.environment.clear(); + record.latest.description = "must not commit".into(); + record.latest.revision_id = "changed".into(); + Err(invalid("injected validation failure")) + }); + assert!(result.is_err()); + let after = current(&store); + assert_eq!(after.environment, before.latest.environment); + assert_eq!(after.description, before.latest.description); + assert_eq!(after.revision_id, before.latest.revision_id); +} + +#[tokio::test] +async fn test_should_derive_public_status_and_guard_mutations_and_publication() { + let (root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let pending = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(pending.state.as_deref(), Some("Pending")); + assert!(pending.code_sha256.is_none()); + assert_eq!(pending.last_update_status.as_deref(), Some("InProgress")); + assert!( + provider + .publish_version("worker", &PublishVersionInput::default()) + .is_err() + ); + assert!( + provider + .update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + description: Some("no".into()), + ..Default::default() + } + ) + .is_err() + ); + assert!( + provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v2".into()), + ..Default::default() + } + ) + .await + .is_err() + ); + assert!( + provider + .store() + .settle_image_deployment(&job, Err(ImageDeploymentFailure::ImageNotFound)) + .unwrap() + ); + let failed = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(failed.state.as_deref(), Some("Failed")); + assert_eq!( + failed.state_reason.as_deref(), + Some(ImageDeploymentFailure::ImageNotFound.reason()) + ); + assert_eq!(failed.state_reason_code.as_deref(), Some("ImageDeleted")); + assert_eq!(failed.last_update_status.as_deref(), Some("Failed")); + assert!( + provider + .publish_version("worker", &PublishVersionInput::default()) + .is_err() + ); + drop(provider); + drop(root); +} + +#[tokio::test] +async fn test_should_keep_published_leases_and_report_resolved_uri() { + let (_root, store) = store(); + let job = store.insert_image_deployment(record()).unwrap(); + let (image, released, _) = image("worker:v1", 'a', Architecture::Arm64); + store.settle_image_deployment(&job, Ok(image)).unwrap(); + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let published = provider + .publish_version("worker", &PublishVersionInput::default()) + .unwrap(); + assert_eq!(published.version.as_deref(), Some("1")); + let code = provider + .get_function("worker", Some("1")) + .unwrap() + .code + .unwrap(); + assert_eq!(code.image_uri.as_deref(), Some("worker:v1")); + assert!( + code.resolved_image_uri + .unwrap() + .starts_with("local://sha256:") + ); + let update = update(provider.store(), "worker:v2"); + provider + .store() + .settle_image_deployment(&update, Ok(image_fixture("worker:v2", 'b'))) + .unwrap(); + assert!(!released.has_changed().unwrap()); + provider.delete_function("worker", Some("1")).await.unwrap(); + assert!(released.has_changed().is_err()); +} +fn image_fixture(reference: &str, letter: char) -> PinnedImage { + image(reference, letter, Architecture::Arm64).0 +} + +#[tokio::test] +async fn test_should_rollback_invalid_effective_configuration_and_guard_publish_keys() { + let (_root, store) = store(); + let before = ready(&store).latest; + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + assert!( + provider + .update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + description: Some("must not commit".into()), + image_config: Some(ImageConfig { + entry_point: Some(Vec::new()), + command: Some(Vec::new()), + working_directory: None + }), + ..Default::default() + } + ) + .is_err() + ); + assert_eq!(current(provider.store()).description, before.description); + assert_eq!(current(provider.store()).revision_id, before.revision_id); + for input in [ + PublishVersionInput { + revision_id: Some("stale".into()), + ..Default::default() + }, + PublishVersionInput { + code_sha256: Some("wrong".into()), + ..Default::default() + }, + ] { + assert!(provider.publish_version("worker", &input).is_err()); + } + assert!( + provider + .store() + .get("worker") + .unwrap() + .unwrap() + .versions + .is_empty() + ); +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_restore_in_progress_snapshots_as_failed_without_inventing_a_lease( + #[case] updating: bool, +) { + let (_root, store) = store(); + if updating { + ready(&store); + update(&store, "worker:v2"); + } else { + store.insert_image_deployment(record()).unwrap(); + } + let json = serde_json::to_vec(&store.export_snapshot()).unwrap(); + let restore_root = tempfile::tempdir().unwrap(); + let restored = FunctionStore::new(restore_root.path()); + restored + .import_snapshot(serde_json::from_slice(&json).unwrap()) + .await + .unwrap(); + let after = current(&restored); + assert_eq!( + after.deployment, + if updating { + DeploymentState::UpdateFailed(ImageDeploymentFailure::Cancelled) + } else { + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + } + ); + assert!(after.image.pinned().is_none()); + assert_eq!(after.image.resolved().is_some(), updating); +} + +#[tokio::test] +async fn test_should_reject_contradictory_snapshot_states_and_accept_legacy_active_metadata() { + let (_root, store) = store(); + ready(&store); + let baseline = current(&store).revision_id; + let mut snapshot = store.export_snapshot(); + snapshot.functions.first_mut().unwrap().latest.state = "Failed".into(); + assert!(store.import_snapshot(snapshot).await.is_err()); + assert_eq!(current(&store).revision_id, baseline); + let mut legacy = serde_json::to_value(store.export_snapshot()).unwrap(); + legacy["functions"][0]["latest"] + .as_object_mut() + .unwrap() + .remove("deployment"); + let restore_root = tempfile::tempdir().unwrap(); + let restored = FunctionStore::new(restore_root.path()); + restored + .import_snapshot(serde_json::from_value::(legacy).unwrap()) + .await + .unwrap(); + assert_eq!(current(&restored).deployment, DeploymentState::Ready); + assert!(matches!( + current(&restored).image, + ImageArtifact::Restored(_) + )); +} + +#[test] +fn test_should_reject_nil_invalid_and_unknown_deployment_state_values() { + assert!(DeploymentGeneration::try_from(Uuid::nil()).is_err()); + for value in [Uuid::nil().to_string(), "x".repeat(2048), "invalid".into()] { + assert!( + serde_json::from_value::(serde_json::Value::String(value)) + .is_err() + ); + } + assert!(serde_json::from_str::(r#"{"kind":"ready","extra":true}"#).is_err()); + let state = DeploymentState::Creating(DeploymentGeneration(Uuid::new_v4())); + assert_eq!( + serde_json::from_slice::(&serde_json::to_vec(&state).unwrap()).unwrap(), + state + ); +} + +#[rstest] +#[case("hash")] +#[case("zero")] +#[case("oversize")] +#[tokio::test] +async fn test_should_reject_tampered_resolved_snapshot_metadata_atomically( + #[case] corruption: &str, +) { + let (_root, store) = store(); + let before = ready(&store).latest; + let mut snapshot = store.export_snapshot(); + let version = &mut snapshot.functions.first_mut().unwrap().latest; + match corruption { + "hash" => version.code_sha256 = "forged".into(), + "zero" => version.code_size = 0, + "oversize" => version.code_size = crate::artifact::MAX_IMAGE_BYTES + 1, + _ => unreachable!("trusted case"), + } + assert!(store.import_snapshot(snapshot).await.is_err()); + let after = current(&store); + assert_eq!(after.revision_id, before.revision_id); + assert_eq!(after.code_sha256, before.code_sha256); + assert!(after.image.pinned().unwrap().is_live()); +} + +#[test] +fn test_should_reject_a_live_image_size_that_disagrees_with_its_proof() { + let (_root, store) = store(); + let before = ready(&store).latest; + assert!( + store + .update("worker", |record| record.latest.code_size += 1) + .is_err() + ); + assert_eq!(current(&store).code_size, before.code_size); +} + +#[test] +fn test_should_commit_only_one_of_two_concurrent_completion_results() { + let (_root, store) = store(); + ready(&store); + let job = update(&store, "worker:v2"); + let barrier = std::sync::Barrier::new(2); + let wins = std::thread::scope(|threads| { + let handles: Vec<_> = ['b', 'c'] + .into_iter() + .map(|letter| { + let store = &store; + let job = &job; + let barrier = &barrier; + threads.spawn(move || { + let prepared = image("worker:v2", letter, Architecture::Arm64).0; + barrier.wait(); + store.settle_image_deployment(job, Ok(prepared)).unwrap() + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .filter(|won| *won) + .count() + }); + assert_eq!(wins, 1); + assert_eq!(current(&store).deployment, DeploymentState::Ready); +} + +#[tokio::test] +async fn test_should_preserve_failed_readiness_when_restoring_legacy_zip_progress() { + let (_root, store) = store(); + let mut source = record(); + source.latest.package_type = "Zip".into(); + source.latest.runtime = Some("provided.al2023".into()); + source.latest.handler = Some("bootstrap".into()); + source.latest.image_uri = None; + store.insert(source).unwrap(); + let mut snapshot = store.export_snapshot(); + let version = &mut snapshot.functions.first_mut().unwrap().latest; + version.state = "Pending".into(); + version.deployment = None; + store.import_snapshot(snapshot).await.unwrap(); + assert_eq!( + current(&store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); +} + +#[tokio::test] +async fn test_should_reject_image_update_publication_before_mutating_the_previous_revision() { + let (_root, store) = store(); + let before = ready(&store).latest; + let provider = RustackLambda::with_store(store, LambdaConfig::default()); + let outcome = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v2".into()), + publish: Some(true), + ..Default::default() + }, + ) + .await; + assert!(matches!( + outcome, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + let after = current(provider.store()); + assert_eq!(after.revision_id, before.revision_id); + assert_eq!(after.code_sha256, before.code_sha256); + assert_eq!(after.deployment, DeploymentState::Ready); +} diff --git a/crates/rustack-lambda-core/src/executor/auto.rs b/crates/rustack-lambda-core/src/executor/auto.rs index 9c01cfcb..5f8e69ff 100644 --- a/crates/rustack-lambda-core/src/executor/auto.rs +++ b/crates/rustack-lambda-core/src/executor/auto.rs @@ -90,8 +90,9 @@ mod tests { #[test] fn test_should_not_use_squib_for_image_package() { let mut request = InvokeRequest::fixture(); - Arc::make_mut(&mut request.revision).artifact = - DeploymentArtifact::Image(ResolvedImage::fixture(Architecture::Arm64)); + Arc::make_mut(&mut request.revision).artifact = DeploymentArtifact::Image( + crate::artifact::PinnedImage::fixture(ResolvedImage::fixture(Architecture::Arm64)), + ); assert!(!AutoExecutor::should_use_squib(&request)); } } diff --git a/crates/rustack-lambda-core/src/function_config.rs b/crates/rustack-lambda-core/src/function_config.rs index d580bea6..f48e59e7 100644 --- a/crates/rustack-lambda-core/src/function_config.rs +++ b/crates/rustack-lambda-core/src/function_config.rs @@ -143,10 +143,19 @@ pub(crate) fn validate_execution_settings( &version.package_type, version.zip_bytes.is_some(), version.code_path.is_some(), - version.resolved_image.as_ref(), + version.image.resolved(), architecture, version.image_uri.as_deref(), )?; + validate_image_fingerprint( + version.image.resolved(), + &version.code_sha256, + version.code_size, + version + .image + .pinned() + .map(|image| image.unpacked_size().get()), + )?; Ok(architecture) } @@ -173,9 +182,35 @@ pub(crate) fn validate_snapshot_settings( version.resolved_image.as_ref(), architecture, version.image_uri.as_deref(), + )?; + validate_image_fingerprint( + version.resolved_image.as_ref(), + &version.code_sha256, + version.code_size, + None, ) } +fn validate_image_fingerprint( + image: Option<&ResolvedImage>, + hash: &str, + size: u64, + pinned_size: Option, +) -> Result<(), LambdaServiceError> { + if let Some(image) = image { + if hash + != image + .code_sha256() + .map_err(|_| invalid("Invalid image fingerprint"))? + || !(1..=crate::artifact::MAX_IMAGE_BYTES).contains(&size) + || pinned_size.is_some_and(|proven| proven != size) + { + return Err(invalid("Image deployment metadata mismatch")); + } + } + Ok(()) +} + fn validate_layout( package_type: &str, has_zip: bool, diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index 4f51b405..b597811b 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod artifact; pub mod code; pub mod config; +pub mod deployment; #[cfg(unix)] pub mod docker; pub mod error; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 5dd3bdaa..66a27618 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -57,9 +57,10 @@ use rustack_lambda_model::{ #[cfg(test)] use crate::artifact::Architecture; use crate::{ - artifact::{DeploymentArtifact, ImageReference}, + artifact::{DeploymentArtifact, ImageArtifact, ImageReference}, code::{S3CodeFetchError, S3CodeFetcher, UnavailableS3CodeFetcher}, config::LambdaConfig, + deployment::DeploymentState, error::LambdaServiceError, executor::{ AutoExecutor, ExecutionRevision, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, @@ -508,6 +509,7 @@ impl RustackLambda { } } + validate_image_publication(&package_type, input.publish.unwrap_or(false))?; crate::function_config::validate_create_settings(&input)?; let processed = self.process_code(name, "$LATEST", code_source).await?; let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); @@ -537,9 +539,9 @@ impl RustackLambda { package_type: package_type.clone(), code_path, image_uri, - resolved_image: None, + image: ImageArtifact::Unresolved, zip_bytes, - state: "Active".to_owned(), + deployment: DeploymentState::Ready, last_modified: now.clone(), architectures, ephemeral_storage_size, @@ -605,15 +607,16 @@ impl RustackLambda { let version = resolve_version(&record, qualifier)?; let config = self.build_function_configuration(&record, version); + let is_image = version.package_type == "Image"; let code_location = FunctionCodeLocation { - repository_type: Some("S3".to_owned()), - location: Some(format!( + repository_type: Some(if is_image { "ECR" } else { "S3" }.into()), + location: (!is_image).then(|| format!( "https://awslambda-{region}-tasks.s3.{region}.amazonaws.com/snapshots/{account}/{name}", region = self.config.default_region, account = self.config.account_id, )), image_uri: version.image_uri.clone(), - resolved_image_uri: None, + resolved_image_uri: version.image.resolved().map(crate::artifact::ResolvedImage::resolved_uri), }; let tags = if record.tags.is_empty() { @@ -688,6 +691,8 @@ impl RustackLambda { let (name, _) = resolve_function_ref(function_ref)?; let should_publish = input.publish.unwrap_or(false); let before = self.get_record(&name)?; + before.latest.deployment.ensure_mutable()?; + validate_image_publication(&before.latest.package_type, should_publish)?; if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) || (before.latest.package_type == "Image" && code_source.image_uri.is_none()) { @@ -707,34 +712,39 @@ impl RustackLambda { let processed = self.process_code(&name, "$LATEST", code_source).await?; let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); - let config = - self.store - .update_if_revision(&name, Some(&before.latest.revision_id), |record| { - let now = now_iso8601(); - record.latest.code_sha256 = code_sha256; - record.latest.code_size = code_size; - record.latest.zip_bytes = zip_bytes; - record.latest.code_path = code_path; - record.latest.image_uri = image_uri; - record.latest.resolved_image = None; - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - - if let Some(archs) = input.architectures.clone() { - record.latest.architectures = archs; - } - if should_publish { - let number = record.next_version; - record.next_version = number.saturating_add(1); - let mut published = record.latest.clone(); - published.version = number.to_string(); - let configuration = self.build_function_configuration(record, &published); - record.versions.insert(number, published); - configuration - } else { - self.build_function_configuration(record, &record.latest) - } - })?; + let config = self.store.try_update_if_revision( + &name, + Some(&before.latest.revision_id), + |record| { + record.latest.deployment.ensure_mutable()?; + let now = now_iso8601(); + record.latest.code_sha256 = code_sha256; + record.latest.code_size = code_size; + record.latest.zip_bytes = zip_bytes; + record.latest.code_path = code_path; + record.latest.image_uri = image_uri; + record.latest.image = ImageArtifact::Unresolved; + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + + if let Some(archs) = input.architectures.clone() { + record.latest.architectures = archs; + } + record.latest.deployment = DeploymentState::Ready; + let configuration = if should_publish { + let number = record.next_version; + record.next_version = number.saturating_add(1); + let mut published = record.latest.clone(); + published.version = number.to_string(); + let configuration = self.build_function_configuration(record, &published); + record.versions.insert(number, published); + configuration + } else { + self.build_function_configuration(record, &record.latest) + }; + Ok(configuration) + }, + )?; processed.commit(); info!(function_name = %name, "updated Lambda function code"); @@ -749,112 +759,80 @@ impl RustackLambda { ) -> Result { let (name, _) = resolve_function_ref(function_ref)?; - // Validate handler length (Appendix C: max 128 chars). - if let Some(ref handler) = input.handler { - if handler.len() > 128 { - return Err(LambdaServiceError::InvalidParameter { - message: "Handler must be at most 128 characters".to_owned(), - }); - } - } - // Validate description length (Appendix C: max 256 chars). - if let Some(ref desc) = input.description { - if desc.len() > 256 { - return Err(LambdaServiceError::InvalidParameter { - message: "Description must be at most 256 characters".to_owned(), - }); - } - } - // Validate timeout (Appendix C: 1-900 seconds). - if let Some(timeout) = input.timeout { - if !(1..=900).contains(&timeout) { - return Err(LambdaServiceError::InvalidParameter { - message: "Timeout must be between 1 and 900 seconds".to_owned(), - }); - } - } - // Validate memory size (Appendix C: 128-10240 MB). - if let Some(memory) = input.memory_size { - if !(128..=10_240).contains(&memory) { - return Err(LambdaServiceError::InvalidParameter { - message: "Memory size must be between 128 and 10240 MB".to_owned(), - }); - } - } - // Validate ephemeral storage (Appendix C: 512-10240 MB). - if let Some(ref ephemeral) = input.ephemeral_storage { - if !(512..=10_240).contains(&ephemeral.size) { - return Err(LambdaServiceError::InvalidParameter { - message: "Ephemeral storage must be between 512 and 10240 MB".to_owned(), - }); - } - } - // Validate environment variables total size (Appendix C: 4 KB). - if let Some(ref env) = input.environment { - if let Some(ref vars) = env.variables { - let total_size: usize = vars.iter().map(|(k, v)| k.len() + v.len()).sum(); - if total_size > 4096 { - return Err(LambdaServiceError::InvalidParameter { - message: "Environment variables total size exceeds 4 KB limit".to_owned(), - }); - } - } - } + validate_configuration_update(input)?; - let config = self.store.update(&name, |record| { - let now = now_iso8601(); + let before = self.get_record(&name)?; + let config = self.store.try_update_if_revision( + &name, + Some(&before.latest.revision_id), + |record| { + record.latest.deployment.ensure_mutable()?; + let now = now_iso8601(); - if let Some(runtime) = &input.runtime { - record.latest.runtime = Some(runtime.clone()); - } - if let Some(role) = &input.role { - record.latest.role.clone_from(role); - } - if let Some(handler) = &input.handler { - record.latest.handler = Some(handler.clone()); - } - if let Some(description) = &input.description { - record.latest.description.clone_from(description); - } - if let Some(timeout) = input.timeout { - record.latest.timeout = timeout; - } - if let Some(memory_size) = input.memory_size { - record.latest.memory_size = memory_size; - } - if let Some(env) = &input.environment { - record.latest.environment = env.variables.clone().unwrap_or_default(); - } - if let Some(layers) = &input.layers { - record.latest.layers.clone_from(layers); - } - if let Some(ephemeral) = &input.ephemeral_storage { - record.latest.ephemeral_storage_size = ephemeral.size; - } - if let Some(vpc) = &input.vpc_config { - record.latest.vpc_config = Some(vpc.clone()); - } - if let Some(dlc) = &input.dead_letter_config { - record.latest.dead_letter_config = Some(dlc.clone()); - } - if let Some(tc) = &input.tracing_config { - record.latest.tracing_config = Some(tc.clone()); - } - if let Some(ic) = &input.image_config { - record.latest.image_config = Some(ic.clone()); - } - if let Some(lc) = &input.logging_config { - record.latest.logging_config = Some(lc.clone()); - } - if let Some(ss) = &input.snap_start { - record.latest.snap_start = Some(ss.clone()); - } + if let Some(runtime) = &input.runtime { + record.latest.runtime = Some(runtime.clone()); + } + if let Some(role) = &input.role { + record.latest.role.clone_from(role); + } + if let Some(handler) = &input.handler { + record.latest.handler = Some(handler.clone()); + } + if let Some(description) = &input.description { + record.latest.description.clone_from(description); + } + if let Some(timeout) = input.timeout { + record.latest.timeout = timeout; + } + if let Some(memory_size) = input.memory_size { + record.latest.memory_size = memory_size; + } + if let Some(env) = &input.environment { + record.latest.environment = env.variables.clone().unwrap_or_default(); + } + if let Some(layers) = &input.layers { + record.latest.layers.clone_from(layers); + } + if let Some(ephemeral) = &input.ephemeral_storage { + record.latest.ephemeral_storage_size = ephemeral.size; + } + if let Some(vpc) = &input.vpc_config { + record.latest.vpc_config = Some(vpc.clone()); + } + if let Some(dlc) = &input.dead_letter_config { + record.latest.dead_letter_config = Some(dlc.clone()); + } + if let Some(tc) = &input.tracing_config { + record.latest.tracing_config = Some(tc.clone()); + } + if let Some(ic) = &input.image_config { + record.latest.image_config = Some(ic.clone()); + } + if let Some(lc) = &input.logging_config { + record.latest.logging_config = Some(lc.clone()); + } + if let Some(ss) = &input.snap_start { + record.latest.snap_start = Some(ss.clone()); + } - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - self.build_function_configuration(record, &record.latest) - })?; + if matches!(record.latest.deployment, DeploymentState::UpdateFailed(_)) { + record.latest.deployment = DeploymentState::Ready; + } + if let Some(image) = record.latest.image.resolved() { + image + .metadata() + .execution + .plan(record.latest.image_config.as_ref()) + .map_err(|_| { + invalid_parameter("Invalid effective image entry configuration") + })?; + } + Ok(self.build_function_configuration(record, &record.latest)) + }, + )?; info!(function_name = %name, "updated Lambda function configuration"); Ok(config) @@ -1086,7 +1064,7 @@ impl RustackLambda { payload: &[u8], ) -> Result { let architecture = crate::function_config::validate_execution_settings(version)?; - if version.state != "Active" { + if version.deployment.state() != "Active" { return Err(LambdaServiceError::ResourceNotReady { message: "Function revision is not Active".into(), }); @@ -1108,7 +1086,7 @@ impl RustackLambda { DeploymentArtifact::Zip(artifact) } "Image" => { - let image = version.resolved_image.as_ref().ok_or_else(|| { + let image = version.image.resolved().ok_or_else(|| { LambdaServiceError::ResourceNotReady { message: "Image deployment has not been resolved".into(), } @@ -1123,7 +1101,21 @@ impl RustackLambda { "Image metadata does not match the captured revision", )); } - DeploymentArtifact::Image(image.clone()) + image + .metadata() + .execution + .plan(version.image_config.as_ref()) + .map_err(|_| { + invalid_parameter("Invalid effective image entry configuration") + })?; + let lease = version + .image + .pinned() + .filter(|image| image.is_live()) + .ok_or_else(|| LambdaServiceError::ResourceNotReady { + message: "Image availability has not been acquired".into(), + })?; + DeploymentArtifact::Image(lease.clone()) } _ => return Err(invalid_parameter("Unsupported deployment package type")), }; @@ -1173,22 +1165,51 @@ impl RustackLambda { ) -> Result { let (name, _) = resolve_function_ref(function_ref)?; - let config = self.store.update(&name, |record| { - let version_num = record.next_version; - record.next_version += 1; + let config = + self.store + .try_update_if_revision(&name, input.revision_id.as_deref(), |record| { + if record.latest.deployment != DeploymentState::Ready { + return Err(LambdaServiceError::ResourceConflict { + message: "Function deployment must be successful before publication" + .into(), + }); + } + if record.latest.package_type == "Image" + && record + .latest + .image + .pinned() + .is_none_or(|image| !image.is_live()) + { + return Err(LambdaServiceError::ResourceNotReady { + message: "Image availability must be acquired before publication" + .into(), + }); + } + if input + .code_sha256 + .as_ref() + .is_some_and(|hash| hash != &record.latest.code_sha256) + { + return Err(invalid_parameter( + "CodeSha256 does not match the latest revision", + )); + } + let version_num = record.next_version; + record.next_version += 1; - let mut published = record.latest.clone(); - published.version = version_num.to_string(); - if let Some(desc) = &input.description { - published.description.clone_from(desc); - } - published.revision_id = uuid::Uuid::new_v4().to_string(); - published.last_modified = now_iso8601(); + let mut published = record.latest.clone(); + published.version = version_num.to_string(); + if let Some(desc) = &input.description { + published.description.clone_from(desc); + } + published.revision_id = uuid::Uuid::new_v4().to_string(); + published.last_modified = now_iso8601(); - let config = self.build_function_configuration(record, &published); - record.versions.insert(version_num, published); - config - })?; + let config = self.build_function_configuration(record, &published); + record.versions.insert(version_num, published); + Ok(config) + })?; info!(function_name = %name, "published Lambda function version"); Ok(config) @@ -2954,6 +2975,7 @@ impl RustackLambda { optimization_status: Some("Off".to_owned()), }); + let state_failure = version.deployment.creation_failure(); FunctionConfiguration { function_name: Some(record.name.clone()), function_arn: Some(arn), @@ -2970,7 +2992,7 @@ impl RustackLambda { timeout: Some(version.timeout), memory_size: Some(version.memory_size), last_modified: Some(version.last_modified.clone()), - code_sha256: Some(version.code_sha256.clone()), + code_sha256: (!version.code_sha256.is_empty()).then(|| version.code_sha256.clone()), version: Some(version.version.clone()), environment: env_response, vpc_config: vpc_response, @@ -2978,9 +3000,9 @@ impl RustackLambda { tracing_config: tracing_response, revision_id: Some(version.revision_id.clone()), layers, - state: Some(version.state.clone()), - state_reason: None, - state_reason_code: None, + state: Some(version.deployment.state().into()), + state_reason: state_failure.map(|failure| failure.reason().into()), + state_reason_code: state_failure.map(|failure| failure.code().into()), package_type: Some(version.package_type.clone()), architectures: Some(version.architectures.clone()), ephemeral_storage: Some(EphemeralStorage { @@ -2989,9 +3011,15 @@ impl RustackLambda { logging_config: version.logging_config.clone(), snap_start: snap_start_response, image_config_response, - last_update_status: Some("Successful".to_owned()), - last_update_status_reason: None, - last_update_status_reason_code: None, + last_update_status: Some(version.deployment.update_status().into()), + last_update_status_reason: version + .deployment + .failure() + .map(|failure| failure.reason().into()), + last_update_status_reason_code: version + .deployment + .failure() + .map(|failure| failure.code().into()), } } @@ -3007,8 +3035,75 @@ impl RustackLambda { } } -/// Get current time in ISO 8601 format matching AWS Lambda conventions. -fn now_iso8601() -> String { +/// Reject publication before asynchronous image preparation can begin. +fn validate_image_publication(package_type: &str, publish: bool) -> Result<(), LambdaServiceError> { + if package_type == "Image" && publish { + return Err(invalid_parameter( + "Image deployment requires a separate PublishVersion after successful preparation", + )); + } + Ok(()) +} + +fn validate_configuration_update( + input: &UpdateFunctionConfigurationInput, +) -> Result<(), LambdaServiceError> { + // Validate handler length (Appendix C: max 128 chars). + if let Some(ref handler) = input.handler { + if handler.len() > 128 { + return Err(LambdaServiceError::InvalidParameter { + message: "Handler must be at most 128 characters".to_owned(), + }); + } + } + // Validate description length (Appendix C: max 256 chars). + if let Some(ref desc) = input.description { + if desc.len() > 256 { + return Err(LambdaServiceError::InvalidParameter { + message: "Description must be at most 256 characters".to_owned(), + }); + } + } + // Validate timeout (Appendix C: 1-900 seconds). + if let Some(timeout) = input.timeout { + if !(1..=900).contains(&timeout) { + return Err(LambdaServiceError::InvalidParameter { + message: "Timeout must be between 1 and 900 seconds".to_owned(), + }); + } + } + // Validate memory size (Appendix C: 128-10240 MB). + if let Some(memory) = input.memory_size { + if !(128..=10_240).contains(&memory) { + return Err(LambdaServiceError::InvalidParameter { + message: "Memory size must be between 128 and 10240 MB".to_owned(), + }); + } + } + // Validate ephemeral storage (Appendix C: 512-10240 MB). + if let Some(ref ephemeral) = input.ephemeral_storage { + if !(512..=10_240).contains(&ephemeral.size) { + return Err(LambdaServiceError::InvalidParameter { + message: "Ephemeral storage must be between 512 and 10240 MB".to_owned(), + }); + } + } + // Validate environment variables total size (Appendix C: 4 KB). + if let Some(ref env) = input.environment { + if let Some(ref vars) = env.variables { + let total_size: usize = vars.iter().map(|(k, v)| k.len() + v.len()).sum(); + if total_size > 4096 { + return Err(LambdaServiceError::InvalidParameter { + message: "Environment variables total size exceeds 4 KB limit".to_owned(), + }); + } + } + } + Ok(()) +} + +/// Get current time as an ISO 8601 string in AWS Lambda format. +pub(crate) fn now_iso8601() -> String { chrono::Utc::now() .format("%Y-%m-%dT%H:%M:%S%.3f+0000") .to_string() @@ -3435,7 +3530,11 @@ mod tests { version.zip_bytes = None; let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); version.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); - version.resolved_image = Some(resolved); + version.code_sha256 = resolved + .code_sha256() + .map_err(|_| invalid_parameter("Invalid unit image hash"))?; + version.code_size = resolved.metadata().size_bytes; + version.image = ImageArtifact::Pinned(crate::artifact::PinnedImage::fixture(resolved)); version.image_config = Some(rustack_lambda_model::types::ImageConfig { entry_point: Some(vec!["/custom/entrypoint".into()]), command: Some(vec![]), @@ -3467,6 +3566,98 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_should_reject_image_create_publication_before_inserting_a_resource() { + let provider = test_provider(); + let mut input = sample_create_input("image-publish"); + input.package_type = Some("Image".into()); + input.runtime = None; + input.handler = None; + input.code = rustack_lambda_model::types::FunctionCode { + image_uri: Some("worker:v1".into()), + ..Default::default() + }; + input.publish = Some(true); + assert!(matches!( + provider.create_function(input).await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + assert!(!provider.store.contains("image-publish").unwrap()); + } + + #[tokio::test] + async fn test_should_capture_and_retain_only_live_image_availability() + -> Result<(), LambdaServiceError> { + let provider = test_provider(); + provider + .create_function(sample_create_input("image-lease")) + .await?; + let record = provider.get_record("image-lease")?; + let mut version = record.latest.clone(); + version.package_type = "Image".into(); + version.runtime = None; + version.handler = None; + version.code_path = None; + version.zip_bytes = None; + let resolved = crate::artifact::ResolvedImage::fixture(Architecture::X86_64); + version.image_uri = Some(resolved.metadata().requested_reference.as_str().into()); + version.code_sha256 = resolved + .code_sha256() + .map_err(|_| invalid_parameter("Unit image hash"))?; + version.code_size = resolved.metadata().size_bytes; + version.image = ImageArtifact::Restored(resolved.clone()); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + let (release, released) = tokio::sync::watch::channel(()); + let (stop, stopping) = tokio::sync::watch::channel(false); + version.image = ImageArtifact::Pinned(crate::artifact::PinnedImage::new( + resolved, + version + .code_size + .try_into() + .map_err(|_| invalid_parameter("Unit image size"))?, + release, + Arc::new(std::sync::atomic::AtomicBool::new(true)), + stopping, + )); + let request = provider + .build_invoke_request(&record, &version, None, b"{}") + .await?; + let hash = version.code_sha256.clone(); + version.code_sha256 = "tampered".into(); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::InvalidParameter { .. }) + )); + version.code_sha256 = hash; + stop.send_replace(true); + assert!(matches!( + provider + .build_invoke_request(&record, &version, None, b"{}") + .await, + Err(LambdaServiceError::ResourceNotReady { .. }) + )); + drop(version); + assert!( + !released + .has_changed() + .map_err(|_| invalid_parameter("Lease lost before request"))? + ); + assert!(matches!( + request.revision.artifact, + DeploymentArtifact::Image(_) + )); + drop(request); + assert!(released.has_changed().is_err()); + Ok(()) + } + #[tokio::test] async fn test_should_reject_unresolved_or_inconsistent_artifacts_before_work_admission() -> Result<(), LambdaServiceError> { @@ -3497,7 +3688,7 @@ mod tests { .await, Err(LambdaServiceError::ResourceNotReady { .. }) )); - image.resolved_image = Some(resolved); + image.image = ImageArtifact::Restored(resolved); image.architectures = vec!["arm64".into()]; assert!(matches!( provider diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index 3c7c928d..11ca8689 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -304,9 +304,9 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, - resolved_image: None, + image: crate::artifact::ImageArtifact::Unresolved, zip_bytes: None, - state: "Active".to_owned(), + deployment: crate::deployment::DeploymentState::Ready, last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), architectures: vec!["x86_64".to_owned()], ephemeral_storage_size: 512, diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index 79609a26..95c36a80 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -22,7 +22,8 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::{ - artifact::{ResolvedImage, ValidatedArtifact}, + artifact::{ImageArtifact, ResolvedImage, ValidatedArtifact}, + deployment::DeploymentState, error::LambdaServiceError, identity::FunctionResourceId, }; @@ -156,12 +157,12 @@ pub struct VersionRecord { pub code_path: Option, /// Container image URI. pub image_uri: Option, - /// Immutable inspected image metadata, absent until deployment resolution. - pub resolved_image: Option, + /// Unresolved, restored metadata, or shared live image availability. + pub image: ImageArtifact, /// Raw zip bytes. pub zip_bytes: Option, - /// Function state (`Active`, `Pending`, etc.). - pub state: String, + /// Authoritative readiness and update progress. + pub deployment: DeploymentState, /// ISO 8601 last-modified timestamp. pub last_modified: String, /// Supported architectures. @@ -681,8 +682,11 @@ pub struct VersionRecordSnapshot { pub resolved_image: Option, /// Raw zip bytes encoded as standard base64. pub zip_bytes_base64: Option, - /// Function state. + /// Legacy function state, derived from deployment on new exports. pub state: String, + /// Authoritative deployment state in current snapshots. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deployment: Option, /// ISO 8601 last-modified timestamp. pub last_modified: String, /// Supported architectures. @@ -783,12 +787,13 @@ impl VersionRecordSnapshot { environment: record.environment, package_type: record.package_type, image_uri: record.image_uri, - resolved_image: record.resolved_image, + resolved_image: record.image.resolved().cloned(), zip_bytes_base64: record .zip_bytes .as_ref() .map(|bytes| STANDARD.encode(bytes.as_ref())), - state: record.state, + state: record.deployment.state().into(), + deployment: Some(record.deployment), last_modified: record.last_modified, architectures: record.architectures, ephemeral_storage_size: record.ephemeral_storage_size, @@ -848,9 +853,11 @@ impl VersionRecordSnapshot { package_type: self.package_type, code_path, image_uri: self.image_uri, - resolved_image: self.resolved_image, + image: self + .resolved_image + .map_or(ImageArtifact::Unresolved, ImageArtifact::Restored), zip_bytes, - state: self.state, + deployment: DeploymentState::restore(&self.state, self.deployment)?, last_modified: self.last_modified, architectures: self.architectures, ephemeral_storage_size: self.ephemeral_storage_size, @@ -895,7 +902,11 @@ impl FunctionStore { return Err(invalid("Invalid function version or concurrency")); } for (number, version) in &record.versions { - if *number == 0 || version.version != number.to_string() { + if *number == 0 + || version.version != number.to_string() + || version.deployment != DeploymentState::Ready + || (version.package_type == "Image" && version.image.resolved().is_none()) + { return Err(invalid("Published version number mismatch")); } } @@ -1011,6 +1022,32 @@ impl FunctionStore { ) -> Result where F: FnOnce(&mut FunctionRecord) -> R, + { + self.try_update_if_revision(name, expected, |record| Ok(f(record))) + } + + /// Apply a fallible mutation to a private candidate and commit only on success. + /// + /// # Errors + /// Rejects stale revisions, invalid records, missing functions or closure failures; + /// a closure error leaves all live metadata unchanged. + /// + /// ```no_run + /// # fn update(store: &rustack_lambda_core::storage::FunctionStore) -> Result<(), rustack_lambda_core::error::LambdaServiceError> { + /// store.try_update_if_revision("worker", None, |record| { + /// record.latest.description = "updated".into(); + /// Ok(()) + /// })?; + /// # Ok(()) } + /// ``` + pub fn try_update_if_revision( + &self, + name: &str, + expected: Option<&str>, + f: F, + ) -> Result + where + F: FnOnce(&mut FunctionRecord) -> Result, { crate::resolver::FunctionName::parse(name)?; match self.functions.get_mut(name) { @@ -1021,7 +1058,7 @@ impl FunctionStore { }); } let mut candidate = entry.value().clone(); - let result = f(&mut candidate); + let result = f(&mut candidate)?; if candidate.name != name || candidate.resource_id != entry.resource_id { return Err(LambdaServiceError::InvalidParameter { message: "Function identity is immutable".into(), @@ -1875,9 +1912,9 @@ mod tests { package_type: "Zip".to_owned(), code_path: None, image_uri: None, - resolved_image: None, + image: ImageArtifact::Unresolved, zip_bytes: None, - state: "Active".to_owned(), + deployment: DeploymentState::Ready, last_modified: "2024-01-01T00:00:00.000+0000".to_owned(), architectures: vec!["x86_64".to_owned()], ephemeral_storage_size: 512, diff --git a/specs/index.md b/specs/index.md index 2c818173..9bdd8589 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Production Docker executor/provider integration and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement and leased invocation capture are implemented; retained provider deployment ownership/API wiring, the Docker executor and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index d2a5179f..9eed393d 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -36,9 +36,9 @@ Cost: two containers per warm environment. Each relay has a separate 64 MiB memo ### 2.1 Immutable artifacts -Replace mutually inconsistent execution-request Option fields with `DeploymentArtifact::Zip(ValidatedArtifact)` or `Image(ResolvedImage)`, preserving public wire schemas. ExecutionRevision captures function resource UUID, resolved version, artifact/digest, architecture, ImageConfig, environment fingerprint, memory, timeout, ephemeral storage and backend policy. Use typed-builder for multi-field construction. +Replace mutually inconsistent execution-request Option fields with `DeploymentArtifact::Zip(ValidatedArtifact)` or `Image(PinnedImage)`, preserving public wire schemas. ExecutionRevision captures function resource UUID, resolved version, artifact/digest, architecture, ImageConfig, environment fingerprint, memory, timeout, ephemeral storage and backend policy. Use typed-builder for multi-field construction. -Keep the caller's requested function ARN/reference separately from the resolved numeric version. Runtime Invoked-Function-Arn preserves the invoked qualifier, while ExecutedVersion identifies the actual version. Environment reuse keys use resolved identity rather than the mutable alias string. Image CodeSha256 derives from resolved content metadata, not the requested tag text; CodeSize uses inspected image metadata with the local representation documented, and never claims to be a ZIP byte count. +Keep the caller's requested function ARN/reference separately from the resolved numeric version. Runtime Invoked-Function-Arn preserves the invoked qualifier, while ExecutedVersion identifies the actual version. Environment reuse keys use resolved identity rather than the mutable alias string. Image CodeSha256 is the base64 canonical inspected execution fingerprint, independent of the requested tag text; CodeSize uses proven unpacked storage bytes (§2.9), and never claims to be a ZIP byte count. ResolvedImage records requested reference, registry manifest digest when available, daemon content image ID, platform and inspection metadata hash. Invoke by content ID/digest, never by a freshly resolved tag. GetFunction.Code.ImageUri returns the requested reference; ResolvedImageUri returns repo@digest for registry images. A local unpushed image uses the explicitly local extension `local://sha256:`; never invent an ECR manifest digest. Snapshots preserve this distinction; restoring such an image on another daemon requires importing the same content ID. @@ -52,7 +52,7 @@ Create(Image) rejects Runtime/Handler, defaults omitted Architectures to [x86_64 Validate function MemorySize in 128..10,240 MiB, Timeout in 1..900 seconds, and EphemeralStorage in 512..10,240 MiB before allocation. User environment has at most 128 pairs; keys are 2..64 ASCII bytes matching `[A-Za-z][A-Za-z0-9_]+`, individual values are≤4,096 bytes and the total remains≤4 KiB. The key/count limits are local restrictions. Function name/qualifier/artifact validation remains governed by Lambda hardening. Parse error payloads only within their 32 KiB allowance; stackTrace has≤100 entries, each≤1,024 bytes, also constrained by that total. -UpdateFunctionCode keeps the previous revision usable while LastUpdateStatus=InProgress. Success atomically replaces latest; failure preserves the old digest/config and exposes a failed-update reason. RevisionId compare-and-swap prevents concurrent overwrite. Initial release rejects Image Create/Update with Publish=true using InvalidParameterValueException: asynchronous resolution cannot safely return a published version within the control deadline. This is a documented local limitation. A separate PublishVersion succeeds after Active/Succeeded and captures the resolved image and configuration. +UpdateFunctionCode keeps the previous revision usable while LastUpdateStatus=InProgress. Success atomically replaces latest; failure preserves the old digest/config and exposes a failed-update reason. RevisionId compare-and-swap prevents concurrent overwrite. Initial release rejects Image Create/Update with Publish=true using InvalidParameterValueException: asynchronous resolution cannot safely return a published version within the control deadline. This is a documented local limitation. A separate PublishVersion succeeds after Active/Successful and captures the resolved image and configuration. Tag movement never changes deployed code. Published versions are immutable; aliases move references. Update/delete/recreate cannot reuse an old environment. In-flight calls keep their captured revision. Reject package-type changes. DryRun validates input, function/qualifier and permission shape without starting a container or executing code. @@ -89,7 +89,7 @@ Provider / restore Store + staging worker Artifact ownership │ 5. store/request release │ child exit → last lease → remove │ ``` -An Image variant carries ResolvedImage, never a mutable tag alone. Its validated metadata contains the requested ImageReference, daemon SHA-256 content ID, optional repository/manifest reference, Linux architecture, inspection SHA-256 and inspected size. Digests use exactly 64 lowercase hexadecimal digits; inspected image size is bounded to 10 GiB locally. ImageReference accepts bounded Docker repository components, tags, SHA-256 digest suffixes and explicit DNS/IPv4/bracketed-IPv6 registry ports; URLs, userinfo, controls, invalid separators and empty components reject. Repository names are≤255 bytes and the whole reference≤1,024 bytes. Registry DNS labels are≤63 bytes; explicit ports use at most five decimal digits in1..65,535, an advertised local restriction. Bracketed IPv6 uses hexadecimal/colon syntax, excluding zone identifiers and dotted mapped forms. Follow the [distribution reference grammar](https://github.com/distribution/reference/blob/main/regexp.go), implemented with the pinned winnow parser rather than shell parsing. References are parsed again on deserialization. Missing resolution metadata cannot become an executable image; an existing metadata-only image record fails invocation explicitly until its deployment is resolved. +An Image variant carries a live PinnedImage with immutable ResolvedImage metadata, never a mutable tag alone. Its validated metadata contains the requested ImageReference, daemon SHA-256 content ID, optional repository/manifest reference, Linux architecture, inspection SHA-256 and inspected size. Digests use exactly 64 lowercase hexadecimal digits; inspected image size is bounded to 10 GiB locally. ImageReference accepts bounded Docker repository components, tags, SHA-256 digest suffixes and explicit DNS/IPv4/bracketed-IPv6 registry ports; URLs, userinfo, controls, invalid separators and empty components reject. Repository names are≤255 bytes and the whole reference≤1,024 bytes. Registry DNS labels are≤63 bytes; explicit ports use at most five decimal digits in1..65,535, an advertised local restriction. Bracketed IPv6 uses hexadecimal/colon syntax, excluding zone identifiers and dotted mapped forms. Follow the [distribution reference grammar](https://github.com/distribution/reference/blob/main/regexp.go), implemented with the pinned winnow parser rather than shell parsing. References are parsed again on deserialization. Missing resolution metadata cannot become an executable image; an existing metadata-only image record fails invocation explicitly until its deployment is resolved. Artifact replacement does not change public AWS wire shapes. Native and Squib accept only Zip and consume its lease; Docker accepts Image in M0 and adds Zip in M4. The resolved architecture is a closed x86_64/arm64 enum, not an unchecked list. Capture validates numeric settings and full image overrides before work admission. No per-invoke image inspection or pulling is introduced by this contract. @@ -129,7 +129,7 @@ ImageResolver::new(DockerPort) exposes resolve_local(reference, architecture, Im First inspect the requested reference once and capture its canonical, addressable daemon Id. On negotiated API≥1.49, inspect that captured immutable Id with a percent-encoded JSON OCI platform containing exactly os=linux and architecture=amd64/arm64, avoiding a second lookup of the mutable tag. Retain the addressable image_id and optional selected_manifest digest separately: a selected child Id returned by containerd inspection is not necessarily independently addressable for create. Every environment create supplies the captured image_id plus the explicit linux/amd64 or linux/arm64 platform parameter; it never substitutes an unqualified tag or assumes the child digest is a local image object. Older APIs retain single-platform inspection only with verified classic-store evidence. Validate the resulting OS/architecture, positive size≤10GiB and canonical SHA-256 Id. If a Descriptor is present, accept only OCI image-manifest or Docker schema2 image-manifest media types, canonical digest and a compatible optional platform. Reject index/manifest-list/attestation/unknown descriptors. A platform-selected containerd Id may be a child manifest digest rather than the legacy config digest. Validate that returned Id, but use the captured addressable parent Id for create and retain the selected descriptor digest for identity/hash checks. Never create from a parent index without the declared platform. Reject explicit nonempty Manifests in the descriptor-free fallback. Require matching classic GraphDriver and /info driver names from overlay2/aufs/btrfs/devicemapper/vfs/zfs/fuse-overlayfs, well-formed bounded DriverStatus rows and no io.containerd.snapshotter.v1 driver-type marker. Normalize missing/null DriverStatus to empty after matching classic evidence, since classic VFS legitimately reports no status rows. A missing Descriptor or nonempty GraphDriver alone is insufficient. Ambiguous/downlevel containerd resolution fails explicitly rather than choosing the daemon default platform. -RepoDigests has≤128 validated digest references. Repository identity comparison applies Docker familiar-name normalization: absent registry defaults to docker.io, index.docker.io aliases docker.io, and a one-component Docker Hub repository gains library/. Registry host comparison is ASCII case-insensitive; repository paths remain case-sensitive. The original request spelling determines the short-local URI extension before normalization. A requested digest must match the selected descriptor digest or an advertised parent RepoDigest for the same repository; otherwise fail inspection. For explicit registry-domain references with matching repository evidence, use that repository plus the selected descriptor digest as registry_manifest; a legacy descriptor-free image may use a matching manifest RepoDigest, choosing the requested digest or otherwise a deterministic lexical digest. Do not use a digest from an unrelated alias/repository. A short local reference always uses registry_manifest=None and local://Id, even if BuildKit has populated RepoDigests. The returned registry URI reflects daemon metadata, not a network publication/existence check. The captured addressable Id must remain available through the deployment/version/artifact lifetime. The deployment actor still needs an owned availability lease/pin policy; tag movement must never cause it to execute a newer image. Missing externally removed content fails explicitly. CodeSize/size_bytes uses the selected daemon-reported Size, whose containerd representation can be compressed content rather than unpacked size. The deployment actor must separately prove the10GiB unpacked-image quota before Active; this remains a task1.3 gate and is not established by the read-only local resolver. +RepoDigests has≤128 validated digest references. Repository identity comparison applies Docker familiar-name normalization: absent registry defaults to docker.io, index.docker.io aliases docker.io, and a one-component Docker Hub repository gains library/. Registry host comparison is ASCII case-insensitive; repository paths remain case-sensitive. The original request spelling determines the short-local URI extension before normalization. A requested digest must match the selected descriptor digest or an advertised parent RepoDigest for the same repository; otherwise fail inspection. For explicit registry-domain references with matching repository evidence, use that repository plus the selected descriptor digest as registry_manifest; a legacy descriptor-free image may use a matching manifest RepoDigest, choosing the requested digest or otherwise a deterministic lexical digest. Do not use a digest from an unrelated alias/repository. A short local reference always uses registry_manifest=None and local://Id, even if BuildKit has populated RepoDigests. The returned registry URI reflects daemon metadata, not a network publication/existence check. The captured addressable Id must remain available through the deployment/version/artifact lifetime. The owned availability lease/pin policy in §2.9 must be retained by the deployment actor; tag movement must never cause it to execute a newer image. Missing externally removed content fails explicitly. ImageMetadata.size_bytes retains the selected daemon-reported Size, whose containerd representation can be compressed content rather than unpacked size. API CodeSize uses the separate proven unpacked size from §2.9. The deployment actor must obtain this proof before Active; the read-only local resolver alone does not establish the10GiB unpacked-image quota. Validate inherited ENTRYPOINT/CMD with the same argv limits as function ImageConfig, preserving omitted versus explicitly empty overrides. Docker's single-empty-entrypoint reset means no entrypoint. Compose the effective entrypoint+command, require its first executable argument to be nonempty, and reject NULs/bounds before publication; do not require a shell or bootstrap name. An absent/empty inherited working directory means Docker's root default; an explicit override still must be absolute. Inherited image Env is≤128 entries,≤4KiB per entry and≤64KiB total, with no NUL; image Env is distinct from the function's stricter4KiB environment budget. Validate a bounded ASCII environment key of1..256 bytes before `=` (or a key-only deletion marker); preserve ordered values for Docker semantics. Image USER is bounded≤256 printable ASCII bytes. Reject image-declared VOLUMEs in this local profile: automatic anonymous writable mounts would bypass rootfs/tmpfs policy. The eventual container builder must also disable image healthchecks and apply the specified nonroot-user policy. @@ -188,6 +188,38 @@ Owner shutdown is a separate priority signal. The runtime first joins deployment └──────────────────────────────────────────────────────────────────────┘ ``` +### 2.10 Deployment state and atomic settlement + +One typed DeploymentState owns both State and LastUpdateStatus; do not independently mutate string status fields. Ready maps to Active/Successful, Creating(generation) to Pending/InProgress, CreateFailed(reason) to Failed/Failed, Updating(generation) to Active/InProgress and UpdateFailed(reason) to Active/Failed. Reasons are a closed, bounded, static failure vocabulary, never Docker response bodies, image argv, credentials or arbitrary error strings. New generations are nonnil random UUIDs. PublishVersion requires Ready; code/configuration mutations reject Creating/Updating under the same store lock. A failed update can be retried or repaired by a valid configuration mutation without losing the previously usable artifact. Retrying an initially failed creation remains Pending until its first usable image is prepared. + +The store returns a closed ImageDeployment candidate after reserving initial insertion or update. Its settlement key binds function resource UUID, original latest revision and generation. Unprepared candidates clear unverified code hashes and size; initial Pending/Failed responses omit CodeSha256 and report CodeSize=0 until content is proven. Updating responses retain the previous verified metadata. Updates retain the complete previous latest artifact/configuration while the candidate owns new reference/architecture/configuration. Completion takes one store lock, checks all three key fields plus the pending state and validates the candidate before replacement. A stale or duplicate completion returns false and drops its image lease; it cannot mutate a deleted/recreated resource. Store/import/capture validate every resolved image CodeSha256 against the same base64 fingerprint, require CodeSize in1..10GiB and, for live pins, require exact equality to proven unpacked size. Restored size is re-proved during P5 availability reacquisition. Successful settlement requires a live PinnedImage with the candidate's exact requested reference and architecture and a valid effective entry plan. It sets the content-derived CodeSha256 from the canonical inspected execution fingerprint, CodeSize from the proven unpacked quota, a fresh revision ID and Ready. Initial failure leaves CreateFailed; update failure changes only deployment status and preserves old code/configuration/revision. A later retained deployment supervisor must reserve work before these store mutations and settle cancellation/panic through this same key; the store contract alone does not establish accepted-job ownership. + +VersionRecord holds an ImageArtifact enum: Unresolved, Restored immutable metadata, or Pinned live artifact. Metadata cannot masquerade as a lease. Invocation captures only a live PinnedImage into DeploymentArtifact::Image, revalidates its effective ImageConfig plan and keeps the lease through backend cleanup. Numeric publication clones the same lease; deleting latest or moving a tag cannot invalidate it. Restored metadata is inspectable but cannot execute until P5 reacquires exact captured content, without resolving the original mutable tag. + +Snapshots retain the legacy state field and add an optional typed deployment field. New exports derive both from the same enum. Import rejects contradictory fields and invalid/nil generations; legacy Active maps to Ready and legacy Failed/Pending to a closed failure state. A saved in-progress creation restores as CreateFailed(Cancelled), and an in-progress update restores as UpdateFailed(Cancelled) with its old artifact. Live leases are never serialized or invented on restore. Published versions must be Ready. The P5 quiescence owner must prevent exporting accepted in-progress work; these conversions also make direct/legacy import fail closed. + +Required consumer sequence (the retained deployment worker/API wiring remains pending): + +```text +Provider + retained worker FunctionStore ImageLeaseManager + │ reserve bounded work slot │ │ + │ insert / begin update ────────────▶ │ validate; allocate generation │ + │ ◀── closed candidate + key ──────── │ commit Pending / InProgress │ + │ │ │ + │ prepare captured reference ────────┼─────────────────────────────▶ │ + │ │ ◀── Invoke captures old pin │ + │ ◀── live pin / closed failure ─────┼────────────────────────────── │ + │ │ │ + │ settle(candidate, result) ────────▶ │ resource + revision + │ + │ │ generation checked together │ + │ │ │ + │ ◀── true: atomically committed ──── │ Ready or preserved-old Failed│ + │ ◀── false: stale/deleted result ─── │ no mutation; release new pin │ + │ join worker before freeing slot │ │ +``` + +Required tests prove initial Pending/success/failure, preservation of old code while updating and after failure, concurrent mutation rejection, stale/duplicate settlement, delete/recreate isolation, live-lease capture/revocation, numeric-version ownership and snapshot migration/contradiction checks. Docker remains unavailable in capability reporting until the retained deployment owner, executor/pool and startup probes are wired and qualified. + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index b17acf4b..586bb588 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -121,7 +121,8 @@ Snapshot coverage: | Name tombstones | Yes | Retain remaining90-day reservations; reject overflow rather than shorten retention | | RUNNING records in imported/unexpected archives | No resumption | Convert to ABORTED/Rustack.ExecutionInterrupted before serving; never replay uncertain side effects | | Callback tokens/live handles/pagination handles | No | Fresh runtime epoch invalidates old handles; no plaintext token archive | -| Lambda image content references/config | Yes | Validate same digest/content availability before image execution; no tag re-resolution | +| Lambda image content references/config | Yes | Validate same digest/content availability before image execution; no tag re-resolution; serialized metadata never grants a live PinnedImage | +| Lambda deployment readiness/update state | Yes | Apply [runtime §2.10](./rustack-workflows-20-lambda-runtime-design.md#210-deployment-state-and-atomic-settlement): reject contradictory legacy/typed status; abandoned initial/update work becomes failure, retaining prior update code | | Docker image layers, containers, tmpfs and code-process state | No | Images must be preloaded/allowed pulls must fetch exact digest; environments start cold | Use the existing versioned service archives: `services/stepfunctions/meta.ss.zst` and `data.ss.zst` for definitions/indices and bounded history payloads. Store durable DTOs, not Arc/compiler heap/channel layouts. Recompile definitions with an archived profile version. Unknown profile versions or invalid references fail import atomically. Add versioned Lambda image metadata to its existing archive, preserving the existing artifact protections. Imported run records do not authorize deletion of containers from another owner. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 6a21357d..13ac2382 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -83,6 +83,9 @@ Local image resolution now extends `test-lambda-docker-wire` with a second real Owned image preparation now extends the same real wire suite: acquire private aliases/live PinnedImages, prove selected unpacked size, retain leases through handler/container lifetime, remove containers, release/join pins, then stop DockerWire. Both retained architecture lanes pass and subsequent scoped image queries find no private aliases. The33 additional cases cover size proof, shared lease release, precommit rejection, complete outer500 with delayed nested commit, initial404 uncertainty, discarded successful responses, conflicting aliases, lost/failed DELETE, capacity observation and retained shutdown/worker/root recovery including off-runtime Drop. Source-pinned exceptions treat only exact fresh-alias400/404 and wire Busy as no-effect; HTTP500 remains uncertain. These gates do not establish provider publication/CAS, snapshot live-lease reacquisition, previous-run reconciliation or a production executor. + +Deployment settlement foundation (§2.10) must independently prove resource/revision/generation CAS, simultaneous duplicate completion, preservation of old code through updates/failure, pending mutation/publication rejection, static public reasons, fallible transaction rollback, and real lease capture/revocation. Tampered resolved snapshot hash/size must reject atomically; saved pending work must restore as failure with metadata-only images. A private-builder compile-fail doctest prevents arbitrary external candidate construction. These are store/capture proofs; they do not close L01/provider admission, retained deployment supervision or Docker executor acceptance. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index a613a0b7..be2763b5 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,7 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Provider deployment/status/CAS and leased invocation capture, directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native SDK/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained provider deployment ownership/API wiring, directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement provider deployment/status/CAS and leased invocation capture using ImageLeaseManager, then task1.3 environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe operations, followed by task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement the retained deployment supervisor and provider Image Create/Update API wiring above the typed store settlement and ImageLeaseManager, then task1.3 environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe operations, followed by task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 46c69b9a..9647a67a 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -197,6 +197,21 @@ The real Docker wire harness now acquires PinnedImages, keeps them through actua Next consume these artifacts in provider deployment/status/CAS and invocation capture, followed by pool/environment actors, startup directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and complete P1/M0 acceptance. No Docker execution capability is advertised by the current binary yet. +## P1 task1.3 deployment settlement and capture review + +Independent read-only review covers typed deployment state, store-issued candidates, atomic settlement and fallible mutation rollback, snapshot migration, public status projection and leased invocation capture. It explicitly excludes the still-pending retained deployment supervisor and provider Image Create/Update admission/API wiring. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W43 | P2; `deployment.rs`, generated candidate builder | Public TypedBuilder generation bypassed store-issued candidate construction despite private fields. Builder method, type and final build are now module-private | Compile-fail external construction doctest plus independent recheck | +| W44 | P2; `function_config.rs`, resolved-image hash/size | Imported or mutated resolved records could claim an unrelated CodeSha256/CodeSize. Central canonical base64 fingerprint validation now covers store/import/capture; live sizes must match unpacked proof and restored sizes remain bounded | Tampered hash/zero/oversize snapshot rejection, live-size mismatch rollback and capture corruption tests | +| W45 | P3; `rustack-core/src/http.rs:375`, workspace validation | The existing2ms-progress/10ms-idle real-clock test can legitimately hit Idle when host scheduling stalls, defeating its total-deadline-only expectation. Pure timer tests now use Tokio paused time, with test-util explicitly enabled only for core dev builds | All five HTTP tests pass, including identical idle/tie/absolute-deadline assertions, without changing production behavior | +| W46 | P2; `provider.rs`, self-review of publication guard integration | Rejecting publication only inside PublishVersion would let Image Create(Publish=true) insert before returning an error. Create/Update now reject this unsupported asynchronous combination before code processing or mutation; initial candidates clear unverified hash/size | Create leaves no resource; update preserves old revision/hash/status; initial Pending omits CodeSha256 | + +Twenty-five new Lambda core cases cover initial success/failure/retry, preserving old revisions during/after updates, independent resource/revision/generation CAS, two concurrent completion results, stale lease release, pending mutation/publication rejection, effective-entry rollback, published lease lifetime and resolved URI, snapshot state reconciliation and metadata-only availability. Invocation captures live leases through its captured request and rejects revoked/restored/tampered artifacts. The final independent recheck found no remaining in-slice issue. All25 new cases pass within346 Lambda core tests;92 core doctests pass, including the external builder rejection. Workspace build/test/fmt/Clippy/rustdoc pass with2,022 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case is not VM execution evidence. Fresh target-only strict boundary validation reports the same65 preexisting Lambda findings and no finding in new deployment/artifact/fingerprint paths; the unfiltered attempt also exposes seven existing model documentation findings. These remain B03, not a green strict gate. B01 remains the16 recorded dependency advisories; no package/version/lock change or new ignore was introduced. + +This foundation is not an accepted asynchronous deployment owner: it cannot prove bounded work-slot reservation, joined worker cancellation/panic or provider Pending admission until the next consumer lands. Docker capability remains unavailable; directory ownership/reconciliation, pulls, pool/environment ownership and P1 security/strict checks remain required. No new package/version is introduced; the core test-only Tokio feature reuses the existing version. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 9edeaf2c..25cd8db0 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -177,6 +177,16 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - Pinned by: [owned image preparation](./rustack-workflows-20-lambda-runtime-design.md#29-owned-image-pins-and-deployment-preparation), [transport/source study](../docs/research/study-bollard-docker-transport-boundaries.md). - Date:2026-10-25. Provider status/CAS, allowed pulls, directory ownership and previous-run reconciliation remain separate required consumers. +## D23 — Store-issued deployment generations and live captured images + +- **Context:** image preparation completes asynchronously while the old function may still execute, be updated, deleted or recreated. Snapshot metadata does not retain daemon availability. +- **Alternatives considered:** independently mutable status strings allow contradictory readiness/progress; revision-only settlement cannot distinguish repeated failed attempts with the same old code; deserializing an executable image from metadata invents ownership. +- **Decision:** use one typed DeploymentState, a module-private builder for store-issued ImageDeployment candidates, and resource UUID/original revision/generation checks in one commit transaction. ImageArtifact distinguishes unresolved/restored/live states; invocation and publication require a live PinnedImage. +- **Why:** failed updates preserve old executable code, stale/duplicate results cannot publish, and a snapshot cannot authorize an unowned Docker execution. One canonical base64 image fingerprint and proven unpacked size bind API metadata to captured content. A fallible store mutation commits only after its closure and record validation both succeed. +- **Migration:** snapshots retain a derived legacy state plus optional typed deployment; contradictory states reject, and abandoned in-progress work restores as a closed failure. Metadata requires exact-content availability reacquisition in P5. +- **Pinned by:** [runtime §2.10](./rustack-workflows-20-lambda-runtime-design.md#210-deployment-state-and-atomic-settlement), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-25. + ## Glossary | Term | Meaning in this specification | From 3713d6fd871c7e73cc329f013f3c374962c06bc8 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 22:38:51 -0700 Subject: [PATCH 016/106] phase 1: supervise asynchronous image deployments and wire provider admission Reserve bounded work and mailbox capacity before Pending metadata, return atomic acceptance receipts, retain keyed settlement guards and worker joins through cancellation/root failure, and cancel stale preparations without replay. Separate overall deployment and Docker read budgets; require explicit executor image readiness. Wire provider image Create/Update and RevisionId preconditions with modeled HTTP 412 responses. Verified 2,043 workspace tests, 366 core cases, 101 core doctests, all standard Rust gates, direct Native provider regressions, actual SDK/HTTP preconditions and both two-case real Docker wire lanes. Independent W47-W50 and wire-test review fixes pass. Correct historical Native SDK labels: that target bypasses SDK/HTTP. B01 advisories, B03 strict baseline, B04 DryRun/request validation and the actual Docker pool/executor/startup plus P2-P5 remain required; M0 is not complete. --- Makefile | 13 + crates/rustack-lambda-core/src/deployment.rs | 65 +- .../src/deployment/manager.rs | 681 +++++++++++++++++ .../src/deployment/manager_tests.rs | 684 ++++++++++++++++++ .../src/deployment_tests.rs | 2 +- .../rustack-lambda-core/src/docker/lease.rs | 27 +- .../src/docker/lease_tests.rs | 30 +- crates/rustack-lambda-core/src/docker/mod.rs | 3 + crates/rustack-lambda-core/src/error.rs | 23 + .../rustack-lambda-core/src/executor/mod.rs | 18 + crates/rustack-lambda-core/src/provider.rs | 486 ++++++++----- crates/rustack-lambda-core/src/storage.rs | 17 +- crates/rustack-lambda-model/src/error.rs | 4 + crates/rustack-lambda-model/src/input.rs | 25 + specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 32 + .../rustack-workflows-60-operations-design.md | 3 +- .../rustack-workflows-70-verification-plan.md | 4 + specs/rustack-workflows-90-impl-plan.md | 6 +- specs/rustack-workflows-93-review.md | 28 +- specs/rustack-workflows-99-decisions.md | 10 + tests/integration/src/test_lambda.rs | 180 +++++ 22 files changed, 2135 insertions(+), 208 deletions(-) create mode 100644 crates/rustack-lambda-core/src/deployment/manager.rs create mode 100644 crates/rustack-lambda-core/src/deployment/manager_tests.rs diff --git a/Makefile b/Makefile index 60d1d767..7a0c1bf4 100644 --- a/Makefile +++ b/Makefile @@ -214,6 +214,7 @@ SERVICE_SMOKE_TESTS := \ test_sts::tests::test_should_get_caller_identity_root \ test_lambda::tests::test_should_create_and_get_function \ test_lambda::tests::test_should_invoke_dry_run \ + test_lambda::tests::test_should_enforce_revision_preconditions_through_the_sdk_wire \ test_lambda::tests::test_should_create_function_with_s3_code \ test_lambda::tests::test_should_update_function_code_from_s3_and_read_new_metadata \ test_lambda::tests::test_should_fail_create_function_with_missing_s3_bucket \ @@ -261,6 +262,18 @@ test-json-boundaries: -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ -W clippy::indexing_slicing -W clippy::panic +# Requires a running Lambda HTTP service at SMOKE_ENDPOINT; executes actual AWS SDK IO. +test-lambda-revision-wire: + @S3_ENDPOINT_URL="$(SMOKE_ENDPOINT)" cargo test -p rustack-integration --lib test_lambda::tests::test_should_enforce_revision_preconditions_through_the_sdk_wire -- --ignored --exact --test-threads=1 + +.PHONY: test-lambda-revision-wire + +# Deployment state, retained image preparation ownership and provider control-plane tests. +test-lambda-deployments: + @cargo test -p rustack-lambda-core --lib deployment + +.PHONY: test-lambda-deployments + test-lambda-runtime: @cargo test -p rustack-lambda-runtime @cargo clippy -p rustack-lambda-runtime --lib --no-deps -- \ diff --git a/crates/rustack-lambda-core/src/deployment.rs b/crates/rustack-lambda-core/src/deployment.rs index ef722a6a..182fc795 100644 --- a/crates/rustack-lambda-core/src/deployment.rs +++ b/crates/rustack-lambda-core/src/deployment.rs @@ -3,8 +3,12 @@ //! A retained worker owns each candidate until settlement. Store keys bind resource //! lifetime, original revision and deployment generation; metadata is never a live lease. +#[cfg(unix)] +mod manager; use std::fmt; +#[cfg(unix)] +pub use manager::{ImageDeploymentConfig, ImageDeploymentManager, ImageDeploymentStatus}; use rustack_lambda_model::types::ImageConfig; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; @@ -308,6 +312,14 @@ impl ImageDeployment { self.candidate.image_config.as_ref() } + #[cfg(unix)] + pub(crate) fn is_current(&self, store: &FunctionStore) -> Result { + Ok(store + .get(&self.name)? + .as_ref() + .is_some_and(|record| self.matches(record))) + } + fn matches(&self, record: &FunctionRecord) -> bool { let state = if self.initial { DeploymentState::Creating(self.generation) @@ -333,8 +345,15 @@ impl FunctionStore { /// ``` pub fn insert_image_deployment( &self, - mut record: FunctionRecord, + record: FunctionRecord, ) -> Result { + self.reserve_image_create(record).map(|(job, _)| job) + } + + pub(crate) fn reserve_image_create( + &self, + mut record: FunctionRecord, + ) -> Result<(ImageDeployment, FunctionRecord), LambdaServiceError> { if record.latest.package_type != "Image" || !matches!(record.latest.image, ImageArtifact::Unresolved) || record.latest.deployment != DeploymentState::Ready @@ -348,8 +367,8 @@ impl FunctionStore { record.latest.code_size = 0; let job = ImageDeployment::new(&record, record.latest.clone(), true)?; record.latest.deployment = DeploymentState::Creating(job.generation); - self.insert(record)?; - Ok(job) + self.insert(record.clone())?; + Ok((job, record)) } /// Reserve an image update while preserving the previous usable revision. @@ -372,8 +391,22 @@ impl FunctionStore { reference: ImageReference, architecture: Architecture, ) -> Result { - self.try_update_if_revision(name, Some(expected), |record| { - if record.resource_id != resource { + self.reserve_image_update(name, resource, expected, reference, architecture, None) + .map(|(job, _)| job) + } + + pub(crate) fn reserve_image_update( + &self, + name: &str, + resource: FunctionResourceId, + expected: &str, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result<(ImageDeployment, FunctionRecord), LambdaServiceError> { + self.try_update_if_revision(name, None, |record| { + check_revision(requested, &record.latest.revision_id)?; + if record.resource_id != resource || record.latest.revision_id != expected { return Err(conflict("Function resource changed")); } record.latest.deployment.ensure_mutable()?; @@ -397,7 +430,7 @@ impl FunctionStore { } else { DeploymentState::Updating(job.generation) }; - Ok(job) + Ok((job, record.clone())) }) } @@ -477,6 +510,26 @@ impl FunctionStore { } } +pub(crate) fn check_revision( + requested: Option<&str>, + actual: &str, +) -> Result<(), LambdaServiceError> { + if let Some(requested) = requested { + if requested.is_empty() + || requested.len() > 100 + || !requested + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + { + return Err(invalid("Invalid RevisionId")); + } + if requested != actual { + return Err(LambdaServiceError::PreconditionFailed); + } + } + Ok(()) +} + fn invalid(message: &str) -> LambdaServiceError { LambdaServiceError::InvalidParameter { message: message.into(), diff --git a/crates/rustack-lambda-core/src/deployment/manager.rs b/crates/rustack-lambda-core/src/deployment/manager.rs new file mode 100644 index 00000000..e8511f66 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment/manager.rs @@ -0,0 +1,681 @@ +//! Bounded deployment admission, retained child joins and keyed failure recovery. +//! +//! Owner Drop intentionally leaves its signalled root running on the captured +//! runtime to drain. The root owns child joins; abnormal root Drop transfers +//! them and their reservations to retained emergency cleanup. Joined shutdown, +//! not diagnostic counts, proves this owner's settlement. The separate image +//! owner still owns critical mutations after a preparation future is cancelled. + +use std::{ + collections::HashMap, + future::Future, + mem, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use http::StatusCode; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, watch}, + task::{Id, JoinError, JoinSet}, + time::{Instant, sleep_until, timeout_at}, +}; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ImageDeployment, ImageDeploymentFailure}; +use crate::{ + artifact::{Architecture, ImageReference, PinnedImage}, + docker::{DockerError, ImageLeasePort}, + error::LambdaServiceError, + executor::ExecutorError, + invocation::CleanupTask, + storage::{FunctionRecord, FunctionStore}, +}; + +/// Validated accepted-deployment capacity and absolute preparation budget. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ImageDeploymentConfig { + jobs: u8, + budget: Duration, +} +impl Default for ImageDeploymentConfig { + fn default() -> Self { + Self { + jobs: 8, + budget: Duration::from_secs(300), + } + } +} +impl ImageDeploymentConfig { + /// Validate one bounded deployment owner; capacity includes queued and running jobs. + /// + /// # Errors + /// Rejects capacity outside1..32 or a budget outside1..3,600 seconds. + /// + /// ``` + /// let config = rustack_lambda_core::deployment::ImageDeploymentConfig::new( + /// 8, std::time::Duration::from_secs(300))?; + /// # Ok::<(),rustack_lambda_core::error::LambdaServiceError>(()) + /// ``` + pub fn new(jobs: u8, budget: Duration) -> Result { + if !(1..=32).contains(&jobs) + || budget < Duration::from_secs(1) + || budget > Duration::from_secs(3600) + { + return Err(invalid("Invalid deployment capacity or deadline")); + } + Ok(Self { jobs, budget }) + } +} + +/// Diagnostic counts; use joined quiescence/shutdown for settlement evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ImageDeploymentStatus { + /// Queued/running jobs whose capacity has not yet been reaped. + pub accepted: usize, + /// Whether both owners currently permit new preparation. + pub accepting: bool, + /// A deployment supervisor failure permanently closed this owner. + pub failed: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Mode { + Running, + Drain, + Cancel, +} +#[derive(Debug, Default)] +struct Health { + failed: AtomicBool, + #[cfg(test)] + crash_root: AtomicBool, + #[cfg(test)] + crash_worker: AtomicBool, +} +fn signal(control: &watch::Sender, mode: Mode) { + control.send_if_modified(|current| { + if *current < mode { + *current = mode; + true + } else { + false + } + }); +} +fn unavailable() -> LambdaServiceError { + LambdaServiceError::ResourceNotReady { + message: "Image deployment owner is unavailable".into(), + } +} +fn invalid(message: &str) -> LambdaServiceError { + LambdaServiceError::InvalidParameter { + message: message.into(), + } +} +fn failed() -> ExecutorError { + ExecutorError::CleanupFailed("Image deployment supervisor failed".into()) +} + +#[derive(Debug, TypedBuilder)] +struct Pending { + job: ImageDeployment, + store: Arc, + health: Arc, + control: watch::Sender, + #[builder(default)] + settled: AtomicBool, +} +impl Pending { + fn finish( + &self, + outcome: Result, + ) -> Result<(), LambdaServiceError> { + if !self.settled.load(Ordering::Acquire) { + self.store.settle_image_deployment(&self.job, outcome)?; + self.settled.store(true, Ordering::Release); + } + Ok(()) + } + fn fail_owner(&self) { + self.health.failed.store(true, Ordering::Release); + signal(&self.control, Mode::Cancel); + } +} +impl Drop for Pending { + fn drop(&mut self) { + if self.settled.load(Ordering::Acquire) { + return; + } + let reason = if self.health.failed.load(Ordering::Acquire) + || *self.control.borrow() != Mode::Cancel + { + self.fail_owner(); + ImageDeploymentFailure::SupervisorFailed + } else { + ImageDeploymentFailure::Cancelled + }; + if self.finish(Err(reason)).is_err() { + self.fail_owner(); + tracing::error!("Abandoned image deployment could not be settled"); + } + } +} + +#[derive(Debug)] +struct Envelope { + id: Uuid, + pending: Arc, + slot: OwnedSemaphorePermit, + deadline: Instant, +} +#[derive(Debug)] +struct Row { + pending: Arc, + _slot: OwnedSemaphorePermit, +} +#[derive(Debug)] +struct Reservation { + queue: mpsc::OwnedPermit, + slot: OwnedSemaphorePermit, + deadline: Instant, +} + +/// Own accepted image deployments through actual worker joins and keyed settlement. +#[derive(Debug, TypedBuilder)] +#[builder( + builder_method(vis = ""), + builder_type(vis = ""), + build_method(vis = "") +)] +#[non_exhaustive] +pub struct ImageDeploymentManager { + store: Arc, + preparation: ImageLeasePort, + config: ImageDeploymentConfig, + capacity: Arc, + sender: mpsc::Sender, + control: watch::Sender, + health: Arc, + task: CleanupTask, + emergency: Arc, +} +impl ImageDeploymentManager { + /// Start a retained owner using an already qualified image preparation port. + /// + /// # Errors + /// Rejects unavailable preparation, excess capacity or lack of an entered runtime. + /// This does not probe or authorize the Docker execution backend. + /// + /// ```no_run + /// # fn start(store:std::sync::Arc, port:rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let owner = rustack_lambda_core::deployment::ImageDeploymentManager::start(store, port, + /// rustack_lambda_core::deployment::ImageDeploymentConfig::default())?; + /// # Ok(()) } + /// ``` + pub fn start( + store: Arc, + preparation: ImageLeasePort, + config: ImageDeploymentConfig, + ) -> Result { + preparation + .available() + .map_err(|_| LambdaServiceError::DockerNotAvailable)?; + if config.jobs > preparation.preparation_jobs() { + return Err(invalid("Deployment jobs exceed image preparation capacity")); + } + let runtime = Handle::try_current().map_err(|_| unavailable())?; + let (sender, inbox) = mpsc::channel(usize::from(config.jobs)); + let (control, _) = watch::channel(Mode::Running); + let health = Arc::new(Health::default()); + let emergency = Arc::new(CleanupTask::default()); + let actor = Actor::builder() + .inbox(inbox) + .control(control.clone()) + .preparation(preparation.clone()) + .health(health.clone()) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + let task = CleanupTask::default(); + task.start_on(&runtime, move || actor.run()); + Ok(Self::builder() + .store(store) + .preparation(preparation) + .config(config) + .capacity(Arc::new(Semaphore::new(usize::from(config.jobs)))) + .sender(sender) + .control(control) + .health(health) + .task(task) + .emergency(emergency) + .build()) + } + + fn available(&self) -> Result<(), LambdaServiceError> { + if *self.control.borrow() != Mode::Running || self.health.failed.load(Ordering::Acquire) { + return Err(unavailable()); + } + self.preparation + .available() + .map_err(|_| LambdaServiceError::DockerNotAvailable) + } + fn reserve(&self) -> Result { + self.available()?; + let deadline = Instant::now() + .checked_add(self.config.budget) + .ok_or_else(unavailable)?; + let slot = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| LambdaServiceError::TooManyRequests)?; + let queue = self + .sender + .clone() + .try_reserve_owned() + .map_err(|_| unavailable())?; + self.available()?; + Ok(Reservation { + queue, + slot, + deadline, + }) + } + fn accept( + &self, + reservation: Reservation, + job: ImageDeployment, + accepted: FunctionRecord, + ) -> FunctionRecord { + let pending = Arc::new( + Pending::builder() + .job(job) + .store(self.store.clone()) + .health(self.health.clone()) + .control(self.control.clone()) + .build(), + ); + reservation.queue.send(Envelope { + id: Uuid::new_v4(), + pending, + slot: reservation.slot, + deadline: reservation.deadline, + }); + accepted + } + + /// Reserve work before inserting an initial Pending image function. + /// + /// # Errors + /// Rejects invalid/duplicate resources, unavailable owners or exhausted capacity. + /// The returned snapshot is captured before enqueue; dropping it does not cancel accepted work. + /// + /// ```no_run + /// # fn create(owner:&rustack_lambda_core::deployment::ImageDeploymentManager, record:rustack_lambda_core::storage::FunctionRecord) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let pending = owner.create(record)?; + /// # Ok(()) } + /// ``` + pub fn create(&self, record: FunctionRecord) -> Result { + let reservation = self.reserve()?; + let (job, accepted) = self.store.reserve_image_create(record)?; + Ok(self.accept(reservation, job, accepted)) + } + + /// Reserve an update while preserving previous code and configuration. + /// + /// # Errors + /// Rejects stale resources/revisions, invalid settings, pending work or unavailable capacity. + /// + /// ```no_run + /// # fn update(owner:&rustack_lambda_core::deployment::ImageDeploymentManager, before:&rustack_lambda_core::storage::FunctionRecord) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let reference = "worker:v2".parse().map_err(|_| rustack_lambda_core::error::LambdaServiceError::DockerNotAvailable)?; + /// owner.update(before, reference, rustack_lambda_core::artifact::Architecture::Arm64)?; + /// # Ok(()) } + /// ``` + pub fn update( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + ) -> Result { + self.update_with_revision(before, reference, architecture, None) + } + + pub(crate) fn update_with_revision( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result { + super::check_revision(requested, &before.latest.revision_id)?; + let reservation = self.reserve()?; + let (job, accepted) = self.store.reserve_image_update( + &before.name, + before.resource_id, + &before.latest.revision_id, + reference, + architecture, + requested, + )?; + Ok(self.accept(reservation, job, accepted)) + } + + /// Observe bounded admission; counts alone never prove clean settlement. + /// + /// ```no_run + /// # fn inspect(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let count = owner.status().accepted; + /// # } + /// ``` + #[must_use] + pub fn status(&self) -> ImageDeploymentStatus { + ImageDeploymentStatus { + accepted: usize::from(self.config.jobs) + .saturating_sub(self.capacity.available_permits()), + accepting: self.available().is_ok(), + failed: self.health.failed.load(Ordering::Acquire), + } + } + + pub(crate) fn cancel(&self) { + signal(&self.control, Mode::Cancel); + } + + async fn join(&self, deadline: Instant) -> Result<(), ExecutorError> { + if Instant::now() >= deadline { + return Err(ExecutorError::DeadlineExceeded); + } + let result = timeout_at(deadline, async { + let primary = self.task.wait().await; + let emergency = self.emergency.wait().await; + primary.and(emergency) + }) + .await + .map_err(|_| ExecutorError::DeadlineExceeded)?; + if Instant::now() >= deadline { + return Err(ExecutorError::DeadlineExceeded); + } + result + } + /// Close admission and drain accepted workers without cancelling preparation. + /// + /// # Errors + /// Returns deadline or cached supervision failure while retaining joins. + /// The separate image owner must also join its critical mutations and cleanup. + /// + /// ```no_run + /// # async fn stop(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let result = owner.quiesce(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn quiesce(&self, deadline: Instant) -> Result<(), ExecutorError> { + signal(&self.control, Mode::Drain); + self.join(deadline).await + } + + /// Cancel accepted preparations, settle records and join this owner's workers. + /// + /// # Errors + /// Returns deadline or cached supervision failure; cancelled waiters retain joins. + /// + /// ```no_run + /// # async fn stop(owner:&rustack_lambda_core::deployment::ImageDeploymentManager) { + /// let result = owner.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; + /// # } + /// ``` + pub async fn shutdown(&self, deadline: Instant) -> Result<(), ExecutorError> { + signal(&self.control, Mode::Cancel); + self.join(deadline).await + } +} +impl Drop for ImageDeploymentManager { + fn drop(&mut self) { + signal(&self.control, Mode::Cancel); + } +} + +#[derive(Debug, TypedBuilder)] +struct Actor { + inbox: mpsc::Receiver, + preparation: ImageLeasePort, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default)] + jobs: JoinSet>, + #[builder(default)] + rows: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + fn fail(&self) { + self.health.failed.store(true, Ordering::Release); + signal(&self.control, Mode::Cancel); + } + async fn run(mut self) -> Result<(), ExecutorError> { + let mut control = self.control.subscribe(); + let mut closed = false; + loop { + if *control.borrow() != Mode::Running { + self.inbox.close(); + } + if closed && self.jobs.is_empty() { + break; + } + tokio::select! { + biased; + _ = control.changed() => {}, + joined = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => if let Some(joined) = joined { self.reap(&joined); }, + envelope = self.inbox.recv(), if !closed => if let Some(envelope) = envelope { + if *control.borrow() == Mode::Cancel { + if envelope.pending.finish(Err(ImageDeploymentFailure::Cancelled)).is_err() { self.fail(); } + } else { self.start(envelope); } + } else { + closed = true; + if *control.borrow() == Mode::Running { signal(&self.control, Mode::Cancel); } + }, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(failed()) + } else { + Ok(()) + } + } + fn start(&mut self, envelope: Envelope) { + #[cfg(test)] + assert!( + !self.health.crash_root.swap(false, Ordering::AcqRel), + "injected deployment root failure" + ); + let Envelope { + id, + pending, + slot, + deadline, + } = envelope; + self.rows.insert( + id, + Row { + pending: pending.clone(), + _slot: slot, + }, + ); + let preparation = self.preparation.clone(); + let control = self.control.subscribe(); + #[cfg(test)] + let health = self.health.clone(); + let worker = self.jobs.spawn(async move { + #[cfg(test)] + assert!( + !health.crash_worker.swap(false, Ordering::AcqRel), + "injected deployment worker failure" + ); + let outcome = prepare(&pending, preparation, control, deadline).await; + let outcome = if pending.health.failed.load(Ordering::Acquire) { + Err(ImageDeploymentFailure::SupervisorFailed) + } else { + outcome + }; + if pending.finish(outcome).is_err() + && pending + .finish(Err(ImageDeploymentFailure::InvalidImage)) + .is_err() + { + return Err(failed()); + } + Ok(id) + }); + self.index.insert(worker.id(), id); + } + fn reap(&mut self, joined: &Result<(Id, Result), JoinError>) { + let task = match joined { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(id) = self.index.remove(&task) else { + self.fail(); + return; + }; + let Some(row) = self.rows.remove(&id) else { + self.fail(); + return; + }; + if !matches!(joined, Ok((_, Ok(returned))) if *returned == id) + || !row.pending.settled.load(Ordering::Acquire) + { + self.fail(); + if row + .pending + .finish(Err(ImageDeploymentFailure::SupervisorFailed)) + .is_err() + { + tracing::error!("Failed image worker could not be settled"); + } + } + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + if self.rows.is_empty() { + return; + } + self.fail(); + let rows = mem::take(&mut self.rows); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + let mut clean = true; + for row in rows.into_values() { + clean &= row + .pending + .finish(Err(ImageDeploymentFailure::SupervisorFailed)) + .is_ok(); + } + if clean { Ok(()) } else { Err(failed()) } + }); + } +} + +async fn prepare( + pending: &Pending, + preparation: ImageLeasePort, + mut control: watch::Receiver, + deadline: Instant, +) -> Result { + let mut changes = pending.store.changes(); + let preparation = preparation.prepare( + pending.job.reference(), + pending.job.architecture(), + pending.job.image_config(), + deadline, + ); + tokio::pin!(preparation); + let result = loop { + check(pending, &control, deadline)?; + match next(preparation.as_mut(), &mut control, &mut changes, deadline).await? { + Progress::Prepared(result) => break result.map_err(|error| map_failure(&error)), + Progress::Changed => {} + } + }; + check(pending, &control, deadline)?; + result +} +#[derive(Debug)] +enum Progress { + Prepared(Result), + Changed, +} +async fn next( + preparation: Pin<&mut impl Future>>, + control: &mut watch::Receiver, + changes: &mut watch::Receiver<()>, + deadline: Instant, +) -> Result { + tokio::select! { + biased; + () = sleep_until(deadline) => Err(ImageDeploymentFailure::DeadlineExceeded), + _ = control.wait_for(|mode| *mode == Mode::Cancel) => Err(ImageDeploymentFailure::Cancelled), + result = preparation => Ok(Progress::Prepared(result)), + notice = changes.changed() => notice.map(|()| Progress::Changed).map_err(|_| ImageDeploymentFailure::SupervisorFailed), + } +} + +fn check( + pending: &Pending, + control: &watch::Receiver, + deadline: Instant, +) -> Result<(), ImageDeploymentFailure> { + if Instant::now() >= deadline { + return Err(ImageDeploymentFailure::DeadlineExceeded); + } + if *control.borrow() == Mode::Cancel { + return Err(ImageDeploymentFailure::Cancelled); + } + if !pending + .job + .is_current(&pending.store) + .map_err(|_| ImageDeploymentFailure::SupervisorFailed)? + { + return Err(ImageDeploymentFailure::Cancelled); + } + Ok(()) +} +fn map_failure(error: &DockerError) -> ImageDeploymentFailure { + match error { + DockerError::Status(StatusCode::NOT_FOUND) => ImageDeploymentFailure::ImageNotFound, + DockerError::Deadline => ImageDeploymentFailure::DeadlineExceeded, + DockerError::Cancelled => ImageDeploymentFailure::Cancelled, + DockerError::Worker => ImageDeploymentFailure::SupervisorFailed, + DockerError::Configuration + | DockerError::Schema + | DockerError::Image(_) + | DockerError::Json(_) + | DockerError::Protocol + | DockerError::TooLarge => ImageDeploymentFailure::InvalidImage, + _ => ImageDeploymentFailure::RuntimeUnavailable, + } +} + +#[cfg(test)] +#[path = "manager_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/deployment/manager_tests.rs b/crates/rustack-lambda-core/src/deployment/manager_tests.rs new file mode 100644 index 00000000..90bc8268 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment/manager_tests.rs @@ -0,0 +1,684 @@ +use std::{ + future::Future, + task::{Context, Poll, Waker}, +}; + +use async_trait::async_trait; +use rstest::rstest; +use rustack_lambda_model::{ + input::{ + CreateFunctionInput, PublishVersionInput, UpdateFunctionCodeInput, + UpdateFunctionConfigurationInput, + }, + types::FunctionCode, +}; +use tokio::time::timeout; + +use super::*; +use crate::{ + config::LambdaConfig, + deployment::{DeploymentState, tests::record}, + docker::{ + ImageLeaseManager, + image_fixture::{Command, FakePins, Mode as DaemonMode}, + }, + executor::{Executor, ExecutorBackend, InvokeRequest, InvokeResponse, NoopExecutor}, + invocation::InvocationContext, + provider::RustackLambda, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn until(condition: impl Fn() -> bool) { + timeout(Duration::from_secs(10), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +fn latest(store: &FunctionStore) -> crate::storage::VersionRecord { + store.get("worker").unwrap().unwrap().latest +} +fn named(name: &str) -> FunctionRecord { + let mut record = record(); + record.name = name.into(); + record.arn = format!("arn:aws:lambda:us-east-1:000000000000:function:{name}"); + record +} +struct Fixture { + root: tempfile::TempDir, + store: Arc, + fake: FakePins, + images: ImageLeaseManager, + manager: ImageDeploymentManager, +} +impl Fixture { + async fn new(mode: DaemonMode, jobs: u8, budget: Duration) -> Self { + let fake = FakePins::new(mode).await; + let images = fake.owner(8, jobs); + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let manager = ImageDeploymentManager::start( + store.clone(), + images.port(), + ImageDeploymentConfig::new(jobs, budget).unwrap(), + ) + .unwrap(); + Self { + root, + store, + fake, + images, + manager, + } + } + async fn idle(&self) { + until(|| self.manager.status().accepted == 0).await; + } + async fn ready(&self) { + let pending = self.manager.create(record()).unwrap(); + assert_eq!(pending.latest.deployment.state(), "Pending"); + self.idle().await; + assert_eq!(latest(&self.store).deployment, DeploymentState::Ready); + } + async fn close(self, failed: bool) { + let Self { + root, + store, + fake, + mut images, + manager, + } = self; + assert_eq!(manager.shutdown(deadline()).await.is_err(), failed); + drop(manager); + drop(store); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; + drop(root); + } +} + +#[tokio::test] +async fn test_should_keep_accepted_preparation_owned_after_response_is_discarded() { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + drop(fixture.manager.create(record()).unwrap()); + fixture.idle().await; + let active = latest(&fixture.store); + assert_eq!(active.deployment, DeploymentState::Ready); + assert_eq!(active.code_size, 4096); + assert!(active.image.pinned().unwrap().is_live()); + assert!(!format!("{:?}", fixture.manager.status()).contains("private")); + drop(active); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_reject_capacity_before_resource_insertion_and_drain_after_cancellation() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(10)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(matches!( + fixture.manager.create(named("other")), + Err(LambdaServiceError::TooManyRequests) + )); + assert!(!fixture.store.contains("other").unwrap()); + assert!(fixture.manager.shutdown(Instant::now()).await.is_err()); + fixture.idle().await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); + assert!(fixture.images.port().status().pins > 0); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_preserve_old_revision_during_update_and_after_preparation_failure() { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + fixture.ready().await; + let before = fixture.store.get("worker").unwrap().unwrap(); + let accepted = fixture + .manager + .update(&before, "missing:v2".parse().unwrap(), Architecture::Arm64) + .unwrap(); + assert_eq!(accepted.latest.deployment.update_status(), "InProgress"); + assert_eq!(accepted.latest.code_sha256, before.latest.code_sha256); + fixture.idle().await; + let failed = latest(&fixture.store); + assert_eq!( + failed.deployment, + DeploymentState::UpdateFailed(ImageDeploymentFailure::ImageNotFound) + ); + assert_eq!(failed.revision_id, before.latest.revision_id); + assert_eq!(failed.code_sha256, before.latest.code_sha256); + drop(failed); + drop(accepted); + drop(before); + let before = fixture.store.get("worker").unwrap().unwrap(); + fixture.fake.commands.send(Command::Retarget).await.unwrap(); + fixture + .manager + .update(&before, "worker:v1".parse().unwrap(), Architecture::Arm64) + .unwrap(); + fixture.idle().await; + assert_eq!(latest(&fixture.store).deployment, DeploymentState::Ready); + assert_ne!( + latest(&fixture.store).code_sha256, + before.latest.code_sha256 + ); + drop(before); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_keep_deployment_deadline_absolute_and_leave_critical_tag_cleanup_owned() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(1)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + // Notifications must recheck identity without restarting the preparation future. + for _ in 0..4 { + fixture + .store + .update("worker", |record| { + record.tags.insert("progress".into(), "value".into()); + }) + .unwrap(); + } + fixture.idle().await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::DeadlineExceeded) + ); + assert!(fixture.images.port().status().pins > 0); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_cancel_deleted_generation_without_waiting_for_its_deployment_deadline() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(300)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + fixture.store.remove("worker").unwrap(); + let mut replacement = record(); + replacement.latest.description = "replacement".into(); + let resource = replacement.resource_id; + fixture.store.insert(replacement).unwrap(); + fixture.idle().await; + assert_eq!( + fixture.store.get("worker").unwrap().unwrap().resource_id, + resource + ); + assert_eq!(latest(&fixture.store).description, "replacement"); + assert!(latest(&fixture.store).image.pinned().is_none()); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_return_the_atomic_acceptance_receipt_after_immediate_deletion() { + let fixture = Fixture::new(DaemonMode::Normal, 1, Duration::from_secs(10)).await; + let reservation = fixture.manager.reserve().unwrap(); + let (job, receipt) = fixture.store.reserve_image_create(record()).unwrap(); + fixture.store.remove("worker").unwrap(); + let accepted = fixture.manager.accept(reservation, job, receipt); + assert_eq!(accepted.latest.deployment.state(), "Pending"); + fixture.idle().await; + assert!(!fixture.manager.status().failed); + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 0); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_retain_join_when_drain_or_shutdown_waiter_is_abandoned() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(10)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let mut waiter = Box::pin(fixture.manager.quiesce(deadline())); + assert!(matches!( + waiter + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + drop(waiter); + assert!(!fixture.manager.status().accepting); + assert_eq!(fixture.manager.status().accepted, 1); + let mut waiter = Box::pin(fixture.manager.shutdown(deadline())); + let _ = waiter + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())); + drop(waiter); + fixture.idle().await; + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + ); + fixture.fake.commands.send(Command::Commit).await.unwrap(); + fixture.close(false).await; +} + +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_settle_failed_workers_or_queued_root_work_and_cache_failure( + #[case] root: bool, +) { + let fixture = Fixture::new(DaemonMode::Normal, 2, Duration::from_secs(10)).await; + if root { + fixture + .manager + .health + .crash_root + .store(true, Ordering::Release); + } else { + fixture + .manager + .health + .crash_worker + .store(true, Ordering::Release); + } + fixture.manager.create(record()).unwrap(); + fixture.idle().await; + assert!(fixture.manager.status().failed); + assert_eq!( + latest(&fixture.store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::SupervisorFailed) + ); + assert!(fixture.manager.shutdown(deadline()).await.is_err()); + assert!(fixture.manager.shutdown(deadline()).await.is_err()); + assert!(fixture.manager.create(named("other")).is_err()); + fixture.close(true).await; +} + +#[tokio::test] +async fn test_should_recover_joined_rows_after_root_drop_outside_an_entered_runtime() { + let fake = FakePins::new(DaemonMode::Never).await; + let mut images = fake.owner(4, 1); + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(FunctionStore::new(root.path())); + let job = store.insert_image_deployment(record()).unwrap(); + let health = Arc::new(Health::default()); + let (control, _) = watch::channel(Mode::Running); + let emergency = Arc::new(CleanupTask::default()); + let (_sender, inbox) = mpsc::channel(1); + let mut actor = Actor::builder() + .inbox(inbox) + .preparation(images.port()) + .control(control.clone()) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(emergency.clone()) + .build(); + let capacity = Arc::new(Semaphore::new(1)); + actor.start(Envelope { + id: Uuid::new_v4(), + pending: Arc::new( + Pending::builder() + .job(job) + .store(store.clone()) + .health(health.clone()) + .control(control) + .build(), + ), + slot: capacity.clone().acquire_owned().await.unwrap(), + deadline: deadline(), + }); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let thread = std::thread::spawn(move || drop(actor)); + tokio::task::spawn_blocking(move || thread.join().unwrap()) + .await + .unwrap(); + emergency.wait().await.unwrap(); + assert!(health.failed.load(Ordering::Acquire)); + assert_eq!(capacity.available_permits(), 1); + assert_eq!( + latest(&store).deployment, + DeploymentState::CreateFailed(ImageDeploymentFailure::SupervisorFailed) + ); + fake.commands.send(Command::Commit).await.unwrap(); + drop(store); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[test] +fn test_should_validate_deployment_budgets_and_map_bounded_failure_classes() { + assert!(ImageDeploymentConfig::new(0, Duration::from_secs(1)).is_err()); + assert!(ImageDeploymentConfig::new(33, Duration::from_secs(1)).is_err()); + assert!(ImageDeploymentConfig::new(1, Duration::from_nanos(1)).is_err()); + assert!(ImageDeploymentConfig::new(1, Duration::from_secs(3601)).is_err()); + assert_eq!( + map_failure(&DockerError::Status(StatusCode::NOT_FOUND)), + ImageDeploymentFailure::ImageNotFound + ); + assert_eq!( + map_failure(&DockerError::Quarantined), + ImageDeploymentFailure::RuntimeUnavailable + ); + assert_eq!( + map_failure(&DockerError::Protocol), + ImageDeploymentFailure::InvalidImage + ); + assert_eq!( + map_failure(&DockerError::Worker), + ImageDeploymentFailure::SupervisorFailed + ); +} + +#[derive(Debug)] +struct ControlPlaneExecutor; +#[async_trait] +impl Executor for ControlPlaneExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + fn available(&self, _: &InvokeRequest) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported( + "Control-plane fixture does not execute code".into(), + )) + } + async fn invoke( + &self, + _: InvokeRequest, + _: InvocationContext, + ) -> Result { + Err(ExecutorError::Unsupported( + "Control-plane fixture does not execute code".into(), + )) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + Ok(()) + } +} +fn input(name: &str) -> CreateFunctionInput { + CreateFunctionInput { + function_name: name.into(), + role: "arn:aws:iam::000000000000:role/test".into(), + package_type: Some("Image".into()), + architectures: Some(vec!["arm64".into()]), + code: FunctionCode { + image_uri: Some("worker:v1".into()), + ..Default::default() + }, + ..Default::default() + } +} +async fn wait_provider(provider: &RustackLambda) { + until(|| { + let config = provider.get_function_configuration("worker", None).unwrap(); + config.last_update_status.as_deref() != Some("InProgress") + }) + .await; + // Give the joined root a chance to reap the completed worker before another admission. + tokio::task::yield_now().await; +} + +#[tokio::test] +async fn test_should_route_provider_image_create_update_status_and_explicit_revision_preconditions() +{ + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(8, 2); + let root = tempfile::tempdir().unwrap(); + let config = LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }; + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + config, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(2, Duration::from_secs(10)).unwrap(), + ) + .unwrap(); + let pending = provider.create_function(input("worker")).await.unwrap(); + assert_eq!(pending.state.as_deref(), Some("Pending")); + assert!(pending.code_sha256.is_none()); + wait_provider(&provider).await; + let before = provider.get_function_configuration("worker", None).unwrap(); + assert_eq!(before.state.as_deref(), Some("Active")); + assert!(before.code_sha256.is_some()); + assert!(matches!( + provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v1".into()), + revision_id: Some("stale".into()), + ..Default::default() + } + ) + .await, + Err(LambdaServiceError::PreconditionFailed) + )); + assert!(matches!( + provider.update_function_configuration( + "worker", + &UpdateFunctionConfigurationInput { + revision_id: Some("stale".into()), + ..Default::default() + } + ), + Err(LambdaServiceError::PreconditionFailed) + )); + fake.commands.send(Command::Retarget).await.unwrap(); + let updating = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("worker:v1".into()), + revision_id: before.revision_id.clone(), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(updating.state.as_deref(), Some("Active")); + assert_eq!(updating.last_update_status.as_deref(), Some("InProgress")); + assert_eq!(updating.code_sha256, before.code_sha256); + wait_provider(&provider).await; + let after = provider.get_function_configuration("worker", None).unwrap(); + assert_ne!(after.code_sha256, before.code_sha256); + assert!(matches!( + provider.publish_version( + "worker", + &PublishVersionInput { + revision_id: before.revision_id, + ..Default::default() + } + ), + Err(LambdaServiceError::PreconditionFailed) + )); + let published = provider + .publish_version("worker", &PublishVersionInput::default()) + .unwrap(); + assert_eq!(published.version.as_deref(), Some("1")); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_unqualified_image_deployment_before_resource_mutation() { + let provider = RustackLambda::new(LambdaConfig::default()); + assert!(matches!( + provider.create_function(input("worker")).await, + Err(LambdaServiceError::DockerNotAvailable) + )); + assert!(!provider.store().contains("worker").unwrap()); + assert!(NoopExecutor::new().image_available().is_err()); + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + assert!( + RustackLambda::new(LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(1)).unwrap() + ) + .is_err() + ); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_poll_ready_preparation_ahead_of_continuously_ready_unrelated_changes() { + let (_stop, mut control) = watch::channel(Mode::Running); + let (publisher, mut changes) = watch::channel(()); + for _ in 0..16 { + publisher.send_replace(()); + let image = + PinnedImage::fixture(crate::artifact::ResolvedImage::fixture(Architecture::Arm64)); + let prepared = std::future::ready(Ok(image)); + tokio::pin!(prepared); + assert!(matches!( + next(prepared.as_mut(), &mut control, &mut changes, deadline()).await, + Ok(Progress::Prepared(Ok(_))) + )); + } +} + +#[tokio::test] +async fn test_should_accept_a_long_overall_budget_while_bounding_each_docker_read() { + let fixture = Fixture::new(DaemonMode::Normal, 1, Duration::from_secs(3600)).await; + fixture.ready().await; + assert_eq!(fixture.fake.stats.tags.load(Ordering::Acquire), 1); + fixture.close(false).await; +} + +#[tokio::test] +async fn test_should_keep_signalled_cleanup_running_when_owner_is_dropped_off_runtime() { + let fixture = Fixture::new(DaemonMode::Never, 1, Duration::from_secs(300)).await; + fixture.manager.create(record()).unwrap(); + fixture + .fake + .stats + .until(|| fixture.fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + let Fixture { + root, + store, + fake, + mut images, + manager, + } = fixture; + let thread = std::thread::spawn(move || drop(manager)); + tokio::task::spawn_blocking(move || thread.join().unwrap()) + .await + .unwrap(); + until(|| { + latest(&store).deployment + == DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + }) + .await; + fake.commands.send(Command::Commit).await.unwrap(); + drop(store); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; + drop(root); +} + +#[tokio::test] +async fn test_should_mark_an_empty_abnormally_dropped_root_as_failed() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let health = Arc::new(Health::default()); + let (control, _) = watch::channel(Mode::Running); + let (_sender, inbox) = mpsc::channel(1); + let actor = Actor::builder() + .inbox(inbox) + .preparation(images.port()) + .control(control.clone()) + .health(health.clone()) + .runtime(Handle::current()) + .emergency(Arc::new(CleanupTask::default())) + .build(); + drop(actor); + assert!(health.failed.load(Ordering::Acquire)); + assert_eq!(*control.borrow(), Mode::Cancel); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_fail_quiescence_at_one_deadline_then_cancel_without_losing_ownership() { + let fake = FakePins::new(DaemonMode::Never).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(300)).unwrap(), + ) + .unwrap(); + provider.create_function(input("worker")).await.unwrap(); + fake.stats + .until(|| fake.stats.tags.load(Ordering::Acquire) == 1) + .await; + assert!(provider.quiesce(Duration::from_millis(10)).await.is_err()); + until(|| { + latest(provider.store()).deployment + == DeploymentState::CreateFailed(ImageDeploymentFailure::Cancelled) + }) + .await; + fake.commands.send(Command::Commit).await.unwrap(); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} + +#[tokio::test] +async fn test_should_reject_deployment_capacity_larger_than_the_preparation_owner() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + assert!( + ImageDeploymentManager::start( + Arc::new(FunctionStore::new(root.path())), + images.port(), + ImageDeploymentConfig::new(2, Duration::from_secs(1)).unwrap() + ) + .is_err() + ); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} diff --git a/crates/rustack-lambda-core/src/deployment_tests.rs b/crates/rustack-lambda-core/src/deployment_tests.rs index 510716df..5db505c3 100644 --- a/crates/rustack-lambda-core/src/deployment_tests.rs +++ b/crates/rustack-lambda-core/src/deployment_tests.rs @@ -48,7 +48,7 @@ fn image( ); (image, released, stop) } -fn record() -> FunctionRecord { +pub(super) fn record() -> FunctionRecord { FunctionRecord { resource_id: FunctionResourceId::new(), name: "worker".into(), diff --git a/crates/rustack-lambda-core/src/docker/lease.rs b/crates/rustack-lambda-core/src/docker/lease.rs index 8e380e1b..0cc114b9 100644 --- a/crates/rustack-lambda-core/src/docker/lease.rs +++ b/crates/rustack-lambda-core/src/docker/lease.rs @@ -34,6 +34,17 @@ use crate::{ invocation::CleanupTask, }; +fn check_preparation_deadline(deadline: Instant) -> Result<(), DockerError> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|value| !value.is_zero()) + .ok_or(DockerError::Deadline)?; + if remaining > Duration::from_secs(3600) { + return Err(DockerError::Configuration); + } + Ok(()) +} + /// Validated image-preparation and live-pin admission/IO budgets. #[derive(Debug, Clone, Copy)] #[non_exhaustive] @@ -190,13 +201,17 @@ pub struct ImageLeasePort { config: ImageLeaseConfig, } impl ImageLeasePort { - fn available(&self) -> Result<(), DockerError> { + pub(crate) fn available(&self) -> Result<(), DockerError> { self.health.check()?; if *self.stop.borrow() { return Err(DockerError::Stopped); } Ok(()) } + pub(crate) fn preparation_jobs(&self) -> u8 { + self.config.jobs + } + /// Prepare a live pinned artifact; dropping this future signals cancellation. /// /// # Errors @@ -218,7 +233,7 @@ impl ImageLeasePort { deadline: Instant, ) -> Result { self.available()?; - check_deadline(deadline)?; + check_preparation_deadline(deadline)?; if let Some(config) = overrides { validate_image_config(config).map_err(|_| DockerError::Configuration)?; } @@ -821,7 +836,7 @@ impl Worker { self.check(deadline, cancel)?; let resolver = ImageResolver::new(self.docker.clone()); let mut stop = self.control.subscribe(); - let image = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.resolve_local(reference,architecture,overrides,deadline)=>result?,}; + let image = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.resolve_local(reference,architecture,overrides,deadline.min(self.config.deadline()))=>result?,}; self.check(deadline, cancel)?; let expected = image.metadata().image_id.clone(); self.progress @@ -843,11 +858,11 @@ impl Worker { .phase .send_replace(Phase::Known(expected.clone())); self.check(deadline, cancel)?; - let actual = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(self.pin.reference(),deadline)=>result?,}; + let actual = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=self.docker.inspect_image(self.pin.reference(),deadline.min(self.config.deadline()))=>result?,}; if actual.id.as_deref() != Some(expected.as_str()) { return Err(DockerError::Quarantined); } - let size = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.unpacked_size(&image,deadline)=>result?,}; + let size = tokio::select! {biased;()=stopped(&mut stop)=>return Err(DockerError::Stopped),_ = cancel.changed()=>return Err(DockerError::Cancelled),result=resolver.unpacked_size(&image,deadline.min(self.config.deadline()))=>result?,}; self.check(deadline, cancel)?; Ok((image, size)) } @@ -904,4 +919,4 @@ impl Worker { #[cfg(test)] #[path = "lease_tests.rs"] -mod tests; +pub(crate) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/lease_tests.rs b/crates/rustack-lambda-core/src/docker/lease_tests.rs index 3d0479b6..a26d9ea3 100644 --- a/crates/rustack-lambda-core/src/docker/lease_tests.rs +++ b/crates/rustack-lambda-core/src/docker/lease_tests.rs @@ -52,15 +52,15 @@ fn manifests(parent: &str, zero: bool) -> Value { } #[derive(Debug, Default)] -struct Stats { - tags: AtomicUsize, - deletes: AtomicUsize, - absent: AtomicUsize, - aliases: AtomicUsize, +pub(crate) struct Stats { + pub(crate) tags: AtomicUsize, + pub(crate) deletes: AtomicUsize, + pub(crate) absent: AtomicUsize, + pub(crate) aliases: AtomicUsize, changed: Notify, } impl Stats { - async fn until(&self, condition: impl Fn() -> bool) { + pub(crate) async fn until(&self, condition: impl Fn() -> bool) { timeout(Duration::from_secs(10), async { loop { let changed = self.changed.notified(); @@ -78,7 +78,7 @@ impl Stats { } } #[derive(Debug, Clone, Copy)] -enum Mode { +pub(crate) enum Mode { Normal, Reject400, Reject404, @@ -92,7 +92,7 @@ enum Mode { LostDelete, HoldDelete, } -enum Command { +pub(crate) enum Command { Retarget, Repair, Commit, @@ -276,15 +276,15 @@ impl Script { } } -struct FakePins { - wire: DockerWire, +pub(crate) struct FakePins { + pub(crate) wire: DockerWire, stop: watch::Sender, - commands: mpsc::Sender, + pub(crate) commands: mpsc::Sender, task: Option>, - stats: Arc, + pub(crate) stats: Arc, } impl FakePins { - async fn new(mode: Mode) -> Self { + pub(crate) async fn new(mode: Mode) -> Self { let daemon = Daemon::new(); let wire = daemon.wire().await; let (stop, stopping) = watch::channel(false); @@ -305,10 +305,10 @@ impl FakePins { stats, } } - fn owner(&self, pins: u16, jobs: u8) -> ImageLeaseManager { + pub(crate) fn owner(&self, pins: u16, jobs: u8) -> ImageLeaseManager { ImageLeaseManager::start(self.wire.port(), scope(), config(pins, jobs)).unwrap() } - async fn close(mut self) { + pub(crate) async fn close(mut self) { self.wire.shutdown().await.unwrap(); self.stop.send_replace(true); self.task.take().unwrap().await.unwrap(); diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index c2c055ca..91d61a89 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -891,3 +891,6 @@ fn encode(value: &impl Serialize) -> Result { #[cfg(test)] mod tests; + +#[cfg(test)] +pub(crate) use lease::tests as image_fixture; diff --git a/crates/rustack-lambda-core/src/error.rs b/crates/rustack-lambda-core/src/error.rs index 3a5637b7..dee3dc69 100644 --- a/crates/rustack-lambda-core/src/error.rs +++ b/crates/rustack-lambda-core/src/error.rs @@ -50,6 +50,10 @@ pub enum LambdaServiceError { message: String, }, + /// An explicit optimistic-update precondition did not match. + #[error("The supplied precondition does not match the current function revision")] + PreconditionFailed, + /// Invalid parameter value. #[error("Invalid parameter: {message}")] InvalidParameter { @@ -152,6 +156,10 @@ impl From for LambdaError { LambdaServiceError::ResourceConflict { ref message } => { LambdaError::resource_conflict(message) } + LambdaServiceError::PreconditionFailed => LambdaError::new( + LambdaErrorCode::PreconditionFailedException, + "The supplied precondition does not match the current function revision", + ), LambdaServiceError::InvalidParameter { ref message } => { LambdaError::invalid_parameter(message) } @@ -236,3 +244,18 @@ mod tests { ); } } + +#[cfg(test)] +mod precondition_tests { + use super::*; + + #[test] + fn test_should_encode_stale_revision_as_http_412_with_the_aws_error_code() { + let error: LambdaError = LambdaServiceError::PreconditionFailed.into(); + assert_eq!(error.code.as_str(), "PreconditionFailedException"); + assert_eq!( + error.code.status_code(), + http::StatusCode::PRECONDITION_FAILED + ); + } +} diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index c401fc8e..2235f837 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -38,6 +38,24 @@ use crate::invocation::InvocationContext; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. #[async_trait] pub trait Executor: std::fmt::Debug + Send + Sync + 'static { + /// Verify image execution readiness before admitting asynchronous deployment. + /// + /// Backends must explicitly opt in after their Docker runtime probes succeed. + /// + /// # Errors + /// The default rejects image deployment for unsupported or disabled backends. + /// + /// ``` + /// # fn check(executor: &dyn rustack_lambda_core::executor::Executor) { + /// let ready = executor.image_available().is_ok(); + /// # } + /// ``` + fn image_available(&self) -> Result<(), ExecutorError> { + Err(ExecutorError::Unsupported( + "Image execution is unavailable".into(), + )) + } + /// Check whether execution is enabled before accepting asynchronous work. /// /// # Errors diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 66a27618..028784cd 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -53,14 +53,13 @@ use rustack_lambda_model::{ TracingConfigResponse, VpcConfigResponse, }, }; +use tokio::time::{Instant, timeout_at}; -#[cfg(test)] -use crate::artifact::Architecture; use crate::{ - artifact::{DeploymentArtifact, ImageArtifact, ImageReference}, + artifact::{Architecture, DeploymentArtifact, ImageArtifact, ImageReference}, code::{S3CodeFetchError, S3CodeFetcher, UnavailableS3CodeFetcher}, config::LambdaConfig, - deployment::DeploymentState, + deployment::{DeploymentState, check_revision}, error::LambdaServiceError, executor::{ AutoExecutor, ExecutionRevision, Executor, ExecutorBackend, InvokeRequest, NativeExecutor, @@ -79,6 +78,11 @@ use crate::{ PolicyDocument, PolicyStatement, StagedArtifact, VersionRecord, compute_sha256, }, }; +#[cfg(unix)] +use crate::{ + deployment::{ImageDeploymentConfig, ImageDeploymentManager}, + docker::ImageLeasePort, +}; /// Selects the AWS `Invoke` invocation type. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -135,13 +139,15 @@ pub enum InvokeOutcome { /// All operations are implemented as methods that return domain types or errors. #[derive(Debug)] pub struct RustackLambda { - store: FunctionStore, + store: Arc, layer_store: LayerStore, esm_store: EventSourceMappingStore, config: LambdaConfig, executor: Arc, code_fetcher: Arc, work: crate::work::WorkManager, + #[cfg(unix)] + deployments: Option, } /// Serializable Lambda provider snapshot. @@ -194,7 +200,7 @@ impl ProcessedCode<'_> { ) } Self::Image(reference) => ( - compute_sha256(reference.as_str().as_bytes()), + String::new(), 0, None, None, @@ -218,13 +224,15 @@ impl RustackLambda { pub fn with_store(store: FunctionStore, config: LambdaConfig) -> Self { let executor = build_executor(&config); Self { - store, + store: Arc::new(store), layer_store: LayerStore::new(), esm_store: EventSourceMappingStore::new(), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -235,13 +243,15 @@ impl RustackLambda { let store = FunctionStore::new(code_dir); let executor = build_executor(&config); Self { - store, + store: Arc::new(store), layer_store: LayerStore::new(), esm_store: EventSourceMappingStore::new(), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -254,13 +264,15 @@ impl RustackLambda { executor: Arc, ) -> Self { Self { - store, + store: Arc::new(store), layer_store: LayerStore::new(), esm_store: EventSourceMappingStore::new(), config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), work: crate::work::WorkManager::new(), + #[cfg(unix)] + deployments: None, } } @@ -274,6 +286,89 @@ impl RustackLambda { self } + /// Attach qualified image preparation after executor readiness probes succeed. + /// + /// # Errors + /// Rejects non-Docker/unavailable executors, duplicate attachment or invalid owner + /// configuration. The runtime must retain the image owner until all deployment/execution + /// consumers settle. + /// + /// ```no_run + /// # fn attach(provider:rustack_lambda_core::provider::RustackLambda, port:rustack_lambda_core::docker::ImageLeasePort) -> Result<(),rustack_lambda_core::error::LambdaServiceError> { + /// let provider = provider.with_image_preparation(port, rustack_lambda_core::deployment::ImageDeploymentConfig::default())?; + /// # Ok(()) } + /// ``` + #[cfg(unix)] + pub fn with_image_preparation( + mut self, + port: ImageLeasePort, + config: ImageDeploymentConfig, + ) -> Result { + if self.config.executor != ExecutorBackend::Docker + || self.executor.image_available().is_err() + || self.deployments.is_some() + { + return Err(LambdaServiceError::DockerNotAvailable); + } + self.deployments = Some(ImageDeploymentManager::start( + self.store.clone(), + port, + config, + )?); + Ok(self) + } + + #[cfg(unix)] + fn image_deployments(&self) -> Result<&ImageDeploymentManager, LambdaServiceError> { + if self.config.executor != ExecutorBackend::Docker + || self.executor.image_available().is_err() + { + return Err(LambdaServiceError::DockerNotAvailable); + } + self.deployments + .as_ref() + .ok_or(LambdaServiceError::DockerNotAvailable) + } + + fn create_image_deployment( + &self, + record: FunctionRecord, + ) -> Result { + #[cfg(unix)] + { + self.image_deployments()?.create(record) + } + #[cfg(not(unix))] + { + drop(record); + Err(LambdaServiceError::DockerNotAvailable) + } + } + + fn update_image_deployment( + &self, + before: &FunctionRecord, + reference: ImageReference, + architecture: Architecture, + requested: Option<&str>, + ) -> Result { + #[cfg(unix)] + { + let accepted = self.image_deployments()?.update_with_revision( + before, + reference, + architecture, + requested, + )?; + Ok(self.build_function_configuration(&accepted, &accepted.latest)) + } + #[cfg(not(unix))] + { + let _ = (before, reference, architecture, requested); + Err(LambdaServiceError::DockerNotAvailable) + } + } + /// Borrow the executor (e.g. for tests asserting backend behavior). #[must_use] pub fn executor(&self) -> &Arc { @@ -283,11 +378,25 @@ impl RustackLambda { /// Stop all warm executor instances. Wired into the rustack server's /// graceful shutdown path. pub async fn shutdown(&self) { + let deadline = Instant::now() + Duration::from_secs(10); + #[cfg(unix)] + if let Some(owner) = &self.deployments { + if owner.shutdown(deadline).await.is_err() { + tracing::error!("Lambda deployment shutdown did not settle"); + } + } self.work.cancel_all(); - if let Err(error) = self.work.quiesce(Duration::from_secs(10)).await { + if let Err(error) = self + .work + .quiesce(deadline.saturating_duration_since(Instant::now())) + .await + { tracing::error!(%error, "Lambda shutdown cancelled unfinished work"); } - if self.executor.shutdown().await.is_err() { + if !matches!( + timeout_at(deadline, self.executor.shutdown()).await, + Ok(Ok(())) + ) { tracing::error!("Lambda executor shutdown did not confirm cleanup"); } } @@ -297,17 +406,26 @@ impl RustackLambda { /// # Errors /// Returns an error after cancelling remaining tasks when the deadline expires. pub async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { - let deadline = tokio::time::Instant::now() + let deadline = Instant::now() .checked_add(timeout) .ok_or_else(|| invalid_parameter("Invalid Lambda quiesce budget"))?; - let cleanup_budget = (timeout / 10).min(Duration::from_secs(1)); + #[cfg(unix)] + if let Some(owner) = &self.deployments { + if let Err(error) = owner.quiesce(deadline).await { + owner.cancel(); + self.work.cancel_all(); + return Err(error.into()); + } + } + let remaining = deadline.saturating_duration_since(Instant::now()); + let cleanup_budget = (remaining / 10).min(Duration::from_secs(1)); let result = self .work - .quiesce(timeout.saturating_sub(cleanup_budget)) + .quiesce(remaining.saturating_sub(cleanup_budget)) .await; if result.is_err() && !matches!( - tokio::time::timeout_at(deadline, self.executor.shutdown()).await, + timeout_at(deadline, self.executor.shutdown()).await, Ok(Ok(())) ) { @@ -574,6 +692,10 @@ impl RustackLambda { created_at: now, }; + if package_type == "Image" { + let accepted = self.create_image_deployment(record)?; + return Ok(self.build_function_configuration(&accepted, &accepted.latest)); + } let should_publish = input.publish.unwrap_or(false); self.store.insert(record)?; processed.commit(); @@ -691,6 +813,7 @@ impl RustackLambda { let (name, _) = resolve_function_ref(function_ref)?; let should_publish = input.publish.unwrap_or(false); let before = self.get_record(&name)?; + check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; before.latest.deployment.ensure_mutable()?; validate_image_publication(&before.latest.package_type, should_publish)?; if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) @@ -709,42 +832,63 @@ impl RustackLambda { )); } + if before.latest.package_type == "Image" { + let reference = code_source + .image_uri + .ok_or_else(|| invalid_parameter("ImageUri is required"))? + .parse() + .map_err(|_| invalid_parameter("Invalid ImageUri"))?; + let architecture = match input + .architectures + .as_ref() + .and_then(|values| values.first()) + { + Some(value) => value + .parse() + .map_err(|_| invalid_parameter("Invalid architecture"))?, + None => crate::function_config::validate_execution_settings(&before.latest)?, + }; + return self.update_image_deployment( + &before, + reference, + architecture, + input.revision_id.as_deref(), + ); + } let processed = self.process_code(&name, "$LATEST", code_source).await?; let (code_sha256, code_size, zip_bytes, code_path, image_uri) = processed.parts(); - let config = self.store.try_update_if_revision( - &name, - Some(&before.latest.revision_id), - |record| { - record.latest.deployment.ensure_mutable()?; - let now = now_iso8601(); - record.latest.code_sha256 = code_sha256; - record.latest.code_size = code_size; - record.latest.zip_bytes = zip_bytes; - record.latest.code_path = code_path; - record.latest.image_uri = image_uri; - record.latest.image = ImageArtifact::Unresolved; - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - - if let Some(archs) = input.architectures.clone() { - record.latest.architectures = archs; - } - record.latest.deployment = DeploymentState::Ready; - let configuration = if should_publish { - let number = record.next_version; - record.next_version = number.saturating_add(1); - let mut published = record.latest.clone(); - published.version = number.to_string(); - let configuration = self.build_function_configuration(record, &published); - record.versions.insert(number, published); - configuration - } else { - self.build_function_configuration(record, &record.latest) - }; - Ok(configuration) - }, - )?; + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + check_staged_revision(record, &before)?; + record.latest.deployment.ensure_mutable()?; + let now = now_iso8601(); + record.latest.code_sha256 = code_sha256; + record.latest.code_size = code_size; + record.latest.zip_bytes = zip_bytes; + record.latest.code_path = code_path; + record.latest.image_uri = image_uri; + record.latest.image = ImageArtifact::Unresolved; + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + + if let Some(archs) = input.architectures.clone() { + record.latest.architectures = archs; + } + record.latest.deployment = DeploymentState::Ready; + let configuration = if should_publish { + let number = record.next_version; + record.next_version = number.saturating_add(1); + let mut published = record.latest.clone(); + published.version = number.to_string(); + let configuration = self.build_function_configuration(record, &published); + record.versions.insert(number, published); + configuration + } else { + self.build_function_configuration(record, &record.latest) + }; + Ok(configuration) + })?; processed.commit(); info!(function_name = %name, "updated Lambda function code"); @@ -762,77 +906,76 @@ impl RustackLambda { validate_configuration_update(input)?; let before = self.get_record(&name)?; - let config = self.store.try_update_if_revision( - &name, - Some(&before.latest.revision_id), - |record| { - record.latest.deployment.ensure_mutable()?; - let now = now_iso8601(); - - if let Some(runtime) = &input.runtime { - record.latest.runtime = Some(runtime.clone()); - } - if let Some(role) = &input.role { - record.latest.role.clone_from(role); - } - if let Some(handler) = &input.handler { - record.latest.handler = Some(handler.clone()); - } - if let Some(description) = &input.description { - record.latest.description.clone_from(description); - } - if let Some(timeout) = input.timeout { - record.latest.timeout = timeout; - } - if let Some(memory_size) = input.memory_size { - record.latest.memory_size = memory_size; - } - if let Some(env) = &input.environment { - record.latest.environment = env.variables.clone().unwrap_or_default(); - } - if let Some(layers) = &input.layers { - record.latest.layers.clone_from(layers); - } - if let Some(ephemeral) = &input.ephemeral_storage { - record.latest.ephemeral_storage_size = ephemeral.size; - } - if let Some(vpc) = &input.vpc_config { - record.latest.vpc_config = Some(vpc.clone()); - } - if let Some(dlc) = &input.dead_letter_config { - record.latest.dead_letter_config = Some(dlc.clone()); - } - if let Some(tc) = &input.tracing_config { - record.latest.tracing_config = Some(tc.clone()); - } - if let Some(ic) = &input.image_config { - record.latest.image_config = Some(ic.clone()); - } - if let Some(lc) = &input.logging_config { - record.latest.logging_config = Some(lc.clone()); - } - if let Some(ss) = &input.snap_start { - record.latest.snap_start = Some(ss.clone()); - } + check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + check_staged_revision(record, &before)?; + record.latest.deployment.ensure_mutable()?; + let now = now_iso8601(); + + if let Some(runtime) = &input.runtime { + record.latest.runtime = Some(runtime.clone()); + } + if let Some(role) = &input.role { + record.latest.role.clone_from(role); + } + if let Some(handler) = &input.handler { + record.latest.handler = Some(handler.clone()); + } + if let Some(description) = &input.description { + record.latest.description.clone_from(description); + } + if let Some(timeout) = input.timeout { + record.latest.timeout = timeout; + } + if let Some(memory_size) = input.memory_size { + record.latest.memory_size = memory_size; + } + if let Some(env) = &input.environment { + record.latest.environment = env.variables.clone().unwrap_or_default(); + } + if let Some(layers) = &input.layers { + record.latest.layers.clone_from(layers); + } + if let Some(ephemeral) = &input.ephemeral_storage { + record.latest.ephemeral_storage_size = ephemeral.size; + } + if let Some(vpc) = &input.vpc_config { + record.latest.vpc_config = Some(vpc.clone()); + } + if let Some(dlc) = &input.dead_letter_config { + record.latest.dead_letter_config = Some(dlc.clone()); + } + if let Some(tc) = &input.tracing_config { + record.latest.tracing_config = Some(tc.clone()); + } + if let Some(ic) = &input.image_config { + record.latest.image_config = Some(ic.clone()); + } + if let Some(lc) = &input.logging_config { + record.latest.logging_config = Some(lc.clone()); + } + if let Some(ss) = &input.snap_start { + record.latest.snap_start = Some(ss.clone()); + } - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - if matches!(record.latest.deployment, DeploymentState::UpdateFailed(_)) { - record.latest.deployment = DeploymentState::Ready; - } - if let Some(image) = record.latest.image.resolved() { - image - .metadata() - .execution - .plan(record.latest.image_config.as_ref()) - .map_err(|_| { - invalid_parameter("Invalid effective image entry configuration") - })?; - } - Ok(self.build_function_configuration(record, &record.latest)) - }, - )?; + if matches!(record.latest.deployment, DeploymentState::UpdateFailed(_)) { + record.latest.deployment = DeploymentState::Ready; + } + if let Some(image) = record.latest.image.resolved() { + image + .metadata() + .execution + .plan(record.latest.image_config.as_ref()) + .map_err(|_| { + invalid_parameter("Invalid effective image entry configuration") + })?; + } + Ok(self.build_function_configuration(record, &record.latest)) + })?; info!(function_name = %name, "updated Lambda function configuration"); Ok(config) @@ -986,7 +1129,7 @@ impl RustackLambda { function_ref: &str, qualifier: Option<&str>, payload: &[u8], - deadline: Option, + deadline: Option, ) -> Result { validate_invoke_payload(payload, MAX_SYNC_PAYLOAD)?; let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; @@ -1031,16 +1174,14 @@ impl RustackLambda { qualifier: Option<&str>, payload: &[u8], asynchronous: bool, - deadline: Option, + deadline: Option, ) -> Result { let version = resolve_version(record, qualifier)?; let capture = self.build_invoke_request(record, version, qualifier, payload); let request = if let Some(deadline) = deadline { - tokio::time::timeout_at(deadline, capture) - .await - .map_err(|_| { - LambdaServiceError::from(crate::executor::ExecutorError::DeadlineExceeded) - })?? + timeout_at(deadline, capture).await.map_err(|_| { + LambdaServiceError::from(crate::executor::ExecutorError::DeadlineExceeded) + })?? } else { capture.await? }; @@ -1165,51 +1306,46 @@ impl RustackLambda { ) -> Result { let (name, _) = resolve_function_ref(function_ref)?; - let config = - self.store - .try_update_if_revision(&name, input.revision_id.as_deref(), |record| { - if record.latest.deployment != DeploymentState::Ready { - return Err(LambdaServiceError::ResourceConflict { - message: "Function deployment must be successful before publication" - .into(), - }); - } - if record.latest.package_type == "Image" - && record - .latest - .image - .pinned() - .is_none_or(|image| !image.is_live()) - { - return Err(LambdaServiceError::ResourceNotReady { - message: "Image availability must be acquired before publication" - .into(), - }); - } - if input - .code_sha256 - .as_ref() - .is_some_and(|hash| hash != &record.latest.code_sha256) - { - return Err(invalid_parameter( - "CodeSha256 does not match the latest revision", - )); - } - let version_num = record.next_version; - record.next_version += 1; + let config = self.store.try_update_if_revision(&name, None, |record| { + check_revision(input.revision_id.as_deref(), &record.latest.revision_id)?; + if record.latest.deployment != DeploymentState::Ready { + return Err(LambdaServiceError::ResourceConflict { + message: "Function deployment must be successful before publication".into(), + }); + } + if record.latest.package_type == "Image" + && record + .latest + .image + .pinned() + .is_none_or(|image| !image.is_live()) + { + return Err(LambdaServiceError::ResourceNotReady { + message: "Image availability must be acquired before publication".into(), + }); + } + if input + .code_sha256 + .as_ref() + .is_some_and(|hash| hash != &record.latest.code_sha256) + { + return Err(LambdaServiceError::PreconditionFailed); + } + let version_num = record.next_version; + record.next_version += 1; - let mut published = record.latest.clone(); - published.version = version_num.to_string(); - if let Some(desc) = &input.description { - published.description.clone_from(desc); - } - published.revision_id = uuid::Uuid::new_v4().to_string(); - published.last_modified = now_iso8601(); + let mut published = record.latest.clone(); + published.version = version_num.to_string(); + if let Some(desc) = &input.description { + published.description.clone_from(desc); + } + published.revision_id = uuid::Uuid::new_v4().to_string(); + published.last_modified = now_iso8601(); - let config = self.build_function_configuration(record, &published); - record.versions.insert(version_num, published); - Ok(config) - })?; + let config = self.build_function_configuration(record, &published); + record.versions.insert(version_num, published); + Ok(config) + })?; info!(function_name = %name, "published Lambda function version"); Ok(config) @@ -3036,6 +3172,20 @@ impl RustackLambda { } /// Reject publication before asynchronous image preparation can begin. +fn check_staged_revision( + record: &FunctionRecord, + before: &FunctionRecord, +) -> Result<(), LambdaServiceError> { + if record.resource_id != before.resource_id + || record.latest.revision_id != before.latest.revision_id + { + return Err(LambdaServiceError::ResourceConflict { + message: "Function changed while deployment was staged".into(), + }); + } + Ok(()) +} + fn validate_image_publication(package_type: &str, publish: bool) -> Result<(), LambdaServiceError> { if package_type == "Image" && publish { return Err(invalid_parameter( @@ -3296,7 +3446,7 @@ mod tests { let provider = RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) .with_code_fetcher(fetcher); - let started = tokio::time::Instant::now(); + let started = Instant::now(); assert!(matches!( provider .create_function(s3_code_input("fetch-timeout")) diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index 95c36a80..0d925f5d 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -20,6 +20,7 @@ use rustack_lambda_model::types::{ }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use tokio::sync::watch; use crate::{ artifact::{ImageArtifact, ResolvedImage, ValidatedArtifact}, @@ -38,6 +39,7 @@ pub struct FunctionStore { /// Own completed artifacts until the store is dropped; in-flight revisions remain readable. artifacts: DashMap, staging: std::sync::Arc, + changes: watch::Sender<()>, } /// Rolls back a newly registered artifact unless its record mutation commits. @@ -956,9 +958,15 @@ impl FunctionStore { code_dir: code_dir.into(), artifacts: DashMap::new(), staging: std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + changes: watch::channel(()).0, } } + #[cfg(unix)] + pub(crate) fn changes(&self) -> watch::Receiver<()> { + self.changes.subscribe() + } + /// Returns a reference to the code directory path. #[must_use] pub fn code_dir(&self) -> &Path { @@ -979,6 +987,7 @@ impl FunctionStore { }), Entry::Vacant(entry) => { entry.insert(record); + self.changes.send_replace(()); Ok(()) } } @@ -1066,6 +1075,7 @@ impl FunctionStore { } self.validate_record(&candidate)?; *entry = candidate; + self.changes.send_replace(()); Ok(result) } None => Err(LambdaServiceError::FunctionNotFound { @@ -1079,7 +1089,11 @@ impl FunctionStore { /// Returns the removed record, or `None` if it did not exist. pub fn remove(&self, name: &str) -> Result, LambdaServiceError> { crate::resolver::FunctionName::parse(name)?; - Ok(self.functions.remove(name).map(|(_, v)| v)) + let removed = self.functions.remove(name).map(|(_, v)| v); + if removed.is_some() { + self.changes.send_replace(()); + } + Ok(removed) } /// List all function records. @@ -1163,6 +1177,7 @@ impl FunctionStore { for artifact in artifacts { artifact.commit(); } + self.changes.send_replace(()); Ok(()) } diff --git a/crates/rustack-lambda-model/src/error.rs b/crates/rustack-lambda-model/src/error.rs index 00768287..52e2f4e0 100644 --- a/crates/rustack-lambda-model/src/error.rs +++ b/crates/rustack-lambda-model/src/error.rs @@ -14,6 +14,8 @@ pub enum LambdaErrorCode { ResourceNotFoundException, /// Resource already exists. ResourceConflictException, + /// An optimistic update precondition did not match. + PreconditionFailedException, /// Request payload too large. RequestTooLargeException, /// Unsupported media type. @@ -39,6 +41,7 @@ impl LambdaErrorCode { Self::InvalidRequestContentException => "InvalidRequestContentException", Self::ResourceNotFoundException => "ResourceNotFoundException", Self::ResourceConflictException => "ResourceConflictException", + Self::PreconditionFailedException => "PreconditionFailedException", Self::RequestTooLargeException => "RequestTooLargeException", Self::UnsupportedMediaTypeException => "UnsupportedMediaTypeException", Self::InvalidRuntimeException => "InvalidRuntimeException", @@ -57,6 +60,7 @@ impl LambdaErrorCode { } Self::ResourceNotFoundException => http::StatusCode::NOT_FOUND, Self::ResourceConflictException => http::StatusCode::CONFLICT, + Self::PreconditionFailedException => http::StatusCode::PRECONDITION_FAILED, Self::RequestTooLargeException => http::StatusCode::PAYLOAD_TOO_LARGE, Self::UnsupportedMediaTypeException => http::StatusCode::UNSUPPORTED_MEDIA_TYPE, Self::InvalidRuntimeException | Self::ResourceNotReadyException => { diff --git a/crates/rustack-lambda-model/src/input.rs b/crates/rustack-lambda-model/src/input.rs index 248928f2..2cb68566 100644 --- a/crates/rustack-lambda-model/src/input.rs +++ b/crates/rustack-lambda-model/src/input.rs @@ -83,6 +83,9 @@ pub struct CreateFunctionInput { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct UpdateFunctionCodeInput { + /// Revision ID precondition for an optimistic update. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, /// Base64-encoded zip file. #[serde(skip_serializing_if = "Option::is_none")] pub zip_file: Option, @@ -113,6 +116,9 @@ pub struct UpdateFunctionCodeInput { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct UpdateFunctionConfigurationInput { + /// Revision ID precondition for an optimistic update. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, /// Runtime. #[serde(skip_serializing_if = "Option::is_none")] pub runtime: Option, @@ -424,3 +430,22 @@ pub struct EventInvokeConfigInput { #[serde(skip_serializing_if = "Option::is_none")] pub destination_config: Option, } + +#[cfg(test)] +mod revision_tests { + use super::{UpdateFunctionCodeInput, UpdateFunctionConfigurationInput}; + + #[test] + fn test_should_preserve_explicit_revision_preconditions_in_both_update_wire_shapes() { + let json = r#"{"RevisionId":"expected-revision"}"#; + let code: UpdateFunctionCodeInput = serde_json::from_str(json).unwrap(); + let configuration: UpdateFunctionConfigurationInput = serde_json::from_str(json).unwrap(); + assert_eq!(code.revision_id.as_deref(), Some("expected-revision")); + assert_eq!( + configuration.revision_id.as_deref(), + Some("expected-revision") + ); + assert_eq!(serde_json::to_string(&code).unwrap(), json); + assert_eq!(serde_json::to_string(&configuration).unwrap(), json); + } +} diff --git a/specs/index.md b/specs/index.md index 9bdd8589..d5b5f848 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement and leased invocation capture are implemented; retained provider deployment ownership/API wiring, the Docker executor and the ASL engine remain pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 9eed393d..b9180f36 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -220,6 +220,38 @@ Provider + retained worker FunctionStore ImageLeaseManager Required tests prove initial Pending/success/failure, preservation of old code while updating and after failure, concurrent mutation rejection, stale/duplicate settlement, delete/recreate isolation, live-lease capture/revocation, numeric-version ownership and snapshot migration/contradiction checks. Docker remains unavailable in capability reporting until the retained deployment owner, executor/pool and startup probes are wired and qualified. +### 2.11 Retained deployment supervisor and provider admission + +ImageDeploymentManager owns an Arc, a qualified ImageLeasePort and bounded accepted work (default8 jobs,1..32 and no greater than the preparation port capacity). ImageDeploymentConfig sets one absolute deployment budget (default300s,1..3,600s), starting before mailbox reservation. Startup requires an entered Tokio runtime; reservation and store mutation are synchronous and contain no suspension point. Reserve an accepted-work permit and an owned bounded-mailbox permit before inserting Pending or beginning an update. Return the configuration captured at acceptance, even if preparation completes before the HTTP response is serialized. No prepared-image hash is invented during this response. + +An accepted envelope has a settlement guard, captured candidate, absolute deadline and capacity permit. Failure before enqueue or loss of a queued envelope settles a closed cancellation/supervision failure through the same store key. The actor registers every candidate and its capacity above the worker before spawning it, retains a JoinSet and task-ID index, and releases accepted capacity only after join (or disposal of work never started). A panicked worker closes admission and triggers failed settlement without overwriting a newer revision. Root failure marks health failed even with zero workers, transfers registered rows and aborted child joins to emergency cleanup on the captured runtime, and retains capacity until every join/settlement finishes. Drop outside an entered runtime signals the same cancellation path; the captured runtime must remain alive. Root Drop/recovery cannot leave a permanently Pending record. + +The overall preparation deadline accepts up to3,600s; each read/metadata stage uses min(overall deadline, now + configured independent Docker IO budget), while critical mutation/cleanup retains its separate IO lifetime. Workers call the real ImageLeasePort once and retain that same preparation future across store-change notifications; never restart it or replay a tag. FunctionStore emits a coalescing change notification after mutations/removal/restore. Ready preparation outcomes take priority over unrelated change notifications after deadline/cancellation, preventing notification starvation; the outcome still undergoes the final key check. Workers subscribe before checking their key, cancel when the candidate is deleted, replaced or no longer pending, and recheck deadline/cancellation/key before settlement. Thus deleting/recreating a name promptly cancels its old preparation, while the image owner retains any uncertain critical mutation and pin cleanup. Closed failure mappings exclude peer text:404→ImageNotFound, invalid settings/inspection/framing/limits→InvalidImage, deadline→DeadlineExceeded, cancellation→Cancelled, and unavailable/busy/quarantined/failed backend→RuntimeUnavailable or SupervisorFailed as appropriate. + +Manager quiesce closes admission and drains queued/running accepted work; shutdown or Drop additionally cancels it. Caller deadlines bound waiters rather than abort retained work. Cancelled waiters retain root/emergency joins and cached results. Successful manager settlement proves its deployment workers joined, not that the separate image owner finished uncertain mutations. The runtime must subsequently join invocation/environment consumers and then the image owner before global cleanup/snapshot consistency can be declared. Root or worker failure remains a cached failure after its failed records are settled. Normal deployment failures produce inspectable Failed/update-failure records without poisoning unrelated admission. + +The provider stores FunctionStore in Arc for retained worker ownership. Its explicit image-runtime attachment requires executor=docker and a backend image-availability check, which defaults to unavailable for Disabled/Auto/Native/Squib and unsupported Docker builds. Create(Image) without this qualified attachment rejects before metadata insertion; it never returns falsely Active. Create/Update(Publish=true) still reject before mutation. Image Create uses manager admission and initial Pending insertion; UpdateFunctionCode/Configuration deserialize RevisionId and check a bounded1..100-byte ASCII token before work and again under the mutation lock. An explicit stale precondition returns PreconditionFailedException/412; an in-progress operation or implicit staged-resource conflict remains ResourceConflictException/409. PublishVersion applies the same revision and CodeSha256 precondition category. These match the [official code API](https://docs.aws.amazon.com/lambda/latest/api/API_UpdateFunctionCode.html) and [configuration API](https://docs.aws.amazon.com/lambda/latest/api/API_UpdateFunctionConfiguration.html); the token bound/ASCII grammar is an advertised local restriction. Image Update validates package/settings/revision, preserves old latest while preparing and returns its acceptance snapshot. ZIP code staging remains synchronous and transactional. Provider shutdown cancels/joins deployments before shutting invocation/executor consumers; quiescence drains deployments and invocation work under one caller deadline. Configuration updates and publication retain the store guards from §2.10. The actual Docker backend/probes remain required before the production server can attach this owner or advertise Docker. + +Required tests use the real bounded wire/image-preparation actors with a scripted local daemon, plus a declared test executor capability for provider-only control-plane qualification. Cover capacity rejection before resource mutation, accepted response abandonment, initial/update success/failure, deadline and shutdown/cancelled waiters, deletion during delayed tag, stale result cleanup, normal failure recovery, worker/root panic including zero-row/queued work and off-runtime Drop, cached failure and provider Pending/Active/status/CAS. A fake executor cannot establish actual Invoke acceptance; the official-RIC Docker executor lane remains a separate P1 gate. + +```text +┌─ Provider + qualified executor / deployment manager ───────────────────┐ +│ validate → reserve slot + mailbox → store CAS → captured response │ +│ │ accepted settlement guard │ +└────────────────────────────────────┼──────────────────────────────────┘ + ▼ bounded envelope +┌─ Retained actor ──────────────────────────────────────────────────────┐ +│ registry + permits → JoinSet workers → keyed settlement → joined reap │ +│ │ │ │ +│ │ └─ store change / stop / deadline cancels│ +│ └─ root failure → captured-runtime emergency join + settlement│ +└────────────────────────────────────┼──────────────────────────────────┘ + ▼ one retained preparation future +┌─ ImageLeaseManager ───────────────────────────────────────────────────┐ +│ owns critical Docker mutations and pin cleanup beyond worker cancel │ +└───────────────────────────────────────────────────────────────────────┘ +``` + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 586bb588..3c2c06f2 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -69,7 +69,8 @@ All budgets are validated once; limits below are local defaults and safe configu | workflowCallbackTokens / workflowCallbackTombstones |4,096 /100,000|1..65,536 /1..1,000,000; closed-token TTL follows execution retention | | lambdaDockerEnvironments |32|1..32 initially, including busy/idle/quarantined environments | | lambdaDockerMemoryBytes |8 GiB|256 MiB..1 TiB; includes relay caps and all allocated environments | -| lambdaDockerDeploymentJobs |8|1..32; queued plus running deployments;≤image pin budget | +| lambdaDockerDeploymentJobs |8|1..32; queued plus running deployments;≤image preparation capacity and pin budget | +| lambdaDockerDeploymentSeconds |300|1..3,600; absolute queue plus preparation deadline | | lambdaDockerImagePins |1024|1..8,192; preparations, live revisions/captures and quarantined aliases | | lambdaDockerIoSeconds |5|1..60; independent bounded management/mutation and partial-control IO budget | | lambdaDockerPullSeconds / lambdaDockerStartSeconds |120 /15|1..600 /1..60 | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 13ac2382..f3637bae 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -86,6 +86,10 @@ Owned image preparation now extends the same real wire suite: acquire private al Deployment settlement foundation (§2.10) must independently prove resource/revision/generation CAS, simultaneous duplicate completion, preservation of old code through updates/failure, pending mutation/publication rejection, static public reasons, fallible transaction rollback, and real lease capture/revocation. Tampered resolved snapshot hash/size must reject atomically; saved pending work must restore as failure with metadata-only images. A private-builder compile-fail doctest prevents arbitrary external candidate construction. These are store/capture proofs; they do not close L01/provider admission, retained deployment supervision or Docker executor acceptance. +Retained deployment ownership (§2.11) now has19 additional actor/provider cases plus one core precondition-error case and one model wire-field case. They use the actual bounded wire/image owners with a scripted local daemon and an explicitly declared test executor, covering accepted capacity, dropped responses/waiters, deletion, deadlines, root/worker/off-runtime failure, atomic receipts and stale revisions. These prove provider control-plane preparation, not actual handler invocation through Docker. A3,600s overall-budget case proves per-read IO separation; a simultaneously-ready notification/result test proves scheduling progress. + +`make test-lambda-invoke-native` exercises real Native functions directly through the provider; it bypasses AWS SDK/HTTP. Historical references to it as a Native SDK target are corrected to Native provider evidence. `make test-lambda-revision-wire SMOKE_ENDPOINT=` separately runs an actual Rust SDK through HTTP: valid configuration/code updates/publication require explicit nonempty advancing revision IDs; stale requests decode as modeled PreconditionFailedException with raw412, and an immediate read proves revision/hash/size/description are unchanged before any successful update. This test is included in the explicit service smoke manifest. It does not establish boto3 or Image handler execution acceptance. The changed image read deadlines are additionally qualified by both two-case real Docker wire suites on Engine29.5.2 arm64 VZ and amd64/QEMU; independent queries find no private image aliases or owned containers. Image metadata reads use the configured5s budget, while the wire harness retains60s stream IO/120s attempt stress allowances; full executor default latency and native x86/Desktop qualification remain open. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index be2763b5..5fc6c997 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -140,9 +140,9 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native SDK/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | +| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native SDK/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained provider deployment ownership/API wiring, directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass; production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next implement the retained deployment supervisor and provider Image Create/Update API wiring above the typed store settlement and ImageLeaseManager, then task1.3 environment/pool actors consuming the qualified DockerWire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe operations, followed by task1.4 acceptance. B01/B03 remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next fix B04 side-effect-free code-update DryRun and bounded request validation before enabling real execution, then implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03 and B04 request-validation behavior remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 9647a67a..78c07386 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -53,7 +53,7 @@ The independent read-only review found three lifecycle/admission defects. All we | W14 | P2; `crates/rustack-lambda-core/src/executor/instance.rs:276` | A full/disabled warm pool dropped a successful instance before reaping. Release now returns rejected ownership; Invoke awaits API shutdown and backend termination before settlement | Two overlapping real Native calls with warm retention0/1 require exactly0/1 surviving child after both responses | | W15 | P2; `crates/rustack-lambda-core/src/executor/native.rs:129` | Known host-architecture mismatch failed only after Event acceptance. Availability now checks architecture before work admission | Real fixture Event rejects the mismatch, then quiesces cleanly without starting the foreign bootstrap | -ExecutionRevision uses shared Arc ownership; InvocationHandle and the bounded work actor preserve UUID/deadline/cancellation through backend cleanup. Waiter cancellation never aborts backend workers. Native child supervisors retain artifact leases and process reservations through reaping, including executor Drop. Tests cover late-result rejection, cleanup timeout/aborted waiters, worker panic,128 accepted/32 running/default8 function limits and permit release after cleanup. The workspace suite passes1,767 tests (392 preexisting ignored), including215 Lambda core tests and five real Native fixture tests. Lambda doctests, standard workspace build/Clippy/rustdoc and the native SDK target pass. Its separately gated Squib test remains excluded as VM evidence. Full Docker and workflow milestones, plus B01/B03, remain required. +ExecutionRevision uses shared Arc ownership; InvocationHandle and the bounded work actor preserve UUID/deadline/cancellation through backend cleanup. Waiter cancellation never aborts backend workers. Native child supervisors retain artifact leases and process reservations through reaping, including executor Drop. Tests cover late-result rejection, cleanup timeout/aborted waiters, worker panic,128 accepted/32 running/default8 function limits and permit release after cleanup. The workspace suite passes1,767 tests (392 preexisting ignored), including215 Lambda core tests and five real Native fixture tests. Lambda doctests, standard workspace build/Clippy/rustdoc and the native provider target pass. Its separately gated Squib test remains excluded as VM evidence. Full Docker and workflow milestones, plus B01/B03, remain required. ## P1 task1.2 protocol slice review @@ -87,7 +87,7 @@ The independent read-only review found three boundary/lifecycle defects. All wer | W21 | P2; `crates/rustack-lambda-runtime/src/http.rs:456` | Collected trailers bypassed per-header validation. All trailers now reject before session mutation | Short, oversized and Invocation-Id trailers return400 without initialization notification or session closure | | W22 | P2; `crates/rustack-lambda-core/src/executor/instance.rs:309` | Revision mismatch cleared old instances through Drop, losing API join ownership. Acquisition now extracts stale instances and awaits bounded parallel retirement outside map guards; empty buckets are removed | Real old API holds an incomplete body while code changes; replacement waits its408 deadline/join and old child reap before invoking new code | -The shared adapter replaces the old pending-map implementation while preserving executor module reexports. Native cold start now races first-poll readiness, init error, process exit and caller cancellation under one init deadline, retiring API/process owners on failure. The Native executor's one cleanup task atomically captures reaper/pool/child ownership, survives cancelled/concurrent shutdown waiters and preserves cleanup failures. Eleven real fixture tests cover slow/no polling, init failure, startup cancellation, version isolation, stale connections, busy cancellation and joined shutdown. The existing native SDK target passes six Native/DryRun scenarios; its separately gated Squib selection remains excluded as VM execution evidence. +The shared adapter replaces the old pending-map implementation while preserving executor module reexports. Native cold start now races first-poll readiness, init error, process exit and caller cancellation under one init deadline, retiring API/process owners on failure. The Native executor's one cleanup task atomically captures reaper/pool/child ownership, survives cancelled/concurrent shutdown waiters and preserves cleanup failures. Eleven real fixture tests cover slow/no polling, init failure, startup cancellation, version isolation, stale connections, busy cancellation and joined shutdown. The existing native provider target passes six Native/DryRun scenarios; its separately gated Squib selection remains excluded as VM execution evidence. Workspace build/fmt/Clippy/rustdoc and1,841 tests pass (392 preexisting ignored). Shared runtime:71 tests,34 doctests and strict production-boundary Clippy; Lambda core:212 tests and26 doctests. Three old core HTTP tests moved into the shared adapter's stronger wire suite. Cargo audit still finds the same16 B01 advisories; no new dependency version/package landed, and deny bans/licenses/sources pass. B03 remains open for old core boundaries. The relay executable/client, Docker executor and ASL engine remain required before milestone completion. @@ -151,7 +151,7 @@ The independent read-only review and final recheck found no remaining concrete i |---|---|---|---| | W32 | P2; `crates/rustack-lambda-core/src/docker/mod.rs`, request admission | Long-lived Wait jobs could fill both reserved cleanup slots and prevent Kill/Remove. Dedicated bounded32-job/eight-mailbox wait lane now leaves two slots for inspect/kill/remove/list; six short business slots remain separate | Held-two-waits regression completes Kill before either wait is released; business saturation admits Remove; exported attach survives wire shutdown without retaining a management slot. D20 records the ownership rule | -DockerWire and DockerPort implement bounded Unix management and strict upgraded attach using generated model-only bollard-stubs1.53.1-rc.29.3.1. All15 new fake-daemon/framing tests pass within227 Lambda core tests. The workspace passes1,903 tests (398 explicitly resource-gated), build/fmt/Clippy/rustdoc; the unchanged strict Lambda boundary debt remains B03, with no new Docker-path findings. Runtime contracts pass97 tests and48 doctests plus strict production checks and actual Linux entry-point Clippy. Eleven real Native fixture tests pass; the native SDK target passes its six actual native/DryRun cases, with its separately gated Squib case still excluded as VM evidence. +DockerWire and DockerPort implement bounded Unix management and strict upgraded attach using generated model-only bollard-stubs1.53.1-rc.29.3.1. All15 new fake-daemon/framing tests pass within227 Lambda core tests. The workspace passes1,903 tests (398 explicitly resource-gated), build/fmt/Clippy/rustdoc; the unchanged strict Lambda boundary debt remains B03, with no new Docker-path findings. Runtime contracts pass97 tests and48 doctests plus strict production checks and actual Linux entry-point Clippy. Eleven real Native fixture tests pass; the native provider target passes its six actual native/DryRun cases, with its separately gated Squib case still excluded as VM evidence. The actual production-wire integration passes against official Python RIC images on arm64 VZ and amd64/QEMU, computing42/7 with warm counters1/2, transferring near6MiB requests/responses and surfacing a real FunctionError. Each run independently removes both named containers, joins stream/wire owners and checks its run-label scope; subsequent daemon queries find no owned wire containers. These stress cases use60s IO/120s attempt budgets rather than claiming default5s performance. The separate five-case relay suite remains required regression evidence on each daemon. @@ -172,7 +172,7 @@ Independent design and implementation review found the issues below. All are fix | W37 | P2; `docker/image.rs:108` | Requiring nonnull DriverStatus rejects legitimate classic VFS. Normalize absent/null rows after classic evidence, retaining marker/shape rejection | Missing/null VFS positives and containerd negatives; historical source pinned | | W38 | P2; actual Docker qualification | Platform-qualified child Id is not independently addressable by lookup/create on the retained containerd store. Bind the requested tag once, platform-inspect its captured parent Id, retain child manifest separately and create from immutable parent plus explicit platform | Initial404 reproduced on both daemons; corrected two-case suites pass on each. Fake parent-a/child-b cases prove the second lookup cannot re-resolve a moved tag | -The resolver performs bounded read-only local inspection only. It validates selected Linux image metadata, inherited argv/environment/user/working-directory bounds, effective entry overrides and local/registry URI representation. No registry pull, credential discovery or image build occurs.61 new tests pass within288 Lambda core tests;58 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,964 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. +The resolver performs bounded read-only local inspection only. It validates selected Linux image metadata, inherited argv/environment/user/working-directory bounds, effective entry overrides and local/registry URI representation. No registry pull, credential discovery or image build occurs.61 new tests pass within288 Lambda core tests;58 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,964 tests and399 explicitly resource-gated cases. Native provider/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. Actual official-RIC wire execution now consumes resolver identities and materialized plans. Near6MiB computed warm calls/errors and a separate clear-command/reset-entrypoint create test pass on both arm64 VZ and amd64/QEMU, with independent stream/container cleanup. Default IO latency, old-daemon execution, native x86/Desktop/rootless and full provider/executor acceptance remain outside this evidence. Strict production Clippy has the same65 existing B03 findings, with none in the new resolver/defaults paths. Hex reuses an existing workspace version/package; audit still reports the16 B01 advisories and deny bans/licenses/sources pass without new ignores. @@ -191,7 +191,7 @@ The independent read-only design and implementation review verified typed privat ImageLeaseManager now owns accepted preparations, live aliases and uncertain cleanup through retained workers/registry reservations. PinnedImage cannot be deserialized from metadata; clones keep the release channel alive, and shutdown revokes further use after consumers settle. Successful or discarded response delivery retains artifact ownership correctly. UnpackedImageSize validates1..10GiB; modern proof selects exactly one available image summary by digest/platform and reads Unpacked, with verified classic fallback. Preparation does not run user code or pull/build images. -Thirty-three new cases pass within321 Lambda core tests, including13 unpacked proof cases, two scalar/shared-lease cases and18 preparation/lifecycle cases. The latter cover exact precommit400/404 rejection, lost/postcommit/late nested500 outcomes, initial404 quarantine, no Tag replay, caller/response abandonment, bounded admission, alias conflict repair, lost/failed DELETE recovery, cancelled/timed shutdown, cached supervision failure and cleanup outside an entered runtime.76 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,997 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. +Thirty-three new cases pass within321 Lambda core tests, including13 unpacked proof cases, two scalar/shared-lease cases and18 preparation/lifecycle cases. The latter cover exact precommit400/404 rejection, lost/postcommit/late nested500 outcomes, initial404 quarantine, no Tag replay, caller/response abandonment, bounded admission, alias conflict repair, lost/failed DELETE recovery, cancelled/timed shutdown, cached supervision failure and cleanup outside an entered runtime.76 core doctests pass. Workspace build/test/fmt/Clippy/rustdoc pass with1,997 tests and399 explicitly resource-gated cases. Native provider/DryRun regressions pass; the separately gated Squib case remains excluded as VM evidence. The real Docker wire harness now acquires PinnedImages, keeps them through actual handler calls/container cleanup, and joins the image manager before DockerWire shutdown. Both two-case suites pass on arm64 VZ and amd64/QEMU, retaining near6MiB computed warm/error and empty-override behavior. Independent daemon queries find no private pin aliases. These are preparation/control-path qualifications, not provider/executor or default-latency acceptance. No dependencies/versions changed. Strict boundary checks report the same65 preexisting B03 findings, with no new pin/preparation/defaults-path finding; B01 remains an open dependency gate. @@ -208,10 +208,25 @@ Independent read-only review covers typed deployment state, store-issued candida | W45 | P3; `rustack-core/src/http.rs:375`, workspace validation | The existing2ms-progress/10ms-idle real-clock test can legitimately hit Idle when host scheduling stalls, defeating its total-deadline-only expectation. Pure timer tests now use Tokio paused time, with test-util explicitly enabled only for core dev builds | All five HTTP tests pass, including identical idle/tie/absolute-deadline assertions, without changing production behavior | | W46 | P2; `provider.rs`, self-review of publication guard integration | Rejecting publication only inside PublishVersion would let Image Create(Publish=true) insert before returning an error. Create/Update now reject this unsupported asynchronous combination before code processing or mutation; initial candidates clear unverified hash/size | Create leaves no resource; update preserves old revision/hash/status; initial Pending omits CodeSha256 | -Twenty-five new Lambda core cases cover initial success/failure/retry, preserving old revisions during/after updates, independent resource/revision/generation CAS, two concurrent completion results, stale lease release, pending mutation/publication rejection, effective-entry rollback, published lease lifetime and resolved URI, snapshot state reconciliation and metadata-only availability. Invocation captures live leases through its captured request and rejects revoked/restored/tampered artifacts. The final independent recheck found no remaining in-slice issue. All25 new cases pass within346 Lambda core tests;92 core doctests pass, including the external builder rejection. Workspace build/test/fmt/Clippy/rustdoc pass with2,022 tests and399 explicitly resource-gated cases. Native SDK/DryRun regressions pass; the separately gated Squib case is not VM execution evidence. Fresh target-only strict boundary validation reports the same65 preexisting Lambda findings and no finding in new deployment/artifact/fingerprint paths; the unfiltered attempt also exposes seven existing model documentation findings. These remain B03, not a green strict gate. B01 remains the16 recorded dependency advisories; no package/version/lock change or new ignore was introduced. +Twenty-five new Lambda core cases cover initial success/failure/retry, preserving old revisions during/after updates, independent resource/revision/generation CAS, two concurrent completion results, stale lease release, pending mutation/publication rejection, effective-entry rollback, published lease lifetime and resolved URI, snapshot state reconciliation and metadata-only availability. Invocation captures live leases through its captured request and rejects revoked/restored/tampered artifacts. The final independent recheck found no remaining in-slice issue. All25 new cases pass within346 Lambda core tests;92 core doctests pass, including the external builder rejection. Workspace build/test/fmt/Clippy/rustdoc pass with2,022 tests and399 explicitly resource-gated cases. Native provider/DryRun regressions pass; the separately gated Squib case is not VM execution evidence. Fresh target-only strict boundary validation reports the same65 preexisting Lambda findings and no finding in new deployment/artifact/fingerprint paths; the unfiltered attempt also exposes seven existing model documentation findings. These remain B03, not a green strict gate. B01 remains the16 recorded dependency advisories; no package/version/lock change or new ignore was introduced. This foundation is not an accepted asynchronous deployment owner: it cannot prove bounded work-slot reservation, joined worker cancellation/panic or provider Pending admission until the next consumer lands. Docker capability remains unavailable; directory ownership/reconciliation, pulls, pool/environment ownership and P1 security/strict checks remain required. No new package/version is introduced; the core test-only Tokio feature reuses the existing version. +## P1 task1.3 retained deployment owner and API review + +Independent read-only review covers accepted-work/mailbox reservation, atomic response receipts, registry/permit ownership above JoinSet children, store-notification cancellation, keyed settlement, root/emergency recovery and provider admission/status/revision behavior. Its final recheck finds no remaining in-slice issue. It does not close the production Docker pool/executor/probe gate. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W47 | P2; `deployment/manager.rs`, preparation select | Prioritizing global store notifications can starve a ready preparation. Poll preparation before unrelated notifications after deadline/cancel and retain post-result identity validation | Sixteen simultaneously-ready notification/result iterations plus one-future/no-replay actor coverage | +| W48 | P2; `docker/lease.rs`, overall/read deadlines | Accepted budgets up to3,600s were forwarded into the wire's600s limit. Validate total preparation separately and bound resolver, alias verification and unpacked proof by min(total, independent IO budget) | Successful actual wire/image preparation under a3,600s deployment budget | +| W49 | P2; self-review, abnormal root Drop | A worker can observe cancellation before abort/join recovery and incorrectly expose ordinary Cancelled for a failed owner. Project failed owner health to SupervisorFailed before settlement; preserve ordinary cancellation when health is sound | Off-runtime root Drop with in-flight tag, worker/root panic and cached-error regressions | +| W50 | P2; self-review, acceptance response capture | A metadata re-read after insertion can race deletion/recreation or fast completion. Store transactions now return an atomic acceptance receipt before enqueue, with no re-read | Delete between reservation and enqueue returns the original Pending receipt, settles stale work without poisoning admission and starts no tag | + +The20 additional Lambda core cases pass within366 tests; the model adds one RevisionId serialization test. Workspace build/test/fmt/Clippy/rustdoc pass with2,043 tests, with400 resource-gated tests after adding the live SDK case.101 core doctests pass, including closed-candidate construction. Native provider/DryRun regression target passes six actual Native/validation scenarios; its separately gated Squib selection remains excluded as VM evidence. The target does not use an SDK/HTTP layer; previous labels in this spec set are corrected. The independent live Rust SDK revision test passes through the running Lambda HTTP gateway, verifying modeled/raw412 responses for all three operations, explicit nonempty advancing revisions and an immediate four-field nonmutation check before any subsequent successful update. Both two-case real Docker wire suites pass again on Engine29.5.2 arm64 VZ and amd64/QEMU after the read-budget change; subsequent alias/container ownership queries are empty. The harness retains60s stream IO/120s attempt budgets and does not establish full executor default latency. No Image Invoke or boto3 evidence is claimed by these tests. + +Fresh target-only strict Clippy has the same65 preexisting B03 findings and none in new owner/lease/candidate code. Audit still reports16 B01 advisories; deny bans/licenses/sources pass while advisories fail, with no new ignore or dependency version/lock change. Next implement the Docker environment/pool executor, directory ownership/reconciliation, allowed pulls and startup probes/configuration before M0 can close; P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | @@ -219,6 +234,7 @@ This foundation is not an accepted asynchronous deployment owner: it cannot prov | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | The initial strict lambda-core boundary run found66 baseline findings:56 missing Errors sections, four documentation-format findings and six indexing/slicing sites; its unfiltered dependency run also found auth/core/model debt. The later JSON slice fixes the four rustack-core formatting findings and passes that crate's full strict production check. Remaining Lambda/auth/model findings still require a fresh final audit; standard workspace Clippy passes | P1 quality work: fix the remaining affected boundaries/docs and rerun strict checks without ignores; standard checks alone do not close this gate | +| B04 | P2; `crates/rustack-lambda-model/src/input.rs:85`, `provider.rs` code-update dispatch | Existing update DTOs ignore unknown JSON fields. The official UpdateFunctionCode DryRun field is currently absent, so a client-supplied DryRun can be ignored and a real code mutation performed | P1 API validation must implement side-effect-free UpdateFunctionCode DryRun and explicitly reject other out-of-profile fields through a bounded decoder. Add live SDK negative/NoMutation coverage before M0 closes; do not silently narrow the declared DryRun guarantee | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 25cd8db0..810f697b 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -187,6 +187,16 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Pinned by:** [runtime §2.10](./rustack-workflows-20-lambda-runtime-design.md#210-deployment-state-and-atomic-settlement), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-25. +## D24 — Retained deployment admission with atomic response receipts + +- **Context:** provider requests may disappear after metadata mutation; deletion, shutdown and actor failure must not orphan Pending records or release capacity before worker join. +- **Alternatives considered:** spawning from the request loses joins on cancellation; reserving only a mailbox does not bound accepted workers; re-reading metadata after enqueue can return a concurrently recreated resource or a falsely Active acceptance response; forwarding the total deployment budget to every RPC violates the wire's shorter IO limits. +- **Decision:** reserve both accepted-work and owned-mailbox capacity before mutation, capture the response receipt in the same store transaction, register guards/permits above JoinSet children, and retain root/emergency joins through waiter cancellation. Store-change notifications cancel stale candidates without replaying preparation; ready results outrank unrelated notifications after deadline/cancel. Each Docker read is bounded independently of the total deployment budget. +- **Why:** request lifetime cannot own accepted asynchronous work. Stale results remain harmless, failed updates preserve old code, and failure/Drop outside the entered runtime still settles keyed records. Executor image readiness defaults to unavailable, so qualified preparation alone cannot enable a nonexistent production backend. +- **Compatibility:** explicit stale RevisionId/CodeSha256 preconditions return412; in-progress operations and implicit staging conflicts remain409. The Native fixture target bypasses SDK/HTTP and is provider execution evidence; separate live SDK wire tests establish protocol evidence. +- **Pinned by:** [runtime §2.11](./rustack-workflows-20-lambda-runtime-design.md#211-retained-deployment-supervisor-and-provider-admission), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-25. + ## Glossary | Term | Meaning in this specification | diff --git a/tests/integration/src/test_lambda.rs b/tests/integration/src/test_lambda.rs index 7ffc15b1..81811cdb 100644 --- a/tests/integration/src/test_lambda.rs +++ b/tests/integration/src/test_lambda.rs @@ -9,6 +9,13 @@ mod tests { use anyhow::{Context, Result, ensure}; use aws_sdk_lambda::{ + operation::{ + publish_version::PublishVersionError, + update_function_code::UpdateFunctionCodeError, + update_function_configuration::{ + UpdateFunctionConfigurationError, UpdateFunctionConfigurationOutput, + }, + }, primitives::Blob, types::{Architecture, Environment, FunctionCode, FunctionUrlAuthType, Runtime}, }; @@ -160,6 +167,179 @@ mod tests { cleanup_function(&client, &name).await; } + async fn check_stale_update_requests( + client: &aws_sdk_lambda::Client, + name: &str, + revision: &str, + ) -> Result<()> { + let error = client + .update_function_configuration() + .function_name(name) + .revision_id(revision) + .description("must not commit") + .send() + .await + .err() + .context("Stale configuration update unexpectedly succeeded")?; + ensure!( + error + .as_service_error() + .is_some_and(UpdateFunctionConfigurationError::is_precondition_failed_exception), + "Wrong configuration error type" + ); + ensure!( + error + .raw_response() + .is_some_and(|response| response.status().as_u16() == 412), + "Wrong configuration HTTP status" + ); + let error = client + .update_function_code() + .function_name(name) + .revision_id(revision) + .zip_file(dummy_zip_blob()) + .send() + .await + .err() + .context("Stale code update unexpectedly succeeded")?; + ensure!( + error + .as_service_error() + .is_some_and(UpdateFunctionCodeError::is_precondition_failed_exception), + "Wrong code error type" + ); + ensure!( + error + .raw_response() + .is_some_and(|response| response.status().as_u16() == 412), + "Wrong code HTTP status" + ); + Ok(()) + } + + async fn check_rejected_update_preserved_metadata( + client: &aws_sdk_lambda::Client, + name: &str, + expected: &UpdateFunctionConfigurationOutput, + ) -> Result<()> { + let current = client + .get_function_configuration() + .function_name(name) + .send() + .await?; + ensure!( + current.revision_id() == expected.revision_id(), + "Rejected update changed revision" + ); + ensure!( + current.code_sha256() == expected.code_sha256(), + "Rejected update changed hash" + ); + ensure!( + current.code_size() == expected.code_size(), + "Rejected update changed size" + ); + ensure!( + current.description() == expected.description(), + "Rejected update changed description" + ); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires running server"] + async fn test_should_enforce_revision_preconditions_through_the_sdk_wire() -> Result<()> { + let client = lambda_client(); + let name = create_test_function(&client, "revision-cas").await; + let result = async { + let initial = client + .get_function_configuration() + .function_name(&name) + .send() + .await?; + let original = initial.revision_id().context("Missing initial revision")?; + let updated = client + .update_function_configuration() + .function_name(&name) + .revision_id(original) + .description("new configuration") + .send() + .await?; + let updated_revision = updated + .revision_id() + .filter(|value| !value.is_empty()) + .context("Missing updated revision")?; + ensure!( + updated_revision != original, + "Configuration revision did not advance" + ); + check_stale_update_requests(&client, &name, original).await?; + check_rejected_update_preserved_metadata(&client, &name, &updated).await?; + let code = client + .update_function_code() + .function_name(&name) + .revision_id(updated_revision) + .zip_file(dummy_zip_blob()) + .send() + .await?; + let code_revision = code + .revision_id() + .filter(|value| !value.is_empty()) + .context("Missing code revision")?; + ensure!( + code_revision != updated_revision, + "Code revision did not advance" + ); + let error = client + .publish_version() + .function_name(&name) + .revision_id(original) + .send() + .await + .err() + .context("Stale publication unexpectedly succeeded")?; + ensure!( + error + .as_service_error() + .is_some_and(PublishVersionError::is_precondition_failed_exception), + "Wrong publication error type" + ); + ensure!( + error + .raw_response() + .is_some_and(|response| response.status().as_u16() == 412), + "Wrong publication HTTP status" + ); + let current = client + .get_function_configuration() + .function_name(&name) + .send() + .await?; + ensure!( + current.description() == Some("new configuration"), + "Rejected mutation changed configuration" + ); + ensure!( + current.revision_id() == code.revision_id(), + "Rejected mutation changed code revision" + ); + let published = client + .publish_version() + .function_name(&name) + .revision_id(code_revision) + .send() + .await?; + ensure!( + published.version() == Some("1"), + "Valid publication did not succeed" + ); + Ok(()) + } + .await; + cleanup_function(&client, &name).await; + result + } + // --------------------------------------------------------------------------- // S3 code packages (issue #34) // --------------------------------------------------------------------------- From 0cef73b7d81aeeea9807b86e9fca4f0e122ba888 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 23:11:04 -0700 Subject: [PATCH 017/106] phase 1: enforce side-effect-free code validation and bounded deployment decoding Close B04 with explicit code DryRun, strict deployment DTOs, borrowed inline ZIP preflight and retained blocking parser admission. Validation does not stage, fetch, tag or publish, and rejects unknown modes before dispatch. Match runtime section 2.12 and record W51-W53 review corrections. Workspace build/test/fmt/Clippy/rustdoc pass: 2072 tests, zero failed, 401 resource-gated. Core has 395 tests and 101 doctests. Actual SDK/HTTP validation and revision regressions pass; independent read-only review has no remaining B04 findings. B01 (16 advisories), B03 (65 strict baseline findings), real Docker executor/directory ownership/pulls/probes and P2-P5 remain required; full P1/M0 is not complete. --- Makefile | 6 + crates/rustack-lambda-core/Cargo.toml | 2 +- crates/rustack-lambda-core/src/artifact.rs | 3 + .../src/deployment/manager_tests.rs | 45 +++ .../src/deployment_input.rs | 253 ++++++++++++++++ .../src/deployment_input_tests.rs | 185 ++++++++++++ crates/rustack-lambda-core/src/handler.rs | 25 +- crates/rustack-lambda-core/src/lib.rs | 1 + crates/rustack-lambda-core/src/provider.rs | 272 +++++++++++++++--- crates/rustack-lambda-model/src/input.rs | 9 +- crates/rustack-lambda-model/src/types.rs | 18 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 10 + .../rustack-workflows-60-operations-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 4 + specs/rustack-workflows-90-impl-plan.md | 4 +- specs/rustack-workflows-93-review.md | 16 +- specs/rustack-workflows-99-decisions.md | 9 + tests/integration/src/test_lambda.rs | 144 ++++++++++ 19 files changed, 938 insertions(+), 72 deletions(-) create mode 100644 crates/rustack-lambda-core/src/deployment_input.rs create mode 100644 crates/rustack-lambda-core/src/deployment_input_tests.rs diff --git a/Makefile b/Makefile index 7a0c1bf4..fb002f61 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,7 @@ SERVICE_SMOKE_TESTS := \ test_lambda::tests::test_should_create_and_get_function \ test_lambda::tests::test_should_invoke_dry_run \ test_lambda::tests::test_should_enforce_revision_preconditions_through_the_sdk_wire \ + test_lambda::tests::test_should_dry_run_code_without_mutation_and_reject_unsupported_deployment_fields \ test_lambda::tests::test_should_create_function_with_s3_code \ test_lambda::tests::test_should_update_function_code_from_s3_and_read_new_metadata \ test_lambda::tests::test_should_fail_create_function_with_missing_s3_bucket \ @@ -268,6 +269,11 @@ test-lambda-revision-wire: .PHONY: test-lambda-revision-wire +test-lambda-validation-wire: + @S3_ENDPOINT_URL="$(SMOKE_ENDPOINT)" cargo test -p rustack-integration --lib test_lambda::tests::test_should_dry_run_code_without_mutation_and_reject_unsupported_deployment_fields -- --ignored --exact --test-threads=1 + +.PHONY: test-lambda-validation-wire + # Deployment state, retained image preparation ownership and provider control-plane tests. test-lambda-deployments: @cargo test -p rustack-lambda-core --lib deployment diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 7bbdb784..586f766e 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -26,7 +26,7 @@ typed-builder = { workspace = true } winnow = { workspace = true } dashmap = { workspace = true } serde = { workspace = true } -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } chrono = { workspace = true } uuid = { workspace = true } tracing = { workspace = true } diff --git a/crates/rustack-lambda-core/src/artifact.rs b/crates/rustack-lambda-core/src/artifact.rs index cd263b86..e6b9559f 100644 --- a/crates/rustack-lambda-core/src/artifact.rs +++ b/crates/rustack-lambda-core/src/artifact.rs @@ -24,6 +24,9 @@ use tempfile::TempDir; use crate::storage::compute_sha256; +pub(crate) const MAX_ZIP_BYTES: usize = 50 * 1024 * 1024; +pub(crate) const MAX_ZIP_BASE64_BYTES: usize = MAX_ZIP_BYTES.div_ceil(3) * 4; + pub(crate) const MAX_IMAGE_BYTES: u64 = 10 * 1024 * 1024 * 1024; #[cfg(test)] use crate::{storage::stage_zip, test_zip::minimal_zip}; diff --git a/crates/rustack-lambda-core/src/deployment/manager_tests.rs b/crates/rustack-lambda-core/src/deployment/manager_tests.rs index 90bc8268..32ab0aea 100644 --- a/crates/rustack-lambda-core/src/deployment/manager_tests.rs +++ b/crates/rustack-lambda-core/src/deployment/manager_tests.rs @@ -682,3 +682,48 @@ async fn test_should_reject_deployment_capacity_larger_than_the_preparation_owne images.shutdown(deadline()).await.unwrap(); fake.close().await; } + +#[tokio::test] +async fn test_should_validate_image_code_dry_run_without_preparation_or_state_changes() { + let fake = FakePins::new(DaemonMode::Normal).await; + let mut images = fake.owner(2, 1); + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_executor( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }, + Arc::new(ControlPlaneExecutor), + ) + .with_image_preparation( + images.port(), + ImageDeploymentConfig::new(1, Duration::from_secs(10)).unwrap(), + ) + .unwrap(); + provider.create_function(input("worker")).await.unwrap(); + wait_provider(&provider).await; + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + let response = provider + .update_function_code( + "worker", + UpdateFunctionCodeInput { + image_uri: Some("missing:new-image".into()), + dry_run: Some(true), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(response.last_update_status.as_deref(), Some("Successful")); + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + assert_eq!(fake.stats.tags.load(Ordering::Acquire), 1); + assert_eq!(fake.stats.absent.load(Ordering::Acquire), 0); + provider.shutdown().await; + drop(provider); + images.shutdown(deadline()).await.unwrap(); + fake.close().await; +} diff --git a/crates/rustack-lambda-core/src/deployment_input.rs b/crates/rustack-lambda-core/src/deployment_input.rs new file mode 100644 index 00000000..fc11439d --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_input.rs @@ -0,0 +1,253 @@ +//! Bounded deployment DTO parsing with one borrowed inline-code exception. +//! +//! Pure blocking parse tasks, completed results and active dispatch retain one +//! of four permits until their bounded input work is released. An +//! abandoned caller intentionally leaves only bounded CPU/memory work running; +//! parsing cannot commit metadata, filesystem or Docker effects. + +use std::collections::BTreeMap; + +use bytes::Bytes; +use rustack_core::{ + http::append_bounded, + json::{JsonLimits, validate}, +}; +use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use tokio::{ + sync::{Semaphore, SemaphorePermit}, + task::spawn_blocking, +}; + +use crate::{artifact::MAX_ZIP_BASE64_BYTES, error::LambdaServiceError}; + +const BODY_BYTES: usize = 96 * 1024 * 1024; +const META_BYTES: usize = 1024 * 1024; +const STRING_BYTES: usize = 16 * 1024; +const KEY_BYTES: usize = 256; +const DEPTH: usize = 8; +const MEMBERS: usize = 4096; +// A fixed aggregate safety ceiling for large deployment parsing. Request +// admission remains separately operator-configurable at the HTTP boundary. +static PARSERS: Semaphore = Semaphore::const_new(4); + +#[derive(Debug, Clone, Copy)] +pub(crate) enum Profile { + Create, + Code, + Configuration, +} + +fn invalid() -> LambdaError { + LambdaError::invalid_parameter("Invalid or unsupported deployment request") +} + +pub(crate) async fn decode( + body: &Bytes, + profile: Profile, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + let body = body.clone(); + run(&PARSERS, move || parse(&body, profile)).await +} + +async fn run( + pool: &'static Semaphore, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + let permit = pool.try_acquire().map_err(|_| { + LambdaError::new( + LambdaErrorCode::TooManyRequestsException, + "Deployment parser capacity exhausted", + ) + })?; + run_admitted(permit, work).await +} + +async fn run_admitted( + permit: SemaphorePermit<'static>, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result<(T, SemaphorePermit<'static>), LambdaError> { + spawn_blocking(move || { + let value = work()?; + Ok((value, permit)) + }) + .await + .map_err(|_| LambdaError::service_error("Deployment parser failed"))? +} + +fn parse(body: &[u8], profile: Profile) -> Result { + lexical(body)?; + // Borrow raw values, so large strings are not allocated before identifying + // the only field allowed to exceed the ordinary metadata allowance. + let fields: BTreeMap = + serde_json::from_slice(body).map_err(|_| invalid())?; + let blob = match profile { + Profile::Code => fields.get("ZipFile").copied(), + Profile::Create => match fields.get("Code") { + Some(code) => { + let code: BTreeMap = + serde_json::from_str(code.get()).map_err(|_| invalid())?; + code.get("ZipFile").copied() + } + None => None, + }, + Profile::Configuration => None, + } + .filter(|value| value.get().as_bytes().first() == Some(&b'"')); + let mut metadata = Vec::new(); + if let Some(blob) = blob { + let raw = blob.get().as_bytes(); + if raw + .len() + .checked_sub(2) + .is_none_or(|size| size > MAX_ZIP_BASE64_BYTES) + { + return Err(invalid()); + } + // Borrowed RawValue is a span in this very input (serde_json's slice + // deserializer contract). Checked integer offsets never dereference a + // raw pointer; bounded slice access re-establishes the range. + let start = raw + .as_ptr() + .addr() + .checked_sub(body.as_ptr().addr()) + .ok_or_else(invalid)?; + let end = start.checked_add(raw.len()).ok_or_else(invalid)?; + append(&mut metadata, body.get(..start).ok_or_else(invalid)?)?; + append(&mut metadata, b"\"\"")?; + append(&mut metadata, body.get(end..).ok_or_else(invalid)?)?; + } else { + append(&mut metadata, body)?; + } + let limits = JsonLimits::new(META_BYTES, DEPTH, MEMBERS, STRING_BYTES, KEY_BYTES) + .map_err(|_| invalid())?; + validate(&metadata, limits).map_err(|_| invalid())?; + serde_json::from_slice(body).map_err(|_| invalid()) +} +fn append(metadata: &mut Vec, bytes: &[u8]) -> Result<(), LambdaError> { + append_bounded(metadata, bytes, 1024 * 1024).map_err(|_| invalid()) +} + +/// Bound structure and encoded keys before borrowed-map key materialization. +fn lexical(body: &[u8]) -> Result<(), LambdaError> { + if body.len() > BODY_BYTES { + return Err(invalid()); + } + let mut cursor = 0usize; + let mut depth = 0usize; + let mut delimiters = 0usize; + while let Some(byte) = body.get(cursor) { + match byte { + b'"' => { + let start = cursor; + cursor = quoted_end(body, cursor)?; + let following = body + .get(cursor..) + .ok_or_else(invalid)? + .iter() + .find(|byte| !byte.is_ascii_whitespace()); + if following == Some(&b':') && cursor.saturating_sub(start) > KEY_BYTES * 6 + 2 { + return Err(invalid()); + } + continue; + } + b'{' | b'[' => { + depth = depth + .checked_add(1) + .filter(|depth| *depth <= DEPTH) + .ok_or_else(invalid)?; + } + b'}' | b']' => { + depth = depth.checked_sub(1).ok_or_else(invalid)?; + } + b',' | b':' => { + delimiters = delimiters + .checked_add(1) + .filter(|count| *count <= MEMBERS * 2) + .ok_or_else(invalid)?; + } + _ => {} + } + cursor = cursor.checked_add(1).ok_or_else(invalid)?; + } + Ok(()) +} +fn quoted_end(body: &[u8], start: usize) -> Result { + let mut cursor = start.checked_add(1).ok_or_else(invalid)?; + loop { + match body.get(cursor) { + Some(b'"') => return cursor.checked_add(1).ok_or_else(invalid), + Some(b'\\') => cursor = cursor.checked_add(2).ok_or_else(invalid)?, + Some(_) => cursor = cursor.checked_add(1).ok_or_else(invalid)?, + None => return Err(invalid()), + } + } +} + +/// Check standard canonical base64 without allocating decoded archive contents. +pub(crate) async fn validate_inline( + encoded: String, + admitted: Option>, +) -> Result, LambdaServiceError> { + let result = if let Some(permit) = admitted { + run_admitted(permit, move || inline(&encoded)).await + } else { + run(&PARSERS, move || inline(&encoded)).await + }; + result + .map(|((), permit)| permit) + .map_err(|error| match error.code { + LambdaErrorCode::TooManyRequestsException => LambdaServiceError::TooManyRequests, + LambdaErrorCode::ServiceException => LambdaServiceError::Internal { + message: "Deployment parser failed".into(), + }, + _ => LambdaServiceError::InvalidParameter { + message: "Invalid inline code encoding".into(), + }, + }) +} + +fn inline(encoded: &str) -> Result<(), LambdaError> { + if encoded.len() > MAX_ZIP_BASE64_BYTES || !encoded.len().is_multiple_of(4) { + return Err(invalid()); + } + let data = encoded.trim_end_matches('='); + let padding = encoded.len().checked_sub(data.len()).ok_or_else(invalid)?; + if padding > 2 { + return Err(invalid()); + } + let mut last = 0; + for byte in data.bytes() { + last = match byte { + b'A'..=b'Z' => byte.checked_sub(b'A'), + b'a'..=b'z' => byte + .checked_sub(b'a') + .and_then(|value| value.checked_add(26)), + b'0'..=b'9' => byte + .checked_sub(b'0') + .and_then(|value| value.checked_add(52)), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } + .ok_or_else(invalid)?; + } + if (padding == 2 && last & 15 != 0) || (padding == 1 && last & 3 != 0) { + return Err(invalid()); + } + let decoded = encoded + .len() + .checked_div(4) + .and_then(|groups| groups.checked_mul(3)) + .and_then(|bytes| bytes.checked_sub(padding)) + .ok_or_else(invalid)?; + if decoded > crate::artifact::MAX_ZIP_BYTES { + return Err(invalid()); + } + Ok(()) +} + +#[cfg(test)] +#[path = "deployment_input_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/deployment_input_tests.rs b/crates/rustack-lambda-core/src/deployment_input_tests.rs new file mode 100644 index 00000000..d72704c9 --- /dev/null +++ b/crates/rustack-lambda-core/src/deployment_input_tests.rs @@ -0,0 +1,185 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use rstest::rstest; +use rustack_lambda_model::input::{ + CreateFunctionInput, UpdateFunctionCodeInput, UpdateFunctionConfigurationInput, +}; +use serde_json::json; +use tokio::sync::oneshot; + +use super::*; + +#[test] +fn test_should_allow_large_zip_only_at_the_selected_deployment_path() { + let zip = "A".repeat(2 * 1024 * 1024); + let code = serde_json::to_vec(&json!({"ZipFile":zip,"DryRun":true})).unwrap(); + let parsed: UpdateFunctionCodeInput = parse(&code, Profile::Code).unwrap(); + assert_eq!(parsed.zip_file.as_deref(), Some(zip.as_str())); + let create = + serde_json::to_vec(&json!({"FunctionName":"worker","Role":"role","Code":{"ZipFile":zip}})) + .unwrap(); + let parsed: CreateFunctionInput = parse(&create, Profile::Create).unwrap(); + assert_eq!(parsed.code.zip_file.as_deref(), Some(zip.as_str())); + assert!(parse::(&code, Profile::Configuration).is_err()); + let wrong = serde_json::to_vec(&json!({"Environment":{"Variables":{"ZipFile":zip}}})).unwrap(); + assert!(parse::(&wrong, Profile::Configuration).is_err()); +} + +#[rstest] +#[case(br#"{"ZipFile":"AA==","ZipFile":"AA=="}"#)] +#[case(br#"{"ZipFile":"AA==","\u005aipFile":"AA=="}"#)] +#[case(br#"{"DryRun":false,"DryRun":true}"#)] +#[case(br#"{"Architectures":[],"DryRun":"true"}"#)] +#[case(br#"{"DryRun":true,"SourceKMSKeyArn":"private-marker"}"#)] +#[case(br#"{"ZipFile":true}"#)] +#[case(br#"{"ZipFile":"unfinished}"#)] +fn test_should_reject_ambiguous_unknown_or_malformed_code_bodies(#[case] bytes: &[u8]) { + let error = parse::(bytes, Profile::Code).unwrap_err(); + assert_eq!(error.message, "Invalid or unsupported deployment request"); + assert!(!format!("{error:?}").contains("private-marker")); +} + +#[test] +fn test_should_recognize_escaped_code_keys_and_reject_nested_duplicate_environment_keys() { + let input = br#"{"FunctionName":"worker","Role":"role","\u0043ode":{"\u005aipFile":"AA=="}}"#; + assert_eq!( + parse::(input, Profile::Create) + .unwrap() + .code + .zip_file + .as_deref(), + Some("AA==") + ); + let duplicate = br#"{"Environment":{"Variables":{"TOKEN":"first","\u0054OKEN":"second"}}}"#; + assert!(parse::(duplicate, Profile::Configuration).is_err()); +} + +#[test] +fn test_should_reject_oversized_keys_strings_depth_and_metadata_before_dispatch() { + let body = serde_json::to_vec(&json!({"Description":"x".repeat(STRING_BYTES+1)})).unwrap(); + assert!(parse::(&body, Profile::Configuration).is_err()); + let key = format!("{{\"{}\":0}}", "x".repeat(KEY_BYTES * 6 + 1)); + assert!(lexical(key.as_bytes()).is_err()); + let nested = format!("{}0{}", "[".repeat(DEPTH + 1), "]".repeat(DEPTH + 1)); + assert!(lexical(nested.as_bytes()).is_err()); + let members = format!("[{}0]", "0,".repeat(MEMBERS * 2 + 1)); + assert!(lexical(members.as_bytes()).is_err()); + let metadata = format!("{{{}\"DryRun\":true}}", " ".repeat(META_BYTES)); + assert!(parse::(metadata.as_bytes(), Profile::Code).is_err()); +} + +#[rstest] +#[case("", true)] +#[case("AA==", true)] +#[case("AAA=", true)] +#[case("AAAA", true)] +#[case("AB==", false)] +#[case("AAB=", false)] +#[case("A===", false)] +#[case("AA=A", false)] +#[case("A", false)] +#[case("AAAA\n", false)] +fn test_should_validate_canonical_base64_without_decoding(#[case] data: &str, #[case] valid: bool) { + assert_eq!(inline(data).is_ok(), valid); +} + +#[test] +fn test_should_accept_exact_zip_size_and_reject_the_first_extra_decoded_byte() { + let bytes = vec![0u8; crate::artifact::MAX_ZIP_BYTES]; + let encoded = STANDARD.encode(bytes); + assert!(inline(&encoded).is_ok()); + let excess = "A".repeat(MAX_ZIP_BASE64_BYTES); + assert!(inline(&excess).is_err()); +} + +#[tokio::test] +async fn test_should_retain_a_parser_permit_after_its_response_waiter_is_dropped() { + static POOL: Semaphore = Semaphore::const_new(1); + let (begun, started) = oneshot::channel(); + let (release, blocked) = std::sync::mpsc::channel(); + let waiter = tokio::spawn(run(&POOL, move || { + begun.send(()).unwrap(); + blocked.recv().unwrap(); + Ok(()) + })); + started.await.unwrap(); + waiter.abort(); + let _ = waiter.await; + assert!(matches!( + run(&POOL, || Ok(())).await.unwrap_err().code, + LambdaErrorCode::TooManyRequestsException + )); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while POOL.available_permits() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(run(&POOL, || Ok(())).await.is_ok()); +} + +#[tokio::test] +async fn test_should_surface_a_pure_parser_worker_panic_without_losing_capacity() { + static POOL: Semaphore = Semaphore::const_new(1); + let error = run::<()>(&POOL, || panic!("injected pure parser panic")) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::ServiceException); + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_apply_strict_nested_deployment_dtos() { + for body in [ + json!({"ImageConfig":{"Unsupported":true}}), + json!({"Environment":{"Unknown":"private-marker"}}), + json!({"VpcConfig":{"Unsupported":true}}), + json!({"LoggingConfig":{"Unsupported":true}}), + ] { + assert!( + parse::( + &serde_json::to_vec(&body).unwrap(), + Profile::Configuration + ) + .is_err() + ); + } + let invalid = + json!({"FunctionName":"worker","Role":"role","Code":{"Unknown":"private-marker"}}); + assert!( + parse::(&serde_json::to_vec(&invalid).unwrap(), Profile::Create) + .is_err() + ); +} + +#[tokio::test] +async fn test_should_keep_capacity_with_a_completed_parse_result_until_it_is_consumed() { + static POOL: Semaphore = Semaphore::const_new(1); + let completed = run(&POOL, || Ok(7)).await.unwrap(); + assert_eq!(completed.0, 7); + assert_eq!(POOL.available_permits(), 0); + assert_eq!( + run(&POOL, || Ok(())).await.unwrap_err().code, + LambdaErrorCode::TooManyRequestsException + ); + drop(completed); + assert_eq!(POOL.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_reuse_admission_for_all_concurrent_inline_validators() { + static POOL: Semaphore = Semaphore::const_new(4); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let permit = POOL.try_acquire().unwrap(); + tasks.spawn(validate_inline("AA==".into(), Some(permit))); + } + let mut results = Vec::new(); + while let Some(result) = tasks.join_next().await { + results.push(result.unwrap().unwrap()); + } + assert_eq!(POOL.available_permits(), 0); + drop(results); + assert_eq!(POOL.available_permits(), 4); +} diff --git a/crates/rustack-lambda-core/src/handler.rs b/crates/rustack-lambda-core/src/handler.rs index 3339f584..64600ab8 100644 --- a/crates/rustack-lambda-core/src/handler.rs +++ b/crates/rustack-lambda-core/src/handler.rs @@ -30,7 +30,10 @@ use rustack_lambda_model::{ types::InvocationType, }; -use crate::provider::{InvokeKind, InvokeOutcome, RustackLambda}; +use crate::{ + deployment_input::{Profile, decode}, + provider::{InvokeKind, InvokeOutcome, RustackLambda}, +}; /// Handler that bridges the HTTP layer to the Lambda provider. #[derive(Debug)] @@ -121,7 +124,7 @@ async fn dispatch( path_params: &PathParams, query: &str, headers: &http::HeaderMap, - body: &[u8], + body: &Bytes, ) -> Result, LambdaError> { let query_params = parse_query_params(query); let qualifier = get_query_param(&query_params, "Qualifier"); @@ -129,9 +132,8 @@ async fn dispatch( match op { // ---- Phase 0: Function CRUD ---- LambdaOperation::CreateFunction => { - let input: CreateFunctionInput = serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, _input_permit): (CreateFunctionInput, _) = + decode(body, Profile::Create).await?; let config = provider .create_function(input) .await @@ -165,11 +167,10 @@ async fn dispatch( LambdaOperation::UpdateFunctionCode => { let function_name = require_path_param(path_params, "FunctionName")?; - let input: UpdateFunctionCodeInput = serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, input_permit): (UpdateFunctionCodeInput, _) = + decode(body, Profile::Code).await?; let config = provider - .update_function_code(function_name, input) + .update_function_code_parsed(function_name, input, input_permit) .await .map_err(LambdaError::from)?; wrap_json_response(200, &config) @@ -177,10 +178,8 @@ async fn dispatch( LambdaOperation::UpdateFunctionConfiguration => { let function_name = require_path_param(path_params, "FunctionName")?; - let input: UpdateFunctionConfigurationInput = - serde_json::from_slice(body).map_err(|e| { - LambdaError::invalid_parameter(format!("Invalid request body: {e}")) - })?; + let (input, _input_permit): (UpdateFunctionConfigurationInput, _) = + decode(body, Profile::Configuration).await?; let config = provider .update_function_configuration(function_name, &input) .map_err(LambdaError::from)?; diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index b597811b..326b3a4f 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -14,6 +14,7 @@ pub mod artifact; pub mod code; pub mod config; pub mod deployment; +mod deployment_input; #[cfg(unix)] pub mod docker; pub mod error; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 028784cd..ad446988 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use tracing::info; /// Maximum deployment package size (50 MB zipped, per Appendix C). -const MAX_ZIP_SIZE: u64 = 50 * 1024 * 1024; +const MAX_ZIP_SIZE: usize = crate::artifact::MAX_ZIP_BYTES; /// Maximum synchronous invoke payload size (6 MB, per Appendix C). const MAX_SYNC_PAYLOAD: usize = 6 * 1024 * 1024; @@ -53,7 +53,10 @@ use rustack_lambda_model::{ TracingConfigResponse, VpcConfigResponse, }, }; -use tokio::time::{Instant, timeout_at}; +use tokio::{ + sync::SemaphorePermit, + time::{Instant, timeout_at}, +}; use crate::{ artifact::{Architecture, DeploymentArtifact, ImageArtifact, ImageReference}, @@ -790,6 +793,24 @@ impl RustackLambda { &self, function_ref: &str, input: UpdateFunctionCodeInput, + ) -> Result { + self.update_code(function_ref, input, None).await + } + + pub(crate) async fn update_function_code_parsed( + &self, + function_ref: &str, + input: UpdateFunctionCodeInput, + permit: SemaphorePermit<'static>, + ) -> Result { + self.update_code(function_ref, input, Some(permit)).await + } + + async fn update_code( + &self, + function_ref: &str, + mut input: UpdateFunctionCodeInput, + mut admitted: Option>, ) -> Result { let code_source = CodeSource { zip_file_b64: input.zip_file.as_deref(), @@ -800,36 +821,27 @@ impl RustackLambda { }; validate_code_source(&code_source)?; - // Validate that some code source is provided. - if code_source.zip_file_b64.is_none() - && code_source.image_uri.is_none() - && code_source.s3_bucket.is_none() - { - return Err(LambdaServiceError::InvalidParameter { - message: "Provide at least one of ZipFile, S3Bucket, or ImageUri.".to_owned(), - }); - } - - let (name, _) = resolve_function_ref(function_ref)?; + let name = mutable_function_name(function_ref)?; let should_publish = input.publish.unwrap_or(false); let before = self.get_record(&name)?; - check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; - before.latest.deployment.ensure_mutable()?; - validate_image_publication(&before.latest.package_type, should_publish)?; - if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) - || (before.latest.package_type == "Image" && code_source.image_uri.is_none()) - { - return Err(invalid_parameter("Deployment package type cannot change")); - } - if input.architectures.as_ref().is_some_and(|architectures| { - architectures.len() != 1 - || architectures - .iter() - .any(|architecture| architecture != "arm64" && architecture != "x86_64") - }) { - return Err(invalid_parameter( - "Exactly one supported architecture is required", - )); + let architecture = validate_code_update(&before, &input, code_source)?; + + if input.dry_run.unwrap_or(false) { + if let Some(reference) = code_source.image_uri { + reference + .parse::() + .map_err(|_| invalid_parameter("Invalid ImageUri"))?; + } + let _inline_permit = if let Some(encoded) = input.zip_file.take() { + Some(crate::deployment_input::validate_inline(encoded, admitted.take()).await?) + } else { + None + }; + let current = self.get_record(&name)?; + check_revision(input.revision_id.as_deref(), ¤t.latest.revision_id)?; + check_staged_revision(¤t, &before)?; + current.latest.deployment.ensure_mutable()?; + return Ok(self.build_function_configuration(¤t, ¤t.latest)); } if before.latest.package_type == "Image" { @@ -838,16 +850,6 @@ impl RustackLambda { .ok_or_else(|| invalid_parameter("ImageUri is required"))? .parse() .map_err(|_| invalid_parameter("Invalid ImageUri"))?; - let architecture = match input - .architectures - .as_ref() - .and_then(|values| values.first()) - { - Some(value) => value - .parse() - .map_err(|_| invalid_parameter("Invalid architecture"))?, - None => crate::function_config::validate_execution_settings(&before.latest)?, - }; return self.update_image_deployment( &before, reference, @@ -901,7 +903,7 @@ impl RustackLambda { function_ref: &str, input: &UpdateFunctionConfigurationInput, ) -> Result { - let (name, _) = resolve_function_ref(function_ref)?; + let name = mutable_function_name(function_ref)?; validate_configuration_update(input)?; @@ -3172,6 +3174,61 @@ impl RustackLambda { } /// Reject publication before asynchronous image preparation can begin. +fn validate_code_update( + before: &FunctionRecord, + input: &UpdateFunctionCodeInput, + code_source: CodeSource<'_>, +) -> Result { + // Validate that some code source is provided. + if code_source.zip_file_b64.is_none() + && code_source.image_uri.is_none() + && code_source.s3_bucket.is_none() + { + return Err(LambdaServiceError::InvalidParameter { + message: "Provide at least one of ZipFile, S3Bucket, or ImageUri.".to_owned(), + }); + } + + check_revision(input.revision_id.as_deref(), &before.latest.revision_id)?; + before.latest.deployment.ensure_mutable()?; + validate_image_publication(&before.latest.package_type, input.publish.unwrap_or(false))?; + if (before.latest.package_type == "Zip" && code_source.image_uri.is_some()) + || (before.latest.package_type == "Image" && code_source.image_uri.is_none()) + { + return Err(invalid_parameter("Deployment package type cannot change")); + } + if input.architectures.as_ref().is_some_and(|architectures| { + architectures.len() != 1 + || architectures + .iter() + .any(|architecture| architecture != "arm64" && architecture != "x86_64") + }) { + return Err(invalid_parameter( + "Exactly one supported architecture is required", + )); + } + match input + .architectures + .as_ref() + .and_then(|values| values.first()) + { + Some(value) => value + .parse() + .map_err(|_| invalid_parameter("Invalid architecture")), + None => crate::function_config::validate_execution_settings(&before.latest), + } +} + +fn mutable_function_name(reference: &str) -> Result { + let (name, qualifier) = resolve_function_ref(reference)?; + if qualifier.as_deref().is_some_and(|value| value != "$LATEST") { + return Err(invalid_parameter( + "Only the unpublished function configuration can be updated", + )); + } + Ok(name) +} + fn check_staged_revision( record: &FunctionRecord, before: &FunctionRecord, @@ -3347,7 +3404,7 @@ fn validate_code_source(source: &CodeSource<'_>) -> Result<(), LambdaServiceErro /// storage or extraction happens, so oversized packages fail fast instead /// of materializing onto disk. fn check_code_size(size_bytes: usize) -> Result<(), LambdaServiceError> { - if size_bytes as u64 > MAX_ZIP_SIZE { + if size_bytes > MAX_ZIP_SIZE { return Err(LambdaServiceError::InvalidParameter { message: format!("Unzipped size must be smaller than {MAX_ZIP_SIZE} bytes"), }); @@ -3543,6 +3600,137 @@ mod tests { } } + async fn artifact_count(path: &std::path::Path) -> usize { + let mut entries = tokio::fs::read_dir(path).await.unwrap(); + let mut count = 0usize; + while entries.next_entry().await.unwrap().is_some() { + count = count.saturating_add(1); + } + count + } + + #[derive(Debug, Default)] + struct CountingFetcher(std::sync::atomic::AtomicUsize); + #[async_trait::async_trait] + impl S3CodeFetcher for CountingFetcher { + async fn fetch_code( + &self, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result { + self.0.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Err(S3CodeFetchError::Internal(anyhow::anyhow!( + "unexpected fetch" + ))) + } + } + + #[tokio::test] + async fn test_should_validate_code_dry_run_without_staging_fetching_or_publishing() { + use base64::{Engine, engine::general_purpose::STANDARD}; + let root = tempfile::tempdir().unwrap(); + let fetcher = Arc::new(CountingFetcher::default()); + let provider = + RustackLambda::with_store(FunctionStore::new(root.path()), LambdaConfig::default()) + .with_code_fetcher(fetcher.clone()); + provider + .create_function(sample_create_input("dry-update")) + .await + .unwrap(); + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + let count = artifact_count(root.path()).await; + for input in [ + UpdateFunctionCodeInput { + zip_file: Some(STANDARD.encode(b"not an archive")), + dry_run: Some(true), + publish: Some(true), + ..Default::default() + }, + UpdateFunctionCodeInput { + s3_bucket: Some("bucket".into()), + s3_key: Some("missing.zip".into()), + dry_run: Some(true), + ..Default::default() + }, + ] { + let result = provider + .update_function_code("dry-update", input) + .await + .unwrap(); + assert_eq!(result.version.as_deref(), Some("$LATEST")); + } + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + assert_eq!(artifact_count(root.path()).await, count); + assert_eq!(fetcher.0.load(std::sync::atomic::Ordering::Acquire), 0); + } + + #[tokio::test] + async fn test_should_reject_invalid_or_qualified_code_dry_run_without_mutation() { + let provider = test_provider(); + provider + .create_function(sample_create_input("dry-invalid")) + .await + .unwrap(); + let before = serde_json::to_value(provider.export_snapshot()).unwrap(); + for input in [ + UpdateFunctionCodeInput { + zip_file: Some("AB==".into()), + dry_run: Some(true), + ..Default::default() + }, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + revision_id: Some("stale".into()), + ..Default::default() + }, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + architectures: Some(Vec::new()), + ..Default::default() + }, + ] { + assert!( + provider + .update_function_code("dry-invalid", input) + .await + .is_err() + ); + } + for reference in ["dry-invalid:1", "dry-invalid:prod"] { + assert!( + provider + .update_function_code( + reference, + UpdateFunctionCodeInput { + zip_file: Some("AA==".into()), + dry_run: Some(true), + ..Default::default() + } + ) + .await + .is_err() + ); + assert!( + provider + .update_function_configuration( + reference, + &UpdateFunctionConfigurationInput::default() + ) + .is_err() + ); + } + assert_eq!( + serde_json::to_value(provider.export_snapshot()).unwrap(), + before + ); + } + #[tokio::test] async fn test_should_reject_invalid_settings_before_fetching_or_staging_code() -> Result<(), Box> { diff --git a/crates/rustack-lambda-model/src/input.rs b/crates/rustack-lambda-model/src/input.rs index 2cb68566..17e741df 100644 --- a/crates/rustack-lambda-model/src/input.rs +++ b/crates/rustack-lambda-model/src/input.rs @@ -12,7 +12,7 @@ use crate::types::{ /// Input for `CreateFunction`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct CreateFunctionInput { /// Function name (1-140 characters). pub function_name: String, @@ -81,8 +81,11 @@ pub struct CreateFunctionInput { /// Input for `UpdateFunctionCode`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct UpdateFunctionCodeInput { + /// Validate parameters without modifying code or publishing a version. + #[serde(skip_serializing_if = "Option::is_none")] + pub dry_run: Option, /// Revision ID precondition for an optimistic update. #[serde(skip_serializing_if = "Option::is_none")] pub revision_id: Option, @@ -114,7 +117,7 @@ pub struct UpdateFunctionCodeInput { /// Input for `UpdateFunctionConfiguration`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct UpdateFunctionConfigurationInput { /// Revision ID precondition for an optimistic update. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/rustack-lambda-model/src/types.rs b/crates/rustack-lambda-model/src/types.rs index 4af421cc..b10e5068 100644 --- a/crates/rustack-lambda-model/src/types.rs +++ b/crates/rustack-lambda-model/src/types.rs @@ -227,7 +227,7 @@ pub enum InvokeMode { /// Function code input for `CreateFunction` and `UpdateFunctionCode`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct FunctionCode { /// Base64-encoded zip file contents. #[serde(skip_serializing_if = "Option::is_none")] @@ -269,7 +269,7 @@ pub struct FunctionCodeLocation { /// Environment variable configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct Environment { /// Environment variable key-value pairs. #[serde(skip_serializing_if = "Option::is_none")] @@ -302,7 +302,7 @@ pub struct EnvironmentError { /// VPC configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct VpcConfig { /// Subnet IDs. #[serde(skip_serializing_if = "Option::is_none")] @@ -329,7 +329,7 @@ pub struct VpcConfigResponse { /// Tracing configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct TracingConfig { /// Tracing mode. #[serde(skip_serializing_if = "Option::is_none")] @@ -347,7 +347,7 @@ pub struct TracingConfigResponse { /// Dead letter queue configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct DeadLetterConfig { /// Target ARN (SQS queue or SNS topic). #[serde(skip_serializing_if = "Option::is_none")] @@ -356,7 +356,7 @@ pub struct DeadLetterConfig { /// Ephemeral storage configuration. #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct EphemeralStorage { /// Size in MB (512-10240). pub size: u32, @@ -370,7 +370,7 @@ impl Default for EphemeralStorage { /// Image configuration override. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct ImageConfig { /// Entry point override. #[serde(skip_serializing_if = "Option::is_none")] @@ -409,7 +409,7 @@ pub struct ImageConfigError { /// Logging configuration. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct LoggingConfig { /// Log format. #[serde(skip_serializing_if = "Option::is_none")] @@ -427,7 +427,7 @@ pub struct LoggingConfig { /// SnapStart configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct SnapStart { /// Apply on. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/specs/index.md b/specs/index.md index d5b5f848..791bc48c 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index b9180f36..392d74eb 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -252,6 +252,16 @@ Required tests use the real bounded wire/image-preparation actors with a scripte └───────────────────────────────────────────────────────────────────────┘ ``` +### 2.12 Side-effect-free code validation and bounded deployment JSON + +UpdateFunctionCode models DryRun explicitly. When true, validate code-source exclusivity/completeness, function and mutable qualifier (unqualified or $LATEST only), revision precondition, current deployment mutability, package compatibility, architecture syntax and local publication restrictions. Validate inline base64 syntax/canonical padding and the50MiB decoded limit without extracting or inspecting an archive. Validate image-reference grammar but do not inspect/pull/tag it; validate S3 source syntax but do not fetch it. No staging directory, code hash/revision/timestamp change, published version, work-slot reservation, container or invocation is produced. Return the current configuration snapshot after rechecking resource/revision following any bounded CPU validation. DryRun=false/omitted keeps actual deployment behavior; Invoke DryRun remains a separate API. + +CreateFunction, UpdateFunctionCode and UpdateFunctionConfiguration reject unknown fields, including unsupported newer AWS modes. Their nested deployment DTOs (code/environment/VPC/tracing/dead-letter/ephemeral/image/logging/SnapStart) reject unknown members too; environment/tag map keys remain dynamic. Unsupported fields must fail before provider dispatch, including when DryRun=true, rather than silently selecting another execution model. Unknown-field/JSON errors return a fixed bounded message without echoing caller text. This follows the [official DryRun parameter](https://docs.aws.amazon.com/lambda/latest/api/API_UpdateFunctionCode.html), with explicitly local validation/response behavior and the existing gateway permission model, not an IAM evaluator. + +A Lambda-specific deployment codec preserves the96MiB HTTP code envelope while bounding metadata to1MiB, container depth8, aggregate members4,096, decoded metadata strings16KiB and decoded keys256 bytes. A lexical pass bounds depth, structural delimiters and encoded keys before any map allocation. It locates borrowed raw JSON values using serde_json's existing raw_value feature (no new package/version); borrowed RawValue retains original input spans without building an arbitrary value tree ([documented ownership](https://docs.rs/serde_json/latest/serde_json/value/struct.RawValue.html#ownership)). Only root ZipFile for code updates or Code.ZipFile for creation receives the large blob exception: raw quoted content≤69,905,068 bytes. Mask that borrowed span to an empty JSON string in a capped metadata buffer, run shared JSON preflight for decoded-key duplication and exact metadata limits, then deserialize the strict DTO from the original bounded body. Escaped keys remain recognized; wrong-path blobs, unknown fields and duplicate keys cannot bypass the metadata budget. Domain settings validation still applies its tighter field caps. + +Pure JSON/base64 validation runs on bounded blocking workers (fixed ceiling4). Permits travel with completed results and deployment dispatch until their input work is released, preventing a backlog of large decoded requests after CPU parsing completes. Inline DryRun validation reuses its admitted request permit rather than acquiring a second slot, avoiding self-induced overload when all four requests validate concurrently. Request cancellation may leave an already-running pure parse task self-owned until it finishes; it owns only bounded bytes/results and cannot commit resources or metadata. Worker panic is surfaced as a bounded error to an attached caller. Image preparation and function execution retain their independent owners. Tests cover large valid inline blobs, wrong-path/oversized metadata, escaped duplicate keys, malformed/deep structures, static errors, parser saturation, ZIP/S3/image DryRun nonmutation, stale revisions/qualified references and actual SDK/HTTP validation. The other API decoders retain their existing profile; this does not claim a global service-parser rewrite. + ## 3. Runtime API and relay transport Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation/{requestId}/response, POST invocation/{requestId}/error and POST init/error. Each environment processes one invocation at a time. Managed Instances concurrency, extensions, streaming and SnapStart restore are excluded. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 3c2c06f2..144a76e6 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -77,6 +77,8 @@ All budgets are validated once; limits below are local defaults and safe configu | lambdaDockerCleanupSeconds |10|1..60; bounded by global shutdown remaining time | | workflowDrainSeconds |5|0..60; bounded by global shutdown remaining time | +Deployment JSON/base64 workers have a fixed aggregate ceiling of four, retained through completed results and provider dispatch; overload returns429. Code bodies remain capped at96MiB, with only the designated inline ZIP span exempted from the1MiB metadata budget. Unknown fields and malformed/budget-exceeding JSON return static400 errors; worker failure is a static500 error. See [runtime §2.12](./rustack-workflows-20-lambda-runtime-design.md#212-side-effect-free-code-validation-and-bounded-deployment-json). These are compile-time safety ceilings, not YAML tuning fields. + Registry/execution mailbox capacities and per-tick quantum are compile-time implementation constants from the ASL design, not runtime tuning knobs. Function init/idle settings reuse existing validated Lambda configuration. CPU and log limits use the exact environment keys above. Effective memory admission may allow fewer than 32 environments. Lower quotas must reject admission or fail the affected execution explicitly, never silently drop data/work. ## 2. Deployment topology and readiness diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index f3637bae..4c4851da 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -90,6 +90,10 @@ Retained deployment ownership (§2.11) now has19 additional actor/provider cases `make test-lambda-invoke-native` exercises real Native functions directly through the provider; it bypasses AWS SDK/HTTP. Historical references to it as a Native SDK target are corrected to Native provider evidence. `make test-lambda-revision-wire SMOKE_ENDPOINT=` separately runs an actual Rust SDK through HTTP: valid configuration/code updates/publication require explicit nonempty advancing revision IDs; stale requests decode as modeled PreconditionFailedException with raw412, and an immediate read proves revision/hash/size/description are unchanged before any successful update. This test is included in the explicit service smoke manifest. It does not establish boto3 or Image handler execution acceptance. The changed image read deadlines are additionally qualified by both two-case real Docker wire suites on Engine29.5.2 arm64 VZ and amd64/QEMU; independent queries find no private image aliases or owned containers. Image metadata reads use the configured5s budget, while the wire harness retains60s stream IO/120s attempt stress allowances; full executor default latency and native x86/Desktop qualification remain open. +`make test-lambda-validation-wire SMOKE_ENDPOINT=` explicitly runs SDK/HTTP code DryRun with inline nonarchive bytes and a nonexistent S3 source; both must succeed without extraction/fetch, revision/hash/size/configuration changes or publication. Unsupported Create KMSKeyArn, UpdateCode SourceKMSKeyArn and UpdateConfiguration KMSKeyArn decode as InvalidParameterValueException; rejected creation must leave no resource. The test is in the explicit service smoke manifest. Image DryRun separately uses the qualified scripted daemon/preparation owner and a declared test executor: old full snapshot and tag count remain unchanged, with no new removal. This is not Image Invoke acceptance. + +The deployment decoder (§2.12) has exact-limit, escaped/duplicate-key, wrong-path blob, malformed/unknown-field and depth/member/string/key/metadata tests. Inline base64 covers canonical padding and the exact50MiB decoded boundary without allocation of decoded bytes. Capacity remains owned after waiter cancellation and after a result becomes ready; four concurrent admitted inline validators reuse permits. Worker panic releases capacity and surfaces a bounded error; capacity429 must not be downgraded to400. Current395 core tests and101 doctests pass; workspace2,072 tests pass,401 resource-gated cases remain separately selected. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 5fc6c997..877b5865 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -142,7 +142,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass; production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass; production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. Next fix B04 side-effect-free code-update DryRun and bounded request validation before enabling real execution, then implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03 and B04 request-validation behavior remain mandatory before P1 closes. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01 dependency security and B03 strict boundary quality remain mandatory before P1 closes; B04 is closed without enabling Docker execution. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 78c07386..97a0032e 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -227,6 +227,20 @@ The20 additional Lambda core cases pass within366 tests; the model adds one Revi Fresh target-only strict Clippy has the same65 preexisting B03 findings and none in new owner/lease/candidate code. Audit still reports16 B01 advisories; deny bans/licenses/sources pass while advisories fail, with no new ignore or dependency version/lock change. Next implement the Docker environment/pool executor, directory ownership/reconciliation, allowed pulls and startup probes/configuration before M0 can close; P2–P5 remain required. +## P1 B04 deployment validation review + +The final independent read-only source review finds no remaining B04 issue, including the SDK rejected-creation absence check. Self-review corrected two admission-lifetime issues; the error-class correction was also rechecked against current source. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W51 | P2; self-review, `deployment_input.rs`, blocking-result ownership | Releasing capacity when CPU parsing ends allows ready large DTOs to accumulate. Return the permit with the result and retain it through provider dispatch | Completed-result capacity test plus cancelled waiter and worker panic capacity tests | +| W52 | P2; self-review, `provider.rs`, inline DryRun admission | Acquiring a second parse permit after HTTP decoding can reject all four already-admitted requests. Reuse the existing permit for the pure base64 worker | Four concurrent admitted validators succeed without nested admission | +| W53 | P2; review, `deployment_input.rs`, service-error conversion | Capacity and worker failure must not collapse into invalid-input400. Preserve TooManyRequests429 and Internal500 with static messages | Current conversion source independently rechecked; capacity/panic/invalid encoding cases remain separately classified | + +29 additional core cases pass:395 tests,101 doctests. They cover ZIP/S3/image DryRun full-snapshot nonmutation, no staging/fetch/tag/publication, stale/qualified requests, strict nested schemas, large valid blobs, wrong-path metadata, exact base64 limits and parse/result ownership. Actual Rust SDK/HTTP validation passes (one explicitly selected case, zero ignored), including Create/UpdateCode/UpdateConfiguration unsupported fields, absent rejected resources, unchanged metadata and version count. The separate SDK revision/412 regression passes again. Native provider/DryRun regression passes six actual scenarios; the gated Squib selection is still not VM execution evidence. + +Final workspace build/test/fmt/Clippy/rustdoc gates pass:2,072 tests, zero failed and401 resource-gated cases. No dependency package/version or Cargo.lock change landed; serde_json's existing raw_value feature supplies borrowed RawValue. Fresh strict core boundary Clippy still reports exactly65 B03 baseline findings, none in the decoder or newly extracted validation helpers. Fresh audit reports the same16 B01 advisories, and deny bans/licenses/sources pass while advisories fail; no new ignores. B04 is closed. B01/B03 and the remaining Docker executor/directory ownership/pulls/probes still block full P1/M0 acceptance; P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | @@ -234,7 +248,7 @@ Fresh target-only strict Clippy has the same65 preexisting B03 findings and none | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | The initial strict lambda-core boundary run found66 baseline findings:56 missing Errors sections, four documentation-format findings and six indexing/slicing sites; its unfiltered dependency run also found auth/core/model debt. The later JSON slice fixes the four rustack-core formatting findings and passes that crate's full strict production check. Remaining Lambda/auth/model findings still require a fresh final audit; standard workspace Clippy passes | P1 quality work: fix the remaining affected boundaries/docs and rerun strict checks without ignores; standard checks alone do not close this gate | -| B04 | P2; `crates/rustack-lambda-model/src/input.rs:85`, `provider.rs` code-update dispatch | Existing update DTOs ignore unknown JSON fields. The official UpdateFunctionCode DryRun field is currently absent, so a client-supplied DryRun can be ignored and a real code mutation performed | P1 API validation must implement side-effect-free UpdateFunctionCode DryRun and explicitly reject other out-of-profile fields through a bounded decoder. Add live SDK negative/NoMutation coverage before M0 closes; do not silently narrow the declared DryRun guarantee | +| B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 810f697b..625f72d5 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -197,6 +197,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Pinned by:** [runtime §2.11](./rustack-workflows-20-lambda-runtime-design.md#211-retained-deployment-supervisor-and-provider-admission), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-25. +## D25 — Explicit DryRun and bounded deployment parsing ownership + +- **Context:** ignored DryRun can mutate code; blindly applying ordinary JSON string limits rejects legitimate inline ZIPs, while parsing the entire code envelope into generic values permits large metadata allocations. +- **Alternatives considered:** permissive DTOs silently discard unsupported modes; decoding ZIPs for validation adds staging and content inspection; releasing a parse permit at CPU completion permits large ready-result backlogs; acquiring another permit inside an admitted DryRun can throttle every admitted request. +- **Decision:** model code DryRun explicitly and validate syntax/preconditions without fetching, staging, Docker admission or publication. Reject unknown deployment fields. Borrow the sole allowed ZIP raw span, cap/validate all remaining metadata, and deserialize strict DTOs on at most four pure blocking workers. Carry permits with results through dispatch and reuse the admitted permit for inline validation. +- **Why:** validation must not produce deployment effects, request cancellation must retain bounded CPU/memory ownership, and supported inline package sizes must not weaken metadata limits. Preserve overload429 and worker-failure500 separately from invalid-input400. This is a deployment boundary change, not a claim of a global parser rewrite or IAM parity. +- **Pinned by:** [runtime §2.12](./rustack-workflows-20-lambda-runtime-design.md#212-side-effect-free-code-validation-and-bounded-deployment-json), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-25. + ## Glossary | Term | Meaning in this specification | diff --git a/tests/integration/src/test_lambda.rs b/tests/integration/src/test_lambda.rs index 81811cdb..94b56715 100644 --- a/tests/integration/src/test_lambda.rs +++ b/tests/integration/src/test_lambda.rs @@ -10,6 +10,8 @@ mod tests { use anyhow::{Context, Result, ensure}; use aws_sdk_lambda::{ operation::{ + create_function::CreateFunctionError, + get_function::GetFunctionError, publish_version::PublishVersionError, update_function_code::UpdateFunctionCodeError, update_function_configuration::{ @@ -340,6 +342,148 @@ mod tests { result } + async fn reject_unsupported_creation( + client: &aws_sdk_lambda::Client, + name: &str, + ) -> Result<()> { + let error = client + .create_function() + .function_name(name) + .runtime(Runtime::Python312) + .role("arn:aws:iam::000000000000:role/test-role") + .handler("index.handler") + .code(FunctionCode::builder().zip_file(dummy_zip_blob()).build()) + .kms_key_arn("arn:aws:kms:us-east-1:000000000000:key/unsupported") + .send() + .await + .err() + .context("Unsupported creation option was ignored")?; + ensure!( + error + .as_service_error() + .is_some_and(CreateFunctionError::is_invalid_parameter_value_exception), + "Wrong unsupported-create error" + ); + let missing = client + .get_function() + .function_name(name) + .send() + .await + .err() + .context("Rejected creation inserted a resource")?; + ensure!( + missing + .as_service_error() + .is_some_and(GetFunctionError::is_resource_not_found_exception), + "Rejected creation was not absent" + ); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires running server"] + async fn test_should_dry_run_code_without_mutation_and_reject_unsupported_deployment_fields() + -> Result<()> { + let client = lambda_client(); + let name = create_test_function(&client, "dry-run-code").await; + let unsupported = func_name("unsupported-create"); + let result = async { + reject_unsupported_creation(&client, &unsupported).await?; + let before = client + .get_function_configuration() + .function_name(&name) + .send() + .await?; + let revision = before + .revision_id() + .filter(|value| !value.is_empty()) + .context("Missing revision")?; + client + .update_function_code() + .function_name(&name) + .revision_id(revision) + .zip_file(Blob::new(b"not an archive".to_vec())) + .dry_run(true) + .publish(true) + .send() + .await?; + client + .update_function_code() + .function_name(&name) + .s3_bucket("missing-bucket") + .s3_key("missing.zip") + .dry_run(true) + .send() + .await?; + let error = client + .update_function_code() + .function_name(&name) + .zip_file(dummy_zip_blob()) + .source_kms_key_arn("arn:aws:kms:us-east-1:000000000000:key/unsupported") + .dry_run(true) + .send() + .await + .err() + .context("Unsupported code option was ignored")?; + ensure!( + error + .as_service_error() + .is_some_and(UpdateFunctionCodeError::is_invalid_parameter_value_exception), + "Wrong unsupported-code error" + ); + let error = client + .update_function_configuration() + .function_name(&name) + .kms_key_arn("arn:aws:kms:us-east-1:000000000000:key/unsupported") + .description("must not commit") + .send() + .await + .err() + .context("Unsupported configuration option was ignored")?; + ensure!( + error.as_service_error().is_some_and( + UpdateFunctionConfigurationError::is_invalid_parameter_value_exception + ), + "Wrong unsupported-config error" + ); + let current = client + .get_function_configuration() + .function_name(&name) + .send() + .await?; + ensure!( + current.revision_id() == Some(revision), + "Validation changed revision" + ); + ensure!( + current.code_sha256() == before.code_sha256(), + "Validation changed hash" + ); + ensure!( + current.code_size() == before.code_size(), + "Validation changed size" + ); + ensure!( + current.description() == before.description(), + "Validation changed configuration" + ); + let versions = client + .list_versions_by_function() + .function_name(&name) + .send() + .await?; + ensure!( + versions.versions().len() == 1, + "Validation published a version" + ); + Ok(()) + } + .await; + cleanup_function(&client, &unsupported).await; + cleanup_function(&client, &name).await; + result + } + // --------------------------------------------------------------------------- // S3 code packages (issue #34) // --------------------------------------------------------------------------- From 6528215741c83266700807bc41a87d6cceac2503 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 23:23:32 -0700 Subject: [PATCH 018/106] phase 1: close strict Lambda and authentication boundary checks Close B03 across six production libraries. Propagate typed HMAC initialization errors through signing and verification, preserve constant-time and S3 normalization behavior, replace guarded indexing with checked patterns, and document actual method-specific failures. Match runtime section 7.1 and record D26 plus independently fixed W54-W56 review findings. Final workspace build/test/fmt/Clippy/rustdoc pass: 2074 tests, zero failed, 401 gated. The new Make target passes 659 unit tests and 165 doctests plus strict production Clippy without new ignores. Four RFC HMAC vectors and existing streaming/query tamper regressions pass; Native provider regressions pass without claiming gated Squib VM evidence. No dependency manifest/version/lock changes. B01, real Docker ownership/pool/executor/pulls/probes and P2-P5 remain required; full P1/M0 is incomplete. --- Makefile | 10 + crates/rustack-auth/src/canonical.rs | 16 +- crates/rustack-auth/src/credentials.rs | 2 +- crates/rustack-auth/src/error.rs | 12 +- crates/rustack-auth/src/lib.rs | 10 +- crates/rustack-auth/src/presigned.rs | 66 ++++--- crates/rustack-auth/src/sigv2.rs | 49 +++-- crates/rustack-auth/src/sigv4.rs | 165 ++++++++++------ crates/rustack-lambda-core/src/config.rs | 10 + .../rustack-lambda-core/src/executor/mod.rs | 2 +- .../src/executor/native.rs | 9 +- .../rustack-lambda-core/src/executor/squib.rs | 4 + crates/rustack-lambda-core/src/invocation.rs | 2 +- crates/rustack-lambda-core/src/provider.rs | 186 +++++++++++++++++- crates/rustack-lambda-core/src/resolver.rs | 15 +- crates/rustack-lambda-core/src/storage.rs | 36 +++- crates/rustack-lambda-http/src/dispatch.rs | 2 + crates/rustack-lambda-http/src/lib.rs | 2 +- crates/rustack-lambda-http/src/router.rs | 2 +- crates/rustack-lambda-model/src/input.rs | 6 +- crates/rustack-lambda-model/src/types.rs | 8 +- crates/rustack-s3-http/src/upload.rs | 9 +- crates/rustack-sqs-http/src/service.rs | 5 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 8 +- specs/rustack-workflows-93-review.md | 18 +- specs/rustack-workflows-99-decisions.md | 9 + 29 files changed, 520 insertions(+), 155 deletions(-) diff --git a/Makefile b/Makefile index fb002f61..bece6c84 100644 --- a/Makefile +++ b/Makefile @@ -289,6 +289,16 @@ test-lambda-runtime: .PHONY: test-json-boundaries test-lambda-runtime +# Strict production-library boundary gate; unit, integration and doctest regressions. +test-lambda-boundaries: + @cargo test -p rustack-auth -p rustack-core -p rustack-lambda-model -p rustack-lambda-http -p rustack-lambda-core -p rustack-lambda-runtime + @cargo clippy -p rustack-auth -p rustack-core -p rustack-lambda-model -p rustack-lambda-http -p rustack-lambda-core -p rustack-lambda-runtime --lib --no-deps -- \ + -D warnings -W clippy::pedantic -W clippy::missing_errors_doc \ + -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used \ + -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-lambda-boundaries + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: diff --git a/crates/rustack-auth/src/canonical.rs b/crates/rustack-auth/src/canonical.rs index 7c72b632..a819e04f 100644 --- a/crates/rustack-auth/src/canonical.rs +++ b/crates/rustack-auth/src/canonical.rs @@ -22,7 +22,7 @@ use crate::error::AuthError; /// The set of characters that must be percent-encoded in URI path segments. /// -/// Per AWS SigV4 spec, all characters except unreserved characters +/// Per AWS `SigV4` spec, all characters except unreserved characters /// (A-Z, a-z, 0-9, `-`, `_`, `.`, `~`) must be encoded. /// Forward slashes in the path are preserved (not encoded). const URI_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC @@ -116,7 +116,7 @@ pub fn build_canonical_uri(path: &str) -> String { /// Parameters are sorted by key name first, then by value for duplicate keys. /// The raw query string values are preserved as-is (no decode/re-encode) because /// different clients use different encoding rules when signing. For example, -/// AWS SDKs percent-encode `:` and `*` but minio-java (via OkHttp) leaves them +/// AWS SDKs percent-encode `:` and `*` but minio-java (via `OkHttp`) leaves them /// raw. The server must use the exact same encoding the client used for signing, /// which is whatever appears in the HTTP request. /// @@ -187,10 +187,10 @@ pub(crate) fn validate_unique_query_names(query: &str) -> Result<(), AuthError> Ok(()) } -/// Build the canonical query string following the AWS SigV4 specification. +/// Build the canonical query string following the AWS `SigV4` specification. /// /// Each parameter name and value is first percent-decoded (so already-encoded -/// input is not double-encoded) and then re-encoded using the SigV4 encoding +/// input is not double-encoded) and then re-encoded using the `SigV4` encoding /// rules: every character except the unreserved set (`A-Z`, `a-z`, `0-9`, `-`, /// `_`, `.`, `~`) is percent-encoded with uppercase hex. Parameters are then /// sorted by encoded name. Duplicate decoded names are rejected because S3 @@ -209,7 +209,7 @@ pub(crate) fn validate_unique_query_names(query: &str) -> Result<(), AuthError> /// signature collisions where tampered parameters still verify. /// /// A literal `+` is treated as a literal plus (re-encoded as `%2B`), per the -/// SigV4 encoding rules. Note that downstream decoders disagree on the +/// `SigV4` encoding rules. Note that downstream decoders disagree on the /// meaning of a raw `+` (S3 decodes a literal plus, API Gateway and Lambda /// decode a space), so verification callers must NOT attempt the normalized /// fallback when the wire query contains a raw `+`: no single normalized @@ -275,7 +275,7 @@ pub fn build_canonical_query_string_normalized(query: &str) -> Result String { sorted.join(";") } -/// Percent-encode a string using the AWS SigV4 encoding rules. +/// Percent-encode a string using the AWS `SigV4` encoding rules. /// /// Every character except the unreserved set (`A-Z`, `a-z`, `0-9`, `-`, `_`, /// `.`, `~`) is percent-encoded with uppercase hexadecimal digits. @@ -374,7 +374,7 @@ fn sigv4_encode(input: &str) -> String { utf8_percent_encode(input, URI_ENCODE_SET).to_string() } -/// URI-encode a single path segment using the AWS SigV4 encoding rules. +/// URI-encode a single path segment using the AWS `SigV4` encoding rules. fn uri_encode(input: &str) -> String { sigv4_encode(input) } diff --git a/crates/rustack-auth/src/credentials.rs b/crates/rustack-auth/src/credentials.rs index b7095120..1a0736ef 100644 --- a/crates/rustack-auth/src/credentials.rs +++ b/crates/rustack-auth/src/credentials.rs @@ -44,7 +44,7 @@ pub struct StaticCredentialProvider { } impl StaticCredentialProvider { - /// Create a new `StaticCredentialProvider` from an iterable of (access_key_id, secret_key) + /// Create a new `StaticCredentialProvider` from an iterable of (`access_key_id`, `secret_key`) /// pairs. pub fn new(credentials: impl IntoIterator) -> Self { Self { diff --git a/crates/rustack-auth/src/error.rs b/crates/rustack-auth/src/error.rs index 31218dae..e4449807 100644 --- a/crates/rustack-auth/src/error.rs +++ b/crates/rustack-auth/src/error.rs @@ -1,11 +1,21 @@ -//! Error types for SigV4 authentication. +//! Error types for `SigV4` authentication. //! //! All authentication failures are represented by [`AuthError`], which provides //! specific variants for each failure mode encountered during signature verification. +use digest::InvalidLength; + /// Errors that can occur during AWS Signature Version 4 authentication. #[derive(Debug, thiserror::Error)] pub enum AuthError { + /// The cryptographic provider rejected MAC initialization. + #[error("Cryptographic initialization failed")] + CryptoInitialization { + /// Original provider error, without key or request contents. + #[source] + source: InvalidLength, + }, + /// Strict authentication requires a credential provider. #[error("Strict authentication requires a credential provider")] MissingCredentialProvider, diff --git a/crates/rustack-auth/src/lib.rs b/crates/rustack-auth/src/lib.rs index 435bd06b..89c8c968 100644 --- a/crates/rustack-auth/src/lib.rs +++ b/crates/rustack-auth/src/lib.rs @@ -1,9 +1,9 @@ //! AWS Signature Version 2 and 4 request authentication for Rustack. //! -//! This crate provides SigV4 and SigV2 signature verification for incoming HTTP +//! This crate provides `SigV4` and `SigV2` signature verification for incoming HTTP //! requests to AWS-compatible services. It supports header-based authentication //! (via the `Authorization` header), presigned URL authentication (via query -//! parameters), and legacy SigV2 authentication (HMAC-SHA1). +//! parameters), and legacy `SigV2` authentication (HMAC-SHA1). //! //! # Overview //! @@ -29,12 +29,12 @@ //! //! # Modules //! -//! - [`canonical`] - Canonical request construction per the SigV4 specification +//! - [`canonical`] - Canonical request construction per the `SigV4` specification //! - [`credentials`] - Credential provider trait and in-memory implementation //! - [`error`] - Authentication error types //! - [`presigned`] - Presigned URL verification -//! - [`sigv2`] - Legacy SigV2 signature verification (HMAC-SHA1) -//! - [`sigv4`] - Main SigV4 signature verification logic +//! - [`sigv2`] - Legacy `SigV2` signature verification (HMAC-SHA1) +//! - [`sigv4`] - Main `SigV4` signature verification logic pub mod canonical; pub mod credentials; diff --git a/crates/rustack-auth/src/presigned.rs b/crates/rustack-auth/src/presigned.rs index b28175bb..053403c1 100644 --- a/crates/rustack-auth/src/presigned.rs +++ b/crates/rustack-auth/src/presigned.rs @@ -116,9 +116,9 @@ pub fn parse_presigned_params(query: &str) -> Result = credential.splitn(5, '/').collect(); - if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { + let [access_key_id, date, region, service, "aws4_request"] = cred_parts.as_slice() else { return Err(AuthError::InvalidCredential); - } + }; let expires: u64 = expires_str .parse() @@ -131,10 +131,10 @@ pub fn parse_presigned_params(query: &str) -> Result Result Result bool { + let signature_matches = |canonical_query: &str| -> Result { let canonical_request = format!( "{method}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n\n{signed_headers_str}\n{UNSIGNED_PAYLOAD}" ); let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); let string_to_sign = build_string_to_sign(&parsed.timestamp, &credential_scope, &canonical_hash); - let expected_signature = compute_signature(&signing_key, &string_to_sign); - bool::from(provided_bytes.ct_eq(expected_signature.as_bytes())) + let expected_signature = compute_signature(&signing_key, &string_to_sign)?; + Ok(bool::from(provided_bytes.ct_eq(expected_signature.as_bytes()))) }; debug!("Built presigned canonical request"); @@ -235,7 +236,7 @@ pub fn verify_presigned( // `X-Amz-Signature`). This matches clients that sign whatever encoding // appears in the URL. let canonical_query = build_canonical_query_string_without_signature(query); - if signature_matches(&canonical_query) { + if signature_matches(&canonical_query)? { debug!(access_key_id = %parsed.access_key_id, "Presigned URL verification succeeded"); return Ok(AuthResult { access_key_id: parsed.access_key_id, @@ -262,9 +263,10 @@ pub fn verify_presigned( // is impossible; the raw attempt above already failed, so fall through to // the signature mismatch below. let normalized_matches = !query.contains('+') - && build_canonical_query_string_without_signature_normalized(query).is_ok_and( - |normalized| normalized != canonical_query && signature_matches(&normalized), - ); + && match build_canonical_query_string_without_signature_normalized(query) { + Ok(normalized) if normalized != canonical_query => signature_matches(&normalized)?, + _ => false, + }; if normalized_matches { debug!(access_key_id = %parsed.access_key_id, "Presigned URL verification succeeded (normalized query string)"); return Ok(AuthResult { @@ -287,9 +289,9 @@ fn build_canonical_query_string_without_signature(query: &str) -> String { build_canonical_query_string(&filter_signature_param(query)) } -/// Build the SigV4-normalized canonical query string excluding `X-Amz-Signature`. +/// Build the `SigV4`-normalized canonical query string excluding `X-Amz-Signature`. /// -/// The remaining parameters are percent-decoded, re-encoded per the SigV4 spec, +/// The remaining parameters are percent-decoded, re-encoded per the `SigV4` spec, /// and sorted. See [`build_canonical_query_string_normalized`]. /// /// # Errors @@ -484,8 +486,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Attacker's rewrite: the same values attached to the opposite raw // names. The normalized canonical query (and signature) is unchanged. @@ -571,9 +573,10 @@ mod tests { &hex::encode(Sha256::digest(canonical.as_bytes())), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(), &to_sign, - ); + ) + .unwrap(); for wire_names in [ "list-type=1&list-type=2", "%6cist-type=1&list-type=2", @@ -641,7 +644,8 @@ mod tests { // Verify the signature computation is correct by manually // replicating the presigned URL flow. - let signing_key = derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(); // Build canonical request for the presigned URL test vector. let canonical_request = "GET\n/test.txt\nX-Amz-Algorithm=AWS4-HMAC-SHA256&\ @@ -662,7 +666,7 @@ mod tests { &canonical_hash, ); - let signature = compute_signature(&signing_key, &string_to_sign); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); assert_eq!( signature, "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404" @@ -698,8 +702,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Build the full query with signature. let full_query = format!("{query_without_sig}&X-Amz-Signature={signature}"); @@ -755,8 +759,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); let full_query = format!("{query_without_sig}&X-Amz-Signature={signature}"); let uri = format!("http://examplebucket.s3.amazonaws.com/test.txt?{full_query}"); @@ -805,8 +809,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Tamper: `%2B` becomes a raw `+` on the wire. let tampered_query = query_without_sig.replace("x=%2B", "x=+"); @@ -859,8 +863,8 @@ mod tests { let credential_scope = format!("{date}/us-east-1/s3/aws4_request"); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3"); - let signature = compute_signature(&signing_key, &string_to_sign); + let signing_key = derive_signing_key(TEST_SECRET_KEY, &date, "us-east-1", "s3").unwrap(); + let signature = compute_signature(&signing_key, &string_to_sign).unwrap(); // Tamper: `%20` becomes a raw `+` on the wire. let tampered_query = query_without_sig.replace("x=%20", "x=+"); diff --git a/crates/rustack-auth/src/sigv2.rs b/crates/rustack-auth/src/sigv2.rs index 7ca9eb5f..26b44988 100644 --- a/crates/rustack-auth/src/sigv2.rs +++ b/crates/rustack-auth/src/sigv2.rs @@ -1,6 +1,6 @@ //! AWS Signature Version 2 verification. //! -//! SigV2 is an older signing mechanism that uses HMAC-SHA1. The `Authorization` +//! `SigV2` is an older signing mechanism that uses HMAC-SHA1. The `Authorization` //! header has the format: //! //! ```text @@ -30,18 +30,18 @@ use crate::{credentials::CredentialProvider, error::AuthError, sigv4::AuthResult type HmacSha1 = Hmac; -/// Check whether the `Authorization` header uses SigV2 format (`AWS AKID:sig`). +/// Check whether the `Authorization` header uses `SigV2` format (`AWS AKID:sig`). #[must_use] pub fn is_sigv2(auth_header: &str) -> bool { auth_header.starts_with("AWS ") && !auth_header.starts_with("AWS4-") } -/// Verify an AWS SigV2-signed HTTP request. +/// Verify an AWS `SigV2`-signed HTTP request. /// /// # Errors /// /// Returns an [`AuthError`] if the header is malformed, the access key is not -/// found, or the signature does not match. +/// found, the MAC provider rejects initialization, or the signature does not match. pub fn verify_sigv2( parts: &http::request::Parts, credential_provider: &dyn CredentialProvider, @@ -63,7 +63,7 @@ pub fn verify_sigv2( debug!("Built SigV2 string to sign"); - let expected_signature = compute_sigv2_signature(&secret_key, &string_to_sign); + let expected_signature = compute_sigv2_signature(&secret_key, &string_to_sign)?; if provided_signature .as_bytes() @@ -83,7 +83,7 @@ pub fn verify_sigv2( } } -/// Parse a SigV2 `Authorization` header: `AWS AKID:Signature`. +/// Parse a `SigV2` `Authorization` header: `AWS AKID:Signature`. fn parse_sigv2_header(header: &str) -> Result<(String, String), AuthError> { let rest = header .strip_prefix("AWS ") @@ -98,7 +98,7 @@ fn parse_sigv2_header(header: &str) -> Result<(String, String), AuthError> { Ok((access_key_id.to_owned(), signature.to_owned())) } -/// Build the SigV2 string to sign from the request parts. +/// Build the `SigV2` string to sign from the request parts. /// /// ```text /// HTTP-Verb + "\n" + @@ -126,7 +126,7 @@ fn build_string_to_sign(parts: &http::request::Parts) -> String { format!("{method}\n{content_md5}\n{content_type}\n{date}\n{amz_headers}{resource}") } -/// Build the CanonicalizedAmzHeaders string. +/// Build the `CanonicalizedAmzHeaders` string. /// /// All x-amz-* headers are lowercased, sorted, and joined with newlines. /// Each header is formatted as `name:value\n`. @@ -155,7 +155,7 @@ fn build_canonicalized_amz_headers(parts: &http::request::Parts) -> String { result } -/// Build the CanonicalizedResource string. +/// Build the `CanonicalizedResource` string. /// /// This is the URI path plus any sub-resource query parameters (sorted). fn build_canonicalized_resource(parts: &http::request::Parts) -> String { @@ -228,13 +228,13 @@ fn build_canonicalized_resource(parts: &http::request::Parts) -> String { } } -/// Compute the SigV2 signature: Base64(HMAC-SHA1(secret, string_to_sign)). -fn compute_sigv2_signature(secret_key: &str, string_to_sign: &str) -> String { - let mut mac = - HmacSha1::new_from_slice(secret_key.as_bytes()).expect("HMAC can accept any key length"); +/// Compute the `SigV2` signature: Base64(HMAC-SHA1(secret, `string_to_sign`)). +fn compute_sigv2_signature(secret_key: &str, string_to_sign: &str) -> Result { + let mut mac = HmacSha1::new_from_slice(secret_key.as_bytes()) + .map_err(|source| AuthError::CryptoInitialization { source })?; mac.update(string_to_sign.as_bytes()); let result = mac.finalize().into_bytes(); - BASE64.encode(result) + Ok(BASE64.encode(result)) } /// Extract a header value as a string, returning empty string if missing. @@ -284,12 +284,27 @@ mod tests { assert!(parse_sigv2_header("NOTAWS key:sig").is_err()); } + #[test] + fn test_should_match_rfc2202_sha1_vectors() { + // https://www.rfc-editor.org/rfc/rfc2202.html section 3, cases 1 and 2. + let first = compute_sigv2_signature(&"\u{b}".repeat(20), "Hi There").unwrap(); + assert_eq!( + hex::encode(BASE64.decode(first).unwrap()), + "b617318655057264e28bc0b6fb378c8ef146be00" + ); + let second = compute_sigv2_signature("Jefe", "what do ya want for nothing?").unwrap(); + assert_eq!( + hex::encode(BASE64.decode(second).unwrap()), + "effcdf6ae5eb2fa2d27416d5f184df9c259a7c79" + ); + } + #[test] fn test_should_compute_sigv2_signature() { - let sig = compute_sigv2_signature("secret", "data"); + let sig = compute_sigv2_signature("secret", "data").unwrap(); assert_ne!(sig, ""); // Base64(HMAC-SHA1) produces a deterministic output. - let sig2 = compute_sigv2_signature("secret", "data"); + let sig2 = compute_sigv2_signature("secret", "data").unwrap(); assert_eq!(sig, sig2); } @@ -300,7 +315,7 @@ mod tests { // Build a simple GET request. let date = "Sat, 28 Feb 2026 12:00:00 GMT"; let string_to_sign = format!("GET\n\n\n{date}\n/test-bucket/"); - let signature = compute_sigv2_signature(TEST_SECRET_KEY, &string_to_sign); + let signature = compute_sigv2_signature(TEST_SECRET_KEY, &string_to_sign).unwrap(); let auth_header = format!("AWS {TEST_ACCESS_KEY}:{signature}"); diff --git a/crates/rustack-auth/src/sigv4.rs b/crates/rustack-auth/src/sigv4.rs index 1adb156c..ced54d58 100644 --- a/crates/rustack-auth/src/sigv4.rs +++ b/crates/rustack-auth/src/sigv4.rs @@ -1,6 +1,6 @@ //! AWS Signature Version 4 verification. //! -//! This module implements the core SigV4 signature verification flow: +//! This module implements the core `SigV4` signature verification flow: //! //! 1. Parse the `Authorization` header to extract the algorithm, credential scope, signed headers, //! and provided signature. @@ -32,7 +32,7 @@ const SUPPORTED_ALGORITHM: &str = "AWS4-HMAC-SHA256"; type HmacSha256 = Hmac; -/// The result of a successful SigV4 verification. +/// The result of a successful `SigV4` verification. #[derive(Debug, Clone)] pub struct AuthResult { /// The access key ID that signed the request. @@ -45,7 +45,7 @@ pub struct AuthResult { pub signed_headers: Vec, } -/// Parsed components of an AWS SigV4 `Authorization` header. +/// Parsed components of an AWS `SigV4` `Authorization` header. /// /// Format: /// ```text @@ -71,7 +71,7 @@ pub struct ParsedAuth { pub signature: String, } -/// Parse an AWS SigV4 `Authorization` header value into its components. +/// Parse an AWS `SigV4` `Authorization` header value into its components. /// /// # Errors /// @@ -108,25 +108,25 @@ pub fn parse_authorization_header(header: &str) -> Result // Parse credential: AKID/date/region/service/aws4_request let cred_parts: Vec<&str> = credential.splitn(5, '/').collect(); - if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { + let [access_key_id, date, region, service, "aws4_request"] = cred_parts.as_slice() else { return Err(AuthError::InvalidCredential); - } + }; let parsed_signed_headers: Vec = signed_headers.split(';').map(ToOwned::to_owned).collect(); Ok(ParsedAuth { algorithm: algorithm.to_owned(), - access_key_id: cred_parts[0].to_owned(), - date: cred_parts[1].to_owned(), - region: cred_parts[2].to_owned(), - service: cred_parts[3].to_owned(), + access_key_id: (*access_key_id).to_owned(), + date: (*date).to_owned(), + region: (*region).to_owned(), + service: (*service).to_owned(), signed_headers: parsed_signed_headers, signature: signature.to_owned(), }) } -/// Build the SigV4 string to sign. +/// Build the `SigV4` string to sign. /// /// Format: /// ```text @@ -157,7 +157,7 @@ pub fn build_string_to_sign( format!("{SUPPORTED_ALGORITHM}\n{timestamp}\n{credential_scope}\n{canonical_request_hash}") } -/// Derive the SigV4 signing key using HMAC-SHA256 chain. +/// Derive the `SigV4` signing key using HMAC-SHA256 chain. /// /// ```text /// DateKey = HMAC-SHA256("AWS4" + secret_key, date) @@ -170,33 +170,51 @@ pub fn build_string_to_sign( /// /// ``` /// use rustack_auth::sigv4::derive_signing_key; +/// # fn example() -> Result<(), rustack_auth::AuthError> { /// /// let key = derive_signing_key( /// "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", /// "20130524", /// "us-east-1", /// "s3", -/// ); +/// )?; /// assert!(!key.is_empty()); +/// # Ok(()) } /// ``` -#[must_use] -pub fn derive_signing_key(secret_key: &str, date: &str, region: &str, service: &str) -> Vec { - let date_key = hmac_sha256(format!("AWS4{secret_key}").as_bytes(), date.as_bytes()); - let date_region_key = hmac_sha256(&date_key, region.as_bytes()); - let date_region_service_key = hmac_sha256(&date_region_key, service.as_bytes()); +/// +/// # Errors +/// Returns [`AuthError::CryptoInitialization`] if the MAC provider rejects a key. +pub fn derive_signing_key( + secret_key: &str, + date: &str, + region: &str, + service: &str, +) -> Result, AuthError> { + let date_key = hmac_sha256(format!("AWS4{secret_key}").as_bytes(), date.as_bytes())?; + let date_region_key = hmac_sha256(&date_key, region.as_bytes())?; + let date_region_service_key = hmac_sha256(&date_region_key, service.as_bytes())?; hmac_sha256(&date_region_service_key, b"aws4_request") } /// Compute the HMAC-SHA256 signature of `data` using the given `signing_key`. /// /// Returns the hex-encoded signature. -#[must_use] -pub fn compute_signature(signing_key: &[u8], data: &str) -> String { - let sig = hmac_sha256(signing_key, data.as_bytes()); - hex::encode(sig) +/// +/// # Errors +/// Returns [`AuthError::CryptoInitialization`] if the MAC provider rejects the key. +/// +/// ``` +/// # fn example() -> Result<(), rustack_auth::AuthError> { +/// let signature = rustack_auth::sigv4::compute_signature(b"secret", "message")?; +/// assert_eq!(signature.len(), 64); +/// # Ok(()) } +/// ``` +pub fn compute_signature(signing_key: &[u8], data: &str) -> Result { + let sig = hmac_sha256(signing_key, data.as_bytes())?; + Ok(hex::encode(sig)) } -/// Verify an AWS SigV4-signed HTTP request. +/// Verify an AWS `SigV4`-signed HTTP request. /// /// This function: /// 1. Parses the `Authorization` header @@ -207,7 +225,7 @@ pub fn compute_signature(signing_key: &[u8], data: &str) -> String { /// /// The query string is verified exactly as it appears on the wire /// (raw-preserving). Unlike [`verify_s3_sigv4`], this does NOT attempt the -/// SigV4-normalized fallback: the normalized form asserts that alternate +/// `SigV4`-normalized fallback: the normalized form asserts that alternate /// wire encodings are equivalent, which is only true when the serving /// endpoint percent-decodes query parameters before acting on them. Whether /// the fallback is safe is a property of the serving endpoint, never of the @@ -222,6 +240,7 @@ pub fn compute_signature(signing_key: &[u8], data: &str) -> String { /// - The access key is not found /// - Required signed headers are missing /// - The signature does not match +/// - The MAC provider rejects initialization pub fn verify_sigv4( parts: &http::request::Parts, body_hash: &str, @@ -230,13 +249,13 @@ pub fn verify_sigv4( verify_sigv4_with_policy(parts, body_hash, credential_provider, false, false) } -/// Verify S3 SigV4, allowing the explicit unsigned-payload protocol exception. +/// Verify S3 `SigV4`, allowing the explicit unsigned-payload protocol exception. /// /// This verifies the seed signature only for streaming markers. Callers MUST additionally /// verify every chunk and signed trailer with [`StreamingVerifier`] before publishing data. /// /// In addition to the raw-preserving attempt, this accepts the -/// SigV4-normalized canonical query string as a fallback (see +/// `SigV4`-normalized canonical query string as a fallback (see /// [`build_canonical_query_string_normalized`]): spec-compliant S3 clients /// such as Transmit 5 sign the encoded form even when the wire carries the /// raw value. This endpoint entry point is the opt-in — S3 percent-decodes @@ -244,12 +263,13 @@ pub fn verify_sigv4( /// (`rustack-s3-http/src/router.rs::parse_query_params`), so the normalized /// equivalence holds. Do NOT use this entry point for an endpoint that reads /// query parameters from the raw query string without decoding (e.g. -/// CloudFront's `DeleteRealtimeLogConfig` reading `Name`): a verified +/// `CloudFront`'s `DeleteRealtimeLogConfig` reading `Name`): a verified /// signature could then authorize a different effective value than the one /// acted upon. Such endpoints must use [`verify_sigv4`]. /// /// # Errors -/// Returns an authentication error for invalid signatures or payload declarations. +/// Returns an authentication error for invalid signatures, payload declarations, +/// credential/header failures or MAC provider initialization failures. pub fn verify_s3_sigv4( parts: &http::request::Parts, body_hash: &str, @@ -324,26 +344,26 @@ fn verify_sigv4_with_policy( parsed.date, parsed.region, parsed.service ); let signing_key = - derive_signing_key(&secret_key, &parsed.date, &parsed.region, &parsed.service); + derive_signing_key(&secret_key, &parsed.date, &parsed.region, &parsed.service)?; let provided_bytes = parsed.signature.as_bytes(); // Check the provided signature against the canonical request built from // `query_string`. Returns true on a constant-time match. #[rustfmt::skip] - let signature_matches = |query_string: &str| -> bool { + let signature_matches = |query_string: &str| -> Result { let canonical_request = format!( "{method}\n{canonical_uri}\n{query_string}\n{canonical_headers}\n\n{signed_headers_str}\n{payload_hash}" ); let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); let string_to_sign = build_string_to_sign(×tamp, &credential_scope, &canonical_hash); - let expected_signature = compute_signature(&signing_key, &string_to_sign); - bool::from(provided_bytes.ct_eq(expected_signature.as_bytes())) + let expected_signature = compute_signature(&signing_key, &string_to_sign)?; + Ok(bool::from(provided_bytes.ct_eq(expected_signature.as_bytes()))) }; // First attempt: raw query string values preserved as-is. This matches // clients that sign whatever encoding appears on the wire (e.g. AWS SDKs // with pre-encoded values, minio-java via OkHttp with raw values). - if signature_matches(&canonical_query) { + if signature_matches(&canonical_query)? { debug!(access_key_id = %parsed.access_key_id, "Signature verification succeeded"); return Ok(AuthResult { access_key_id: parsed.access_key_id, @@ -377,9 +397,10 @@ fn verify_sigv4_with_policy( // the signature mismatch below. let normalized_matches = allow_normalized_fallback && !query.contains('+') - && build_canonical_query_string_normalized(query).is_ok_and(|normalized| { - normalized != canonical_query && signature_matches(&normalized) - }); + && match build_canonical_query_string_normalized(query) { + Ok(normalized) if normalized != canonical_query => signature_matches(&normalized)?, + _ => false, + }; if normalized_matches { debug!(access_key_id = %parsed.access_key_id, "Signature verification succeeded (normalized query string)"); return Ok(AuthResult { @@ -394,7 +415,7 @@ fn verify_sigv4_with_policy( Err(AuthError::SignatureDoesNotMatch) } -/// SigV4 streaming HMAC chain, including terminal chunks and signed trailers. +/// `SigV4` streaming HMAC chain, including terminal chunks and signed trailers. pub struct StreamingVerifier { key: Vec, timestamp: String, @@ -411,7 +432,7 @@ impl std::fmt::Debug for StreamingVerifier { impl StreamingVerifier { /// Authenticate the seed and initialize the chunk chain. /// # Errors - /// Returns seed signature or header validation failures. + /// Returns seed signature/header validation or MAC provider initialization failures. pub fn new( parts: &http::request::Parts, provider: &dyn CredentialProvider, @@ -421,7 +442,7 @@ impl StreamingVerifier { let parsed = parse_authorization_header(&header)?; let secret = provider.get_secret_key(&parsed.access_key_id)?; Ok(Self { - key: derive_signing_key(&secret, &parsed.date, &parsed.region, &parsed.service), + key: derive_signing_key(&secret, &parsed.date, &parsed.region, &parsed.service)?, timestamp: extract_header_value(parts, "x-amz-date")?, scope: format!( "{}/{}/{}/aws4_request", @@ -433,7 +454,8 @@ impl StreamingVerifier { /// Verify a data chunk, including the final zero-size chunk. /// # Errors - /// Rejects a missing, malformed or mismatching chunk signature. + /// Rejects a missing, malformed or mismatching chunk signature, or MAC provider + /// initialization failure. Failure leaves the previous signature unchanged. pub fn verify_chunk(&mut self, actual_sha256: &str, signature: &str) -> Result<(), AuthError> { let text = format!( "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", @@ -448,7 +470,8 @@ impl StreamingVerifier { /// Verify the canonical declared trailer block after the terminal chunk. /// # Errors - /// Returns an error if the trailer signature does not match. + /// Returns an error if the trailer signature does not match or MAC provider + /// initialization fails. Failure leaves the previous signature unchanged. pub fn verify_trailer(&mut self, canonical: &str, signature: &str) -> Result<(), AuthError> { let text = format!( "AWS4-HMAC-SHA256-TRAILER\n{}\n{}\n{}\n{}", @@ -461,7 +484,7 @@ impl StreamingVerifier { } fn verify_next(&mut self, text: &str, signature: &str) -> Result<(), AuthError> { - let expected = compute_signature(&self.key, text); + let expected = compute_signature(&self.key, text)?; if !bool::from(expected.as_bytes().ct_eq(signature.as_bytes())) { return Err(AuthError::SignatureDoesNotMatch); } @@ -537,10 +560,11 @@ fn collect_signed_headers<'a>( } /// Compute HMAC-SHA256 and return the raw bytes. -fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { - let mut mac = HmacSha256::new_from_slice(key).expect("HMAC can accept keys of any length"); +fn hmac_sha256(key: &[u8], data: &[u8]) -> Result, AuthError> { + let mut mac = HmacSha256::new_from_slice(key) + .map_err(|source| AuthError::CryptoInitialization { source })?; mac.update(data); - mac.finalize().into_bytes().to_vec() + Ok(mac.finalize().into_bytes().to_vec()) } /// Compute the SHA-256 hash of the given payload and return it as a hex string. @@ -585,10 +609,26 @@ mod tests { } #[test] - fn test_should_derive_signing_key_matching_aws_test_vector() { - let key = derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE); - // The signing key itself is not published as hex in the AWS docs, - // but we can verify it produces the correct signature when used. + fn test_should_match_rfc4231_sha256_vectors_with_short_and_long_keys() { + // https://www.rfc-editor.org/rfc/rfc4231.html sections 4.2 and 4.7. + assert_eq!( + compute_signature(&[0x0b; 20], "Hi There").unwrap(), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7" + ); + assert_eq!( + compute_signature( + &[0xaa; 131], + "Test Using Larger Than Block-Size Key - Hash Key First" + ) + .unwrap(), + "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54" + ); + } + + #[test] + fn test_should_derive_a_32_byte_signing_key() { + let key = + derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE).unwrap(); assert_eq!(key.len(), 32); // SHA-256 produces 32 bytes } @@ -651,7 +691,8 @@ mod tests { #[test] fn test_should_compute_correct_signature_for_aws_get_object_example() { // Full end-to-end test using the AWS GET Object example. - let signing_key = derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE); + let signing_key = + derive_signing_key(TEST_SECRET_KEY, TEST_DATE, TEST_REGION, TEST_SERVICE).unwrap(); #[rustfmt::skip] let string_to_sign = "AWS4-HMAC-SHA256\n\ @@ -659,7 +700,7 @@ mod tests { 20130524/us-east-1/s3/aws4_request\n\ 7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"; - let signature = compute_signature(&signing_key, string_to_sign); + let signature = compute_signature(&signing_key, string_to_sign).unwrap(); assert_eq!( signature, "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" @@ -684,9 +725,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &text, - ); + ) + .unwrap(); parts.headers.insert( "authorization", format!( @@ -766,7 +808,7 @@ mod tests { hash_payload(b""), digest ); - let signature = compute_signature(&verifier.key, &text); + let signature = compute_signature(&verifier.key, &text).unwrap(); assert!( verifier .verify_chunk(&hash_payload(b"wrong"), &signature) @@ -877,9 +919,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ @@ -929,9 +972,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", service), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", service).unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ Credential={TEST_ACCESS_KEY}/20130524/us-east-1/{service}/aws4_request,\ @@ -1067,9 +1111,10 @@ mod tests { &hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3").unwrap(), &string_to_sign, - ); + ) + .unwrap(); let auth_value = format!( "AWS4-HMAC-SHA256 \ diff --git a/crates/rustack-lambda-core/src/config.rs b/crates/rustack-lambda-core/src/config.rs index ed8fbf39..16291d02 100644 --- a/crates/rustack-lambda-core/src/config.rs +++ b/crates/rustack-lambda-core/src/config.rs @@ -50,6 +50,11 @@ impl LambdaConfig { /// - `LAMBDA_IDLE_TIMEOUT_SECS` (default: `600`) /// - `LAMBDA_INIT_TIMEOUT_SECS` (default: `5`) /// - `LAMBDA_SQUIB_*` variables documented by [`SquibExecutorConfig`]. + /// + /// # Errors + /// + /// Rejects malformed environment values, invalid budgets, unavailable Docker selection, + /// conflicting executor settings or invalid Squib configuration. pub fn from_env() -> Result { Self::from_lookup(|key| match rustack_core::settings::var(key) { Ok(value) => Some(value), @@ -59,6 +64,11 @@ impl LambdaConfig { } /// Parse an explicit environment lookup without changing process globals. + /// + /// # Errors + /// + /// Rejects malformed scalar values, invalid budgets/identifiers, conflicting executor + /// settings, unavailable Docker selection or invalid Squib configuration. pub fn from_lookup( lookup: impl Fn(&str) -> Option, ) -> Result { diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index 2235f837..2adadb35 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -94,7 +94,7 @@ pub trait Executor: std::fmt::Debug + Send + Sync + 'static { /// Stop all warm instances and release resources. /// /// # Errors - /// Returns CleanupFailed if owned resource cleanup cannot be confirmed. + /// Returns `CleanupFailed` if owned resource cleanup cannot be confirmed. /// Dropping a shutdown waiter must retain the underlying cleanup operation. /// /// ```no_run diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 3f97da61..22694953 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -482,7 +482,14 @@ where match reader.read(&mut chunk).await { Ok(0) => break, Ok(n) => { - buf.lock().push(&chunk[..n]); + let Some(bytes) = chunk.get(..n) else { + warn!( + stream = label, + "lambda runtime reader returned an invalid byte count" + ); + break; + }; + buf.lock().push(bytes); debug!(stream = label, bytes = n, "lambda runtime output"); } Err(e) => { diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index 5c03a388..790a9682 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -67,6 +67,10 @@ pub struct SquibExecutorConfig { impl SquibExecutorConfig { /// Read Squib executor configuration from environment variables. + /// + /// # Errors + /// + /// Rejects invalid paths, instance IDs, ports, timeouts, response limits or run budgets. pub fn from_env() -> Result { Self::from_env_reader(|key| match rustack_core::settings::var(key) { Ok(value) => Some(value), diff --git a/crates/rustack-lambda-core/src/invocation.rs b/crates/rustack-lambda-core/src/invocation.rs index a4e6e099..2375297d 100644 --- a/crates/rustack-lambda-core/src/invocation.rs +++ b/crates/rustack-lambda-core/src/invocation.rs @@ -224,7 +224,7 @@ impl InvocationHandle { /// Request cancellation and await confirmed cleanup, independently of the call deadline. /// /// # Errors - /// Returns CleanupFailed when the cleanup waiter expires or settlement is uncertain. + /// Returns `CleanupFailed` when the cleanup waiter expires or settlement is uncertain. /// Cleanup remains owned after this method returns an error or is dropped. /// /// ```no_run diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index ad446988..48d3a17a 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -495,6 +495,11 @@ impl RustackLambda { /// /// Validates the input, stores the deployment package, and inserts /// the function record into the store. + /// + /// # Errors + /// + /// Rejects invalid deployment settings, duplicate functions, unavailable image execution, + /// exhausted deployment admission, or ZIP/S3 staging and storage failures. #[allow(clippy::too_many_lines)] pub async fn create_function( &self, @@ -717,6 +722,11 @@ impl RustackLambda { } /// Get function information including configuration, code location, and tags. + /// + /// # Errors + /// + /// Rejects malformed function references or qualifiers, and missing functions, versions or + /// aliases. pub fn get_function( &self, function_ref: &str, @@ -758,6 +768,11 @@ impl RustackLambda { } /// Get function configuration only. + /// + /// # Errors + /// + /// Rejects malformed function references or qualifiers, and missing functions, versions or + /// aliases. pub fn get_function_configuration( &self, function_ref: &str, @@ -775,6 +790,10 @@ impl RustackLambda { } /// Get the code signing config association for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn get_function_code_signing_config( &self, function_ref: &str, @@ -789,6 +808,13 @@ impl RustackLambda { } /// Update function code. + /// + /// # Errors + /// + /// Rejects invalid or immutable references, incompatible code sources/settings, stale + /// revisions, in-progress deployments or unavailable/exhausted image preparation. ZIP/S3 + /// staging and validation failures propagate; `DryRun` preserves metadata and performs no + /// deployment effects. pub async fn update_function_code( &self, function_ref: &str, @@ -898,6 +924,11 @@ impl RustackLambda { } /// Update function configuration (handler, runtime, env vars, etc.). + /// + /// # Errors + /// + /// Rejects invalid or immutable references/settings, stale revisions, in-progress deployments, + /// missing functions or changes that violate the stored execution configuration. pub fn update_function_configuration( &self, function_ref: &str, @@ -984,6 +1015,12 @@ impl RustackLambda { } /// Delete a function. + /// + /// # Errors + /// + /// Rejects malformed references, nonnumeric deletion qualifiers, missing functions/versions, + /// or artifact-root preflight failures. Final cleanup revalidates the root and can fail + /// after metadata removal if the root changes or filesystem inspection fails. pub async fn delete_function( &self, function_ref: &str, @@ -1068,7 +1105,7 @@ impl RustackLambda { /// /// # Errors /// Rejects invalid payload/reference, missing resources, unavailable execution, - /// exhausted capacity and mapped backend errors. DryRun starts no runtime. + /// exhausted capacity and mapped backend errors. `DryRun` starts no runtime. pub async fn invoke( &self, function_ref: &str, @@ -1301,6 +1338,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Publish a version from `$LATEST`. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions, stale revision/hash preconditions, + /// unsuccessful deployments, unavailable image leases or invalid stored publication settings. pub fn publish_version( &self, function_ref: &str, @@ -1354,6 +1396,10 @@ impl RustackLambda { } /// List versions of a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_versions_by_function( &self, function_ref: &str, @@ -1395,6 +1441,11 @@ impl RustackLambda { } /// Create an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions, duplicate aliases, invalid target versions + /// or targets that do not exist; invalid resulting store records also reject. pub fn create_alias( &self, function_ref: &str, @@ -1464,6 +1515,10 @@ impl RustackLambda { } /// Get an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions and missing aliases. pub fn get_alias( &self, function_ref: &str, @@ -1507,6 +1562,11 @@ impl RustackLambda { } /// Update an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions/aliases, invalid target versions + /// or targets that do not exist; invalid resulting store records also reject. pub fn update_alias( &self, function_ref: &str, @@ -1582,6 +1642,10 @@ impl RustackLambda { } /// Delete an alias. + /// + /// # Errors + /// + /// Rejects malformed references, missing functions and missing aliases. pub fn delete_alias( &self, function_ref: &str, @@ -1603,6 +1667,10 @@ impl RustackLambda { } /// List aliases for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_aliases( &self, function_ref: &str, @@ -1667,6 +1735,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Add a permission to a function's resource policy. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, missing required permission + /// fields or duplicate statement IDs; store validation failures also propagate. pub fn add_permission( &self, function_ref: &str, @@ -1743,6 +1816,10 @@ impl RustackLambda { } /// Remove a permission from a function's resource policy. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent statement IDs. pub fn remove_permission( &self, function_ref: &str, @@ -1770,6 +1847,10 @@ impl RustackLambda { } /// Get the resource policy for a function. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and empty resource policies. pub fn get_policy( &self, function_ref: &str, @@ -1815,6 +1896,11 @@ impl RustackLambda { } /// Tag a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references, missing functions, a merged tag count above50, + /// or resulting records that fail store validation. pub fn tag_resource( &self, resource_arn: &str, @@ -1843,6 +1929,10 @@ impl RustackLambda { } /// Remove tags from a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references, missing functions and store validation failures. pub fn untag_resource( &self, resource_arn: &str, @@ -1860,6 +1950,10 @@ impl RustackLambda { } /// List tags for a resource. + /// + /// # Errors + /// + /// Rejects malformed resource references and missing functions. pub fn list_tags(&self, resource_arn: &str) -> Result { let name = Self::extract_function_name_from_arn(resource_arn)?; let record = self.get_record(&name)?; @@ -1899,6 +1993,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Create a function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, duplicate URL configurations + /// or resulting records that fail store validation. pub fn create_function_url_config( &self, function_ref: &str, @@ -1966,6 +2065,10 @@ impl RustackLambda { } /// Get function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent URL configurations. pub fn get_function_url_config( &self, function_ref: &str, @@ -2001,6 +2104,11 @@ impl RustackLambda { } /// Update function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions, absent URL configurations + /// or resulting records that fail store validation. pub fn update_function_url_config( &self, function_ref: &str, @@ -2054,6 +2162,10 @@ impl RustackLambda { } /// Delete function URL configuration. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent URL configurations. pub fn delete_function_url_config( &self, function_ref: &str, @@ -2080,6 +2192,10 @@ impl RustackLambda { } /// List function URL configurations. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_function_url_configs( &self, function_ref: &str, @@ -2114,6 +2230,11 @@ impl RustackLambda { // --------------------------------------------------------------- /// Publish a new layer version. + /// + /// # Errors + /// + /// Rejects empty/oversized layer names, oversized license text, invalid inline base64, + /// or a layer version that disappears before its assigned metadata is updated. pub fn publish_layer_version( &self, layer_name: &str, @@ -2213,6 +2334,10 @@ impl RustackLambda { } /// Get a specific layer version. + /// + /// # Errors + /// + /// Returns `InvalidParameter` when the requested layer version does not exist. pub fn get_layer_version( &self, layer_name: &str, @@ -2229,6 +2354,10 @@ impl RustackLambda { } /// Get a layer version by its full ARN. + /// + /// # Errors + /// + /// Rejects malformed layer-version ARNs and missing layer versions. pub fn get_layer_version_by_arn( &self, arn: &str, @@ -2238,6 +2367,10 @@ impl RustackLambda { } /// List versions of a layer. + /// + /// # Errors + /// + /// This implementation returns no errors; an unknown layer produces an empty page. pub fn list_layer_versions( &self, layer_name: &str, @@ -2355,6 +2488,10 @@ impl RustackLambda { } /// Delete a layer version. + /// + /// # Errors + /// + /// This implementation returns no errors; deleting an absent version succeeds. pub fn delete_layer_version( &self, layer_name: &str, @@ -2367,6 +2504,10 @@ impl RustackLambda { } /// Add a permission to a layer version's resource policy. + /// + /// # Errors + /// + /// Rejects missing required fields, duplicate statement IDs or missing layer versions. pub fn add_layer_version_permission( &self, layer_name: &str, @@ -2444,6 +2585,10 @@ impl RustackLambda { } /// Get the resource policy for a layer version. + /// + /// # Errors + /// + /// Rejects missing layer versions and empty layer resource policies. pub fn get_layer_version_policy( &self, layer_name: &str, @@ -2490,6 +2635,10 @@ impl RustackLambda { } /// Remove a permission from a layer version's resource policy. + /// + /// # Errors + /// + /// Rejects missing layer versions and absent statement IDs. pub fn remove_layer_version_permission( &self, layer_name: &str, @@ -2793,6 +2942,10 @@ impl RustackLambda { // ----------------------------------------------------------------- /// Put (set) reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references, missing functions and store validation failures. pub fn put_function_concurrency( &self, function_ref: &str, @@ -2811,6 +2964,10 @@ impl RustackLambda { } /// Get reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn get_function_concurrency( &self, function_ref: &str, @@ -2823,6 +2980,10 @@ impl RustackLambda { } /// Delete reserved concurrency for a function. + /// + /// # Errors + /// + /// Rejects malformed function references, missing functions and store validation failures. pub fn delete_function_concurrency( &self, function_ref: &str, @@ -2840,6 +3001,10 @@ impl RustackLambda { // ----------------------------------------------------------------- /// Put (create/replace) an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and store validation failures. pub fn put_function_event_invoke_config( &self, function_ref: &str, @@ -2874,6 +3039,11 @@ impl RustackLambda { } /// Get an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and absent event invoke + /// configurations. pub fn get_function_event_invoke_config( &self, function_ref: &str, @@ -2897,6 +3067,11 @@ impl RustackLambda { } /// Update (merge) an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions or resulting records + /// that fail store validation. An absent configuration is created before merging. pub fn update_function_event_invoke_config( &self, function_ref: &str, @@ -2942,6 +3117,11 @@ impl RustackLambda { } /// Delete an event invoke config for a function qualifier. + /// + /// # Errors + /// + /// Rejects malformed references/qualifiers, missing functions and store validation failures. + /// Deleting an absent event invoke configuration succeeds. pub fn delete_function_event_invoke_config( &self, function_ref: &str, @@ -2959,6 +3139,10 @@ impl RustackLambda { } /// List all event invoke configs for a function. + /// + /// # Errors + /// + /// Rejects malformed function references and missing functions. pub fn list_function_event_invoke_configs( &self, function_ref: &str, diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index 11ca8689..4f9625dc 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -38,6 +38,10 @@ pub fn resolve_function_ref( pub struct FunctionName(String); impl FunctionName { /// Validate an AWS function name. + /// + /// # Errors + /// Rejects empty names, names above64 bytes or characters outside ASCII letters/digits, hyphen + /// and underscore. pub fn parse(value: &str) -> Result { if value.is_empty() || value.len() > 64 @@ -83,6 +87,9 @@ impl FunctionArn { pub struct Qualifier(String); impl Qualifier { /// Validate a version or alias. + /// + /// # Errors + /// Rejects invalid/oversized aliases and zero or overflowing numeric versions. pub fn parse(value: &str) -> Result { if value != "$LATEST" && (value.is_empty() @@ -268,14 +275,14 @@ pub fn layer_version_arn(region: &str, account_id: &str, layer_name: &str, versi pub fn parse_layer_version_arn(arn: &str) -> Result<(String, u64), LambdaServiceError> { let parts: Vec<&str> = arn.split(':').collect(); // Expected: arn:aws:lambda:region:account:layer:name:version = 8 parts - if parts.len() < 8 || parts[0] != "arn" || parts[2] != "lambda" || parts[5] != "layer" { + let ["arn", _, "lambda", _, _, "layer", name, version, ..] = parts.as_slice() else { return Err(LambdaServiceError::InvalidArn { arn: arn.to_owned(), }); - } + }; - let name = parts[6].to_owned(); - let version: u64 = parts[7] + let name = (*name).to_owned(); + let version: u64 = version .parse() .map_err(|_| LambdaServiceError::InvalidArn { arn: arn.to_owned(), diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index 0d925f5d..1e9994de 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -189,7 +189,7 @@ pub struct VersionRecord { pub image_config: Option, /// Logging configuration. pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. pub snap_start: Option, } @@ -461,7 +461,7 @@ pub struct LayerStoreSnapshot { pub struct EventSourceMappingRecord { /// Unique identifier for the mapping. pub uuid: String, - /// ARN of the event source (e.g., SQS queue, DynamoDB stream). + /// ARN of the event source (e.g., SQS queue, `DynamoDB` stream). pub event_source_arn: String, /// ARN of the target Lambda function. pub function_arn: String, @@ -713,7 +713,7 @@ pub struct VersionRecordSnapshot { pub image_config: Option, /// Logging configuration. pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. pub snap_start: Option, } @@ -994,12 +994,20 @@ impl FunctionStore { } /// Get a clone of a function record by name. + /// + /// # Errors + /// + /// Rejects an invalid logical function name; an absent valid name returns `None`. pub fn get(&self, name: &str) -> Result, LambdaServiceError> { crate::resolver::FunctionName::parse(name)?; Ok(self.functions.get(name).map(|r| r.value().clone())) } /// Check whether a function exists. + /// + /// # Errors + /// + /// Rejects an invalid logical function name. pub fn contains(&self, name: &str) -> Result { crate::resolver::FunctionName::parse(name)?; Ok(self.functions.contains_key(name)) @@ -1087,6 +1095,10 @@ impl FunctionStore { /// Remove a function from the store. /// /// Returns the removed record, or `None` if it did not exist. + /// + /// # Errors + /// + /// Rejects an invalid logical function name before changing the store. pub fn remove(&self, name: &str) -> Result, LambdaServiceError> { crate::resolver::FunctionName::parse(name)?; let removed = self.functions.remove(name).map(|(_, v)| v); @@ -1264,12 +1276,23 @@ impl FunctionStore { /// /// Immutable artifacts are retained until store drop so published/in-flight /// references remain valid. No name-derived directory is ever deleted. + /// + /// # Errors + /// + /// Rejects an invalid function name, a symlink/non-directory artifact root, or + /// filesystem inspection failures. pub async fn cleanup_code(&self, function_name: &str) -> Result<(), LambdaServiceError> { crate::resolver::FunctionName::parse(function_name)?; self.validate_root().await } /// Check an immutable artifact before handing it to an executor. + /// + /// # Errors + /// + /// Rejects unowned artifacts, replaced roots, symlinked artifact components/bootstrap, + /// paths outside the root, or filesystem inspection/canonicalization failures. + /// An absent bootstrap is accepted here; execution performs its own readiness check. pub async fn validate_artifact(&self, path: &Path) -> Result<(), LambdaServiceError> { self.validate_root().await?; let owned_root = self @@ -1366,6 +1389,11 @@ impl FunctionStore { } /// Reject a symlink at the operator-selected artifact root. + /// + /// # Errors + /// + /// Rejects a symlink/non-directory root or filesystem inspection errors; an absent + /// root is valid before first staging. pub async fn validate_root(&self) -> Result<(), LambdaServiceError> { match tokio::fs::symlink_metadata(&self.code_dir).await { Ok(meta) if meta.file_type().is_symlink() || !meta.is_dir() => { @@ -1429,7 +1457,7 @@ const MAX_EXTRACTED_SIZE: u64 = 250 * 1024 * 1024; /// and archives whose entries would expand beyond [`MAX_EXTRACTED_SIZE`] /// (zip bombs). ZIP and filesystem failures are always returned to the caller. /// -/// Synchronous std::fs is intentional: this runs inside `spawn_blocking` and +/// Synchronous `std::fs` is intentional: this runs inside `spawn_blocking` and /// the `zip` crate's reader API is itself blocking, so wrapping each I/O in /// tokio would only add overhead. #[allow(clippy::disallowed_methods, clippy::disallowed_types)] diff --git a/crates/rustack-lambda-http/src/dispatch.rs b/crates/rustack-lambda-http/src/dispatch.rs index 5f53c776..1e1c1645 100644 --- a/crates/rustack-lambda-http/src/dispatch.rs +++ b/crates/rustack-lambda-http/src/dispatch.rs @@ -32,6 +32,8 @@ pub trait LambdaHandler: Send + Sync + 'static { } /// Dispatch a Lambda operation to the handler. +/// # Errors +/// Returns the handler error for invalid requests, unavailable resources or failed operations. pub async fn dispatch_operation( handler: &H, op: LambdaOperation, diff --git a/crates/rustack-lambda-http/src/lib.rs b/crates/rustack-lambda-http/src/lib.rs index f8e5ee76..082cf6f4 100644 --- a/crates/rustack-lambda-http/src/lib.rs +++ b/crates/rustack-lambda-http/src/lib.rs @@ -7,7 +7,7 @@ //! - **Service**: Hyper `Service` implementation for the Lambda protocol //! - **Response helpers**: JSON success/error response formatting //! -//! Unlike `awsJson1.1` services (SSM, DynamoDB), Lambda uses REST-style +//! Unlike `awsJson1.1` services (SSM, `DynamoDB`), Lambda uses REST-style //! routing where each operation has a unique method + path combination. //! Path parameters (e.g., `{FunctionName}`) are extracted and passed to //! the handler. diff --git a/crates/rustack-lambda-http/src/router.rs b/crates/rustack-lambda-http/src/router.rs index 5c343f46..4afffc46 100644 --- a/crates/rustack-lambda-http/src/router.rs +++ b/crates/rustack-lambda-http/src/router.rs @@ -115,7 +115,7 @@ fn normalize_date_prefix(path: &str) -> Option { let (date_part, rest) = trimmed.split_once('/')?; // Validate it looks like a date: YYYY-MM-DD (10 chars). - if date_part.len() != 10 || date_part.as_bytes()[4] != b'-' || date_part.as_bytes()[7] != b'-' { + if !matches!(date_part.as_bytes(), [_, _, _, _, b'-', _, _, b'-', _, _]) { return None; } diff --git a/crates/rustack-lambda-model/src/input.rs b/crates/rustack-lambda-model/src/input.rs index 17e741df..544c4bde 100644 --- a/crates/rustack-lambda-model/src/input.rs +++ b/crates/rustack-lambda-model/src/input.rs @@ -71,7 +71,7 @@ pub struct CreateFunctionInput { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, /// Code signing config ARN (stored, not enforced). @@ -164,7 +164,7 @@ pub struct UpdateFunctionConfigurationInput { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart configuration. + /// `SnapStart` configuration. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, } @@ -313,7 +313,7 @@ pub struct PublishLayerVersionInput { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct CreateEventSourceMappingInput { - /// ARN of the event source (e.g., SQS queue, DynamoDB stream, Kinesis stream). + /// ARN of the event source (e.g., SQS queue, `DynamoDB` stream, Kinesis stream). pub event_source_arn: String, /// Function name or ARN. pub function_name: String, diff --git a/crates/rustack-lambda-model/src/types.rs b/crates/rustack-lambda-model/src/types.rs index b10e5068..8dd54d6a 100644 --- a/crates/rustack-lambda-model/src/types.rs +++ b/crates/rustack-lambda-model/src/types.rs @@ -97,7 +97,7 @@ pub enum PackageType { /// Function architecture. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum Architecture { - /// x86_64 architecture. + /// `x86_64` architecture. #[serde(rename = "x86_64")] #[default] X86_64, @@ -425,7 +425,7 @@ pub struct LoggingConfig { pub log_group: Option, } -/// SnapStart configuration (stored, not enforced). +/// `SnapStart` configuration (stored, not enforced). #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct SnapStart { @@ -434,7 +434,7 @@ pub struct SnapStart { pub apply_on: Option, } -/// SnapStart response. +/// `SnapStart` response. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct SnapStartResponse { @@ -627,7 +627,7 @@ pub struct FunctionConfiguration { /// Logging configuration. #[serde(skip_serializing_if = "Option::is_none")] pub logging_config: Option, - /// SnapStart response. + /// `SnapStart` response. #[serde(skip_serializing_if = "Option::is_none")] pub snap_start: Option, /// Image configuration response. diff --git a/crates/rustack-s3-http/src/upload.rs b/crates/rustack-s3-http/src/upload.rs index 29bed563..72d46abd 100644 --- a/crates/rustack-s3-http/src/upload.rs +++ b/crates/rustack-s3-http/src/upload.rs @@ -583,7 +583,7 @@ mod tests { ] { let provider = StaticCredentialProvider::new(vec![("key".to_owned(), "secret".to_owned())]); - let key = derive_signing_key("secret", "20260101", "us-east-1", "s3"); + let key = derive_signing_key("secret", "20260101", "us-east-1", "s3").unwrap(); let canonical = rustack_auth::canonical::build_canonical_request( "PUT", "/bucket/key", @@ -599,7 +599,8 @@ mod tests { "20260101/us-east-1/s3/aws4_request", &rustack_auth::hash_payload(canonical.as_bytes()), ), - ); + ) + .unwrap(); let make_parts = || { let mut request = parts(marker); if marker.ends_with("-TRAILER") { @@ -633,6 +634,7 @@ mod tests { rustack_auth::hash_payload(data) ), ) + .unwrap() }; let first = sign(&seed, b"hello"); let terminal = sign(&first, b""); @@ -644,7 +646,8 @@ mod tests { aws4_request\n{terminal}\n{}", rustack_auth::hash_payload(b"x-amz-checksum-crc32:NhCmhg==\n") ), - ); + ) + .unwrap(); format!("x-amz-checksum-crc32:NhCmhg==\r\nx-amz-trailer-signature:{signature}\r\n") } else { String::new() diff --git a/crates/rustack-sqs-http/src/service.rs b/crates/rustack-sqs-http/src/service.rs index fc77b306..dccd0bd0 100644 --- a/crates/rustack-sqs-http/src/service.rs +++ b/crates/rustack-sqs-http/src/service.rs @@ -248,9 +248,10 @@ mod tests { &rustack_auth::hash_payload(canonical.as_bytes()), ); let signature = compute_signature( - &derive_signing_key("secret", "20260101", "us-east-1", "sqs"), + &derive_signing_key("secret", "20260101", "us-east-1", "sqs").unwrap(), &text, - ); + ) + .unwrap(); request.headers_mut().insert( "authorization", format!( diff --git a/specs/index.md b/specs/index.md index 791bc48c..2f843c5e 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 392d74eb..4aa05c68 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -472,6 +472,14 @@ M4 adds Docker execution of provided.al2023 ZIPs using an operator-pinned OS-onl Apply AGENTS.md § Error Handling/Async/Type/Safety: thiserror sources; retain async-trait for the existing dyn Executor and document object safety; concrete internal actors use ordinary async methods. Use Bytes/Arc revisions, validated artifact enums, redacted Debug, checked memory/deadline arithmetic, forbid unsafe, IO deadlines and supervised JoinSet tasks. Serde control envelopes use camelCase; Runtime API uses AWS headers. Validate raw lengths before parsing. Tests use actual RIC images and cancellation/failure matrices. Tracing contains IDs, durations and classes, not payloads. Avoid per-invoke pull/inspect. Public APIs document Errors and examples; operational docs cover both deployment topologies. +### 7.1 Strict production boundary closure (B03) + +Before P1 closes, `make test-lambda-boundaries` must pass strict production-library Clippy for the Lambda model/HTTP/core, shared core and authentication crates, plus their unit/doc regressions. Apply pedantic, missing Errors docs, indexing/slicing, unwrap/expect and panic checks without new ignores. Document actual method-specific error paths rather than generic boilerplate. Replace guarded external-input indexing with exhaustive slice patterns or checked access, preserving the existing protocol profile; this work does not claim broader layer/IAM compatibility. + +The existing RustCrypto HMAC0.13 / digest0.11 constructors formally return Result even though HMAC accepts arbitrary key lengths ([upstream API](https://docs.rs/hmac/0.13.0/hmac/)). Remove production expect calls by propagating a typed AuthError::CryptoInitialization with the original InvalidLength source. Public derive_signing_key and compute_signature return Result; update direct callers/examples in the same change, with no deprecated infallible shim or fabricated signature fallback. Preserve library HMAC computation and constant-time comparisons; do not hand-roll key normalization to sidestep the fallible API. Header/presigned/streaming verification propagates this error without treating it as a mismatch or trying another signing interpretation. Existing raw-versus-S3-normalized eligibility rules stay unchanged. Independent published HMAC vectors and existing streaming/query tamper regressions pin output semantics. No new crypto dependency/version is needed. + +Native log draining uses checked access for the successful AsyncRead length and terminates on an invalid reader count with a static diagnostic. This cannot bypass existing child/API retirement. These changes remove the known baseline boundary debt; they do not enable Docker or establish P1 acceptance by themselves. + Use the rechecked Bollard0.21.1 generated-type/API contracts through model-only bollard-stubs; the bounded transport decision and quotas are authoritative in §3.6/D19. Reuse Hyper/Tokio/Bytes/ciborium. TLS transport is N/A for this release; future TLS must use rustls + aws-lc-rs rather than enabling a ring-backed SSL default. ## 8. Cross-references diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 4c4851da..44f0286a 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -94,6 +94,8 @@ Retained deployment ownership (§2.11) now has19 additional actor/provider cases The deployment decoder (§2.12) has exact-limit, escaped/duplicate-key, wrong-path blob, malformed/unknown-field and depth/member/string/key/metadata tests. Inline base64 covers canonical padding and the exact50MiB decoded boundary without allocation of decoded bytes. Capacity remains owned after waiter cancellation and after a result becomes ready; four concurrent admitted inline validators reuse permits. Worker panic releases capacity and surfaces a bounded error; capacity429 must not be downgraded to400. Current395 core tests and101 doctests pass; workspace2,072 tests pass,401 resource-gated cases remain separately selected. +`make test-lambda-boundaries` runs unit/integration/doctest regressions and strict production-library Clippy for authentication, shared core, Lambda model/HTTP/core and shared runtime. Require pedantic, missing Errors docs, doc formatting, unwrap/expect, indexing/slicing and panic checks without new ignores. Current catalog has659 unit tests and165 doctests (824 passes); seven Docker cases remain resource-gated and must run via their explicit real-daemon targets for executor qualification. Four fixed RFC2202/RFC4231 HMAC vectors complement existing signature/query/streaming regressions; changing signing helpers to Result must preserve every computed output and propagate provider errors rather than treating them as mismatches. This closes B03 for these production libraries, not every repository test or service. + ## 4. Security acceptance | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 877b5865..e8b93a7f 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -71,7 +71,7 @@ Artifacts: [relay spike](../docs/research/spike-lambda-container-relay.md) and [ Add appropriate Makefile fixture-image/runtime build and `test-lambda-docker` targets. Update workspace dependencies/features once, auditing bollard and the relay artifact. Remove obsolete fallback branches rather than deprecating unavailable execution paths. Keep default disabled, Auto and Native/Squib behavior explicit. -**Exit:** ordinary SDK invokes a real image on the P0 lanes, returns computed output, distinguishes failures, preserves published code across updates, and cleans/quarantines every failed allocation under bounded ownership. Existing Lambda quality gates pass. No requirement is satisfied using a noop/echo executor. +**Exit:** ordinary SDK invokes a real image on the P0 lanes, returns computed output, distinguishes failures, preserves published code across updates, and cleans/quarantines every failed allocation under bounded ownership. Existing Lambda quality gates and `make test-lambda-boundaries` pass, including [runtime §7.1](./rustack-workflows-20-lambda-runtime-design.md#71-strict-production-boundary-closure-b03) typed authentication error propagation and method-specific Errors docs. No requirement is satisfied using a noop/echo executor. ## 5. P2 — API, compiled semantics and sequential engine, closes M1 @@ -140,9 +140,9 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. B01/B03 remain open P1 quality gates. | +| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. B01 remains an open P1 dependency gate; B03 is now closed by the six-library strict pass in §7.1. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass; production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01 dependency security and B03 strict boundary quality remain mandatory before P1 closes; B04 is closed without enabling Docker execution. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B03 now passes the six-library strict boundary gate after typed MAC-error propagation and guarded-index/Errors-doc corrections; W54–W56 are fixed/rechecked. B01 dependency security still must close before P1; B03/B04 closure does not enable Docker execution. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 97a0032e..844ea44a 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -241,13 +241,29 @@ The final independent read-only source review finds no remaining B04 issue, incl Final workspace build/test/fmt/Clippy/rustdoc gates pass:2,072 tests, zero failed and401 resource-gated cases. No dependency package/version or Cargo.lock change landed; serde_json's existing raw_value feature supplies borrowed RawValue. Fresh strict core boundary Clippy still reports exactly65 B03 baseline findings, none in the decoder or newly extracted validation helpers. Fresh audit reports the same16 B01 advisories, and deny bans/licenses/sources pass while advisories fail; no new ignores. B04 is closed. B01/B03 and the remaining Docker executor/directory ownership/pulls/probes still block full P1/M0 acceptance; P2–P5 remain required. +## P1 B03 strict production boundary review + +The six-library strict gate now covers rustack-auth, rustack-core, rustack-lambda-model, rustack-lambda-http, rustack-lambda-core and rustack-lambda-runtime. Fresh initial reports found65 core,47 authentication,7 model and4 HTTP findings (123 total); the shared core/runtime already passed. Method-specific Errors sections replace missing contracts, rustdoc tokens are formatted, and guarded credential/layer/date indexing uses exact slice patterns. Native output reads use checked access. These changes preserve the existing parser profile rather than claiming expanded layer compatibility. + +The two production HMAC expect calls are removed by returning typed CryptoInitialization errors with their original InvalidLength sources. Public derive_signing_key/compute_signature now return Result, all production/header/presigned/streaming callers propagate it, and direct S3/SQS test helpers/examples are adapted. No infallible fallback, custom crypto/key normalization, hidden retry, new dependency/version or lint ignore was introduced. Two added tests check four independent published SHA1/SHA256 vectors, including a131-byte key; existing presigned ambiguity, payload-tamper, seed/chunk/trailer and constant-time verification regressions remain active. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W54 | P3; `provider.rs`, delete_function Errors | Deletion can fail during root revalidation after metadata removal. Document both preflight and possible post-removal cleanup failure | Final source directly rechecked by independent reviewer | +| W55 | P3; `storage.rs`, validate_artifact Errors | Generic executable validation language overstates the check. Name bootstrap/component symlink checks and explicitly preserve missing-bootstrap acceptance | Final source directly rechecked; execution remains responsible for readiness | +| W56 | P3; authentication verifier Errors | Newly propagated MAC initialization failure was documented only on signing helpers. Add it to header/S3/presigned/streaming entry points | All public affected error contracts directly rechecked | + +Self-review also corrected two provider descriptions: layer permission principals are not numerically validated here, and update event-invoke configuration creates absent records before merging. The final independent read-only review confirms all corrections and no remaining in-slice finding. Full P1/M0 acceptance remains separate. + +Final-source workspace build/test/fmt/Clippy/rustdoc gates pass with2,074 tests, zero failed and401 resource-gated cases. `make test-lambda-boundaries` passes659 unit tests and165 doctests (824 total), seven separately gated Docker cases, and strict production Clippy for all six libraries without new ignores. Core remains395 tests/101 doctests; authentication is76 tests/12 doctests. The Native provider target again passes six actual Native/validation scenarios, excluding its gated Squib selection as VM evidence. The most recent audit/deny retains16 B01 advisories and passing bans/licenses/sources; this change has no manifest/version/lock change. Directory ownership/reconciliation, allowed pulls, real Docker pool/executor and startup/configuration/probes still block full P1/M0, and P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| | B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | -| B03 | P2; `crates/rustack-lambda-core/src/executor/native.rs:336`, `src/resolver.rs:271`, `src/storage.rs:1292` and existing provider Errors docs | The initial strict lambda-core boundary run found66 baseline findings:56 missing Errors sections, four documentation-format findings and six indexing/slicing sites; its unfiltered dependency run also found auth/core/model debt. The later JSON slice fixes the four rustack-core formatting findings and passes that crate's full strict production check. Remaining Lambda/auth/model findings still require a fresh final audit; standard workspace Clippy passes | P1 quality work: fix the remaining affected boundaries/docs and rerun strict checks without ignores; standard checks alone do not close this gate | +| B03 | P2; Lambda model/HTTP/core and shared auth/core/runtime production libraries | **Closed:** the six-library strict target passes without new ignores after correcting123 fresh baseline findings, including typed MAC initialization errors, guarded indexing and method-specific Errors contracts | Retain `make test-lambda-boundaries` through P1 acceptance; this is not global workspace strict-lint or Docker executor acceptance | | B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 625f72d5..4e5d0995 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -206,6 +206,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Pinned by:** [runtime §2.12](./rustack-workflows-20-lambda-runtime-design.md#212-side-effect-free-code-validation-and-bounded-deployment-json), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-25. +## D26 — Propagate cryptographic initialization errors through signing APIs + +- **Context:** strict production checks expose two HMAC expect calls on a formally fallible provider API; authentication is part of the Lambda gateway dependency boundary. +- **Alternatives considered:** retaining panic/adding lint ignores violates the boundary contract; fabricating a signature or interpreting an initialization failure as a mismatch hides infrastructure failure; hand-rolling key normalization solely to obtain an infallible constructor duplicates cryptographic logic. +- **Decision:** keep RustCrypto HMAC0.13/digest0.11 computations, return typed CryptoInitialization with its InvalidLength source, and make public derive_signing_key/compute_signature return Result. Update callers/examples directly; provide no deprecated infallible shim. Header/presigned/streaming verifiers propagate failure before comparison/fallback or chain advancement. +- **Why:** no request/key can reach an explicit panic, source errors remain available without embedding credentials, and successful signatures retain their published-vector semantics. Existing S3-only normalized-fallback eligibility and constant-time comparison remain intact. +- **Pinned by:** [runtime §7.1](./rustack-workflows-20-lambda-runtime-design.md#71-strict-production-boundary-closure-b03), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-25. This is an internal-library API migration for direct consumers, not a new AWS wire mode or a dependency upgrade. + ## Glossary | Term | Meaning in this specification | From 58da6b2fd1e04f033d8ff7b05d72ee4efe97dd3f Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Sun, 25 Oct 2026 23:46:10 -0700 Subject: [PATCH 019/106] phase 1: close dependency advisories and preserve strict ZIP validation Close B01 with compatible crypto/TLS/PKI/QUIC updates and ZIP 9.1.1. Remove unused SDK SigV4a selections from the regional local-client profile while retaining model versions and SigV4/HTTP/Tokio. Validate original ZIP mode/name metadata before normalization and reject collapsed raw-name duplicates. Match runtime section 7.2/D27 and independently reviewed W57-W59 corrections. Final workspace build/test/fmt/Clippy/rustdoc pass: 2083 tests, zero failed, 401 gated. Core has 404 tests and 101 doctests; the six-library boundary target has 833 passes and green strict Clippy without new ignores. All audit/deny checks pass; real regional SDK/HTTP validation and revision tests plus Native provider regressions pass. No Squib VM, production Image Invoke or full P1/M0 acceptance is claimed. Directory ownership/reconciliation, pulls, real Docker pool/executor/probes and P2-P5 remain required. --- Cargo.lock | 373 +++++------------- Cargo.toml | 5 +- crates/rustack-lambda-core/src/storage.rs | 261 +++++++++++- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 6 +- specs/rustack-workflows-90-impl-plan.md | 6 +- specs/rustack-workflows-93-review.md | 18 +- specs/rustack-workflows-99-decisions.md | 9 + 9 files changed, 373 insertions(+), 315 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af4050d1..18b6d00e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -515,14 +515,14 @@ dependencies = [ "bytes", "fastrand", "hex", - "hmac 0.13.0", + "hmac", "http 0.2.12", "http 1.5.0", "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2", "tracing", "url", ] @@ -750,19 +750,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hmac 0.13.0", + "hmac", "http 0.2.12", "http 1.5.0", - "p256", "percent-encoding", - "sha2 0.11.0", - "subtle", + "sha2", "time", "tracing", - "zeroize", ] [[package]] @@ -806,7 +802,7 @@ dependencies = [ "md-5", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "tracing", ] @@ -1085,12 +1081,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -1256,7 +1246,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -1337,12 +1327,6 @@ dependencies = [ "yaml-rust2", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -1477,28 +1461,16 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.7.5" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" dependencies = [ "cpubits", "ctutils", "getrandom 0.4.3", "hybrid-array", "num-traits", - "rand_core 0.10.1", + "rand_core", "subtle", "zeroize", ] @@ -1521,14 +1493,14 @@ checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "getrandom 0.4.3", "hybrid-array", - "rand_core 0.10.1", + "rand_core", ] [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" dependencies = [ "cmov", "subtle", @@ -1548,24 +1520,13 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] - [[package]] name = "der" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid 0.10.2", + "const-oid", "zeroize", ] @@ -1582,9 +1543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.6", - "subtle", ] [[package]] @@ -1594,7 +1553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -1616,32 +1575,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.8.2", + "der", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", - "signature 3.0.0", - "spki 0.8.0", + "elliptic-curve", + "rfc6979", + "signature", + "spki", "zeroize", ] @@ -1651,42 +1596,22 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pem-rfc7468", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", - "pkcs8 0.11.0", - "rand_core 0.10.1", - "sec1 0.8.1", + "pkcs8", + "rand_core", + "sec1", "subtle", "zeroize", ] @@ -1728,23 +1653,13 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "ff" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "rand_core 0.10.1", + "rand_core", "subtle", ] @@ -1888,7 +1803,6 @@ checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -1914,7 +1828,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "rand_core 0.10.1", + "rand_core", "wasm-bindgen", ] @@ -1930,25 +1844,14 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "group" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", - "rand_core 0.10.1", + "ff", + "rand_core", "subtle", ] @@ -2023,15 +1926,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0" @@ -2411,11 +2305,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f50113171a713f4a4231ef82eb26703607139b35dcb56241f0ceab2ae1f7d8" dependencies = [ "cpubits", - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", - "primeorder 0.14.0", - "sha2 0.11.0", - "signature 3.0.0", + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", + "signature", "wnaf", ] @@ -2568,6 +2462,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2644,18 +2550,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2685,15 +2579,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2712,24 +2597,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", -] - [[package]] name = "pkcs8" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.2", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -2759,30 +2634,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", - "rand_core 0.10.1", + "ff", + "rand_core", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -2845,9 +2711,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -2903,16 +2769,7 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", + "rand_core", ] [[package]] @@ -2927,7 +2784,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -2936,7 +2793,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -3067,24 +2924,14 @@ dependencies = [ "web-sys", ] -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] @@ -3190,11 +3037,11 @@ dependencies = [ "chrono", "digest 0.11.3", "hex", - "hmac 0.13.0", + "hmac", "http 1.5.0", "percent-encoding", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "subtle", "thiserror", "tokio", @@ -3841,7 +3688,7 @@ dependencies = [ "rustack-lambda-runtime", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "squib", "tempfile", "thiserror", @@ -4003,7 +3850,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "tempfile", "thiserror", "tokio", @@ -4034,7 +3881,7 @@ dependencies = [ "rustack-s3-core", "rustack-s3-model", "rustack-s3-xml", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tokio-test", @@ -4193,7 +4040,7 @@ dependencies = [ "rustack-sns-model", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tracing", @@ -4249,7 +4096,7 @@ dependencies = [ "rustack-sqs-model", "serde", "serde_json", - "sha2 0.11.0", + "sha2", "thiserror", "tokio", "tracing", @@ -4422,9 +4269,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -4486,9 +4333,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4532,29 +4379,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", -] - [[package]] name = "sec1" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 1.0.0", + "base16ct", "ctutils", - "der 0.8.2", + "der", "hybrid-array", "subtle", "zeroize", @@ -4676,7 +4509,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -4708,17 +4541,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -4755,16 +4577,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "3.0.0" @@ -4772,7 +4584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ "digest 0.11.3", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -4828,16 +4640,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -4845,7 +4647,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.2", + "der", ] [[package]] @@ -5490,9 +5292,9 @@ dependencies = [ [[package]] name = "typed-path" -version = "0.12.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" [[package]] name = "typenum" @@ -5698,18 +5500,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -5890,8 +5692,8 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", "primefield", ] @@ -6024,15 +5826,18 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "518b847fd864440c379748ad684c449f293af68978a485a4ff16f7c901c1f7c4" dependencies = [ "crc32fast", "flate2", + "getrandom 0.4.3", "indexmap", "memchr", + "nix", "typed-path", + "windows-sys 0.61.2", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 80d75198..7f010637 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,7 +168,7 @@ regex = "1.13" bytes = "1.12" rand = "0.10" tempfile = "3.27" -zip = { version = "8", default-features = false, features = ["deflate"] } +zip = { version = "9.1.1", default-features = false, features = ["deflate"] } percent-encoding = "2.3" mime = "0.3" futures = "0.3" @@ -191,7 +191,6 @@ aws-sdk-s3 = { version = "1.145", default-features = false, features = [ "default-https-client", "http-1x", "rt-tokio", - "sigv4a", ] } aws-sdk-dynamodb = { version = "1.124", default-features = false, features = [ "default-https-client", @@ -228,7 +227,6 @@ aws-sdk-cloudwatchlogs = { version = "1.150", default-features = false, features aws-sdk-eventbridge = { version = "1.116", default-features = false, features = [ "default-https-client", "rt-tokio", - "sigv4a", ] } aws-sdk-kms = { version = "1", default-features = false, features = [ "default-https-client", @@ -257,7 +255,6 @@ aws-sdk-iam = { version = "1", default-features = false, features = [ aws-sdk-sts = { version = "1", default-features = false, features = [ "default-https-client", "rt-tokio", - "sigv4a", ] } aws-sdk-cloudfront = { version = "1", default-features = false, features = [ "default-https-client", diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index 1e9994de..0a4f985e 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -7,8 +7,9 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt, - io::ErrorKind, + io::{ErrorKind, Read}, path::{Path, PathBuf}, + str::from_utf8, }; use base64::{Engine, engine::general_purpose::STANDARD}; @@ -21,6 +22,7 @@ use rustack_lambda_model::types::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tokio::sync::watch; +use zip::{System, read::ZipFile}; use crate::{ artifact::{ImageArtifact, ResolvedImage, ValidatedArtifact}, @@ -1477,6 +1479,11 @@ fn extract_zip( message: format!("not a valid zip archive: {e}"), })?; + if archive.has_duplicate_names() { + return Err(LambdaServiceError::InvalidZipFile { + message: "Duplicate ZIP entry".into(), + }); + } if archive.len() > 10_000 { return Err(LambdaServiceError::InvalidZipFile { message: "Too many ZIP entries".into(), @@ -1490,28 +1497,17 @@ fn extract_zip( .map_err(|e| LambdaServiceError::InvalidZipFile { message: format!("zip entry {i}: {e}"), })?; - let Some(rel) = entry.enclosed_name() else { - return Err(LambdaServiceError::InvalidZipFile { - message: format!("zip entry has invalid path: {}", entry.name()), - }); - }; - if entry.name().len() > 1024 - || entry.name().contains('\\') - || entry.name().split('/').any(|component| component == "..") - || entry - .unix_mode() - .is_some_and(|mode| mode & 0o170_000 == 0o120_000 || mode & 0o7000 != 0) - || !paths.insert(rel.clone()) - { + let rel = validated_zip_path(&entry)?; + if !paths.insert(rel.clone()) { return Err(LambdaServiceError::InvalidZipFile { - message: "Unsafe or duplicate ZIP entry".into(), + message: "Duplicate ZIP entry".into(), }); } let out_path = target.join(&rel); // Defense in depth: ensure the resolved path stays within target. if !out_path.starts_with(target) { return Err(LambdaServiceError::InvalidZipFile { - message: format!("zip entry escapes extraction root: {}", entry.name()), + message: "ZIP entry escapes extraction root".into(), }); } if entry.is_dir() { @@ -1525,10 +1521,7 @@ fn extract_zip( let remaining = max_extracted.saturating_sub(extracted_total); if entry.size() > remaining { return Err(LambdaServiceError::InvalidZipFile { - message: format!( - "zip archive expands beyond {max_extracted} bytes (entry: {})", - entry.name(), - ), + message: format!("ZIP archive expands beyond {max_extracted} bytes"), }); } if let Some(parent) = out_path.parent() { @@ -1573,6 +1566,36 @@ fn extract_zip( Ok(()) } +/// Check original names and attributes before ZIP9 normalizes decoded metadata. +fn validated_zip_path( + entry: &ZipFile<'_, R>, +) -> Result { + let invalid = || LambdaServiceError::InvalidZipFile { + message: "Unsafe ZIP entry name or attributes".into(), + }; + let raw_name = entry.name_raw(); + if raw_name.len() > 1024 + || (entry.flags().as_u16() & (1 << 11) != 0 && from_utf8(raw_name).is_err()) + { + return Err(invalid()); + } + let name = entry.name().map_err(|_| invalid())?; + let mode = if entry.system() == System::Dos { + 0 + } else { + entry.external_attributes() >> 16 + }; + if name.len() > 1024 + || name.contains('\\') + || name.split('/').any(|component| component == "..") + || mode & 0o170_000 == 0o120_000 + || mode & 0o7000 != 0 + { + return Err(invalid()); + } + entry.enclosed_name().ok_or_else(invalid) +} + /// Compute base64-encoded SHA-256 hash of the given data. #[must_use] pub fn compute_sha256(data: &[u8]) -> String { @@ -2373,6 +2396,204 @@ mod tests { assert!(matches!(err, LambdaServiceError::InvalidZipFile { .. })); } + #[rstest::rstest] + #[case::invalid_utf8(vec![0xff; 11], true)] + #[case::raw_limit(vec![b'a'; 1025], true)] + #[case::decoded_limit(vec![0x82; 513], false)] + #[tokio::test] + async fn test_should_reject_invalid_zip_names_without_publishing_artifacts( + #[case] raw: Vec, + #[case] utf8: bool, + ) { + use std::io::{Cursor, Write}; + + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "a".repeat(raw.len()), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.write_all(b"payload").unwrap(); + let mut bytes = writer.finish().unwrap().into_inner(); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + let flag = if utf8 { 1u16 << 11 } else { 0 }; + bytes + .get_mut(6..8) + .unwrap() + .copy_from_slice(&flag.to_le_bytes()); + bytes + .get_mut(central + 8..central + 10) + .unwrap() + .copy_from_slice(&flag.to_le_bytes()); + bytes + .get_mut(30..30 + raw.len()) + .unwrap() + .copy_from_slice(&raw); + bytes + .get_mut(central + 46..central + 46 + raw.len()) + .unwrap() + .copy_from_slice(&raw); + + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let error = store + .store_zip_code("safe", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert!(store.artifacts.is_empty()); + let mut entries = tokio::fs::read_dir(root.path()).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[rstest::rstest] + #[case::symlink(0o120_777)] + #[case::setuid(0o104_755)] + #[case::setgid(0o102_755)] + #[case::sticky(0o101_755)] + #[tokio::test] + async fn test_should_reject_hostile_zip_attributes_without_replacing_code(#[case] mode: u32) { + use std::io::Cursor; + + let root = tempfile::tempdir().unwrap(); + let code_dir = root.path().join("code"); + let sentinel = root.path().join("sentinel"); + tokio::fs::write(&sentinel, b"outside").await.unwrap(); + let store = FunctionStore::new(&code_dir); + let (old, _, _) = store + .store_zip_code( + "safe", + "$LATEST", + &Bytes::from(crate::test_zip::minimal_zip(b"old")), + ) + .await + .unwrap(); + + // Encode the central-directory attributes directly: safe writer APIs + // deliberately strip special permission bits. The symlink payload would + // target the outside sentinel if materialized as a link. + let mut hostile = crate::test_zip::minimal_zip(sentinel.as_os_str().as_encoded_bytes()); + let central = hostile + .windows(4) + .position(|bytes| bytes == b"PK\x01\x02") + .unwrap(); + hostile + .get_mut(central + 4..central + 6) + .unwrap() + .copy_from_slice(&((3u16 << 8) | 0x14).to_le_bytes()); + hostile + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(mode << 16).to_le_bytes()); + let mut archive = zip::ZipArchive::new(Cursor::new(&hostile)).unwrap(); + assert_eq!( + archive.by_index(0).unwrap().external_attributes() >> 16, + mode + ); + + let error = store + .store_zip_code("safe", "$LATEST", &Bytes::from(hostile)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert_eq!(tokio::fs::read(&sentinel).await.unwrap(), b"outside"); + assert_eq!( + tokio::fs::read(old.join("fixture.txt")).await.unwrap(), + b"old" + ); + assert_eq!(store.artifacts.len(), 1); + let mut entries = tokio::fs::read_dir(&code_dir).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_some()); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_should_reject_duplicate_raw_zip_names_before_indexed_extraction() { + use std::io::{Cursor, Write}; + + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("first", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"one").unwrap(); + writer + .start_file("other", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"two").unwrap(); + let mut bytes = writer.finish().unwrap().into_inner(); + let local = bytes + .windows(4) + .rposition(|part| part == b"PK\x03\x04") + .unwrap(); + let central = bytes + .windows(4) + .rposition(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(local + 30..local + 35) + .unwrap() + .copy_from_slice(b"first"); + bytes + .get_mut(central + 46..central + 51) + .unwrap() + .copy_from_slice(b"first"); + let archive = zip::ZipArchive::new(Cursor::new(&bytes)).unwrap(); + assert_eq!( + archive.len(), + 1, + "upstream metadata deduplicates identical raw names" + ); + assert!(archive.has_duplicate_names()); + + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let error = store + .store_zip_code("duplicate", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::InvalidZipFile { .. })); + assert!(store.artifacts.is_empty()); + let mut entries = tokio::fs::read_dir(root.path()).await.unwrap(); + assert!(entries.next_entry().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_should_preserve_dos_attribute_semantics_without_special_permissions() { + let mut bytes = crate::test_zip::minimal_zip(b"safe"); + let central = bytes + .windows(4) + .position(|part| part == b"PK\x01\x02") + .unwrap(); + bytes + .get_mut(central + 4..central + 6) + .unwrap() + .copy_from_slice(&20u16.to_le_bytes()); + bytes + .get_mut(central + 38..central + 42) + .unwrap() + .copy_from_slice(&(0o104_755u32 << 16).to_le_bytes()); + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let (path, _, _) = store + .store_zip_code("dos", "$LATEST", &Bytes::from(bytes)) + .await + .unwrap(); + assert_eq!( + tokio::fs::read(path.join("fixture.txt")).await.unwrap(), + b"safe" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = tokio::fs::metadata(path.join("fixture.txt")).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o7000, 0); + } + } + #[test] fn test_should_reject_zip_that_expands_beyond_extraction_budget() { use std::io::Write as _; diff --git a/specs/index.md b/specs/index.md index 2f843c5e..528c12f2 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 4aa05c68..13e2527f 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -482,6 +482,14 @@ Native log draining uses checked access for the successful AsyncRead length and Use the rechecked Bollard0.21.1 generated-type/API contracts through model-only bollard-stubs; the bounded transport decision and quotas are authoritative in §3.6/D19. Reuse Hyper/Tokio/Bytes/ciborium. TLS transport is N/A for this release; future TLS must use rustls + aws-lc-rs rather than enabling a ring-backed SSL default. +### 7.2 Dependency security closure (B01) + +P1 must close the16 recorded advisory findings with a current cargo audit/cargo deny run and no new advisory ignores. Upgrade the workspace ZIP reader/writer to the current stable9.1 line (9.1.1 verified against [upstream documentation](https://docs.rs/zip/9.1.1/zip/)); retain default-features=false and only deflate. Reuse bounded manual extraction rather than ZipArchive::extract, and retain rejection of traversal, symlinks, special permission bits, duplicate paths, oversized expansion and corrupt CRCs. Regression fixtures must encode hostile central-directory attributes directly because a safe writer may normalize forbidden permissions; failed staging must retain the old artifact and leave outside sentinels untouched. ZIP9 makes name decoding fallible: bound the raw name to1,024 bytes before decoding, reject decoding failure with a static InvalidZipFile error, then enforce the decoded byte cap and the existing path checks. Never decode lossily or replace a failed name with a sanitized path. The released reader falls back to CP437 even for malformed UTF-8-flagged names, so explicitly require valid UTF-8 when bit11 is set; legacy CP437 remains allowed without that flag, within the decoded cap. The new unix_mode accessor clears special bits: inspect the high16 external-attribute bits before normalization and reject raw symlink/special-permission declarations rather than accidentally admitting them. DOS-origin entries retain DOS attribute semantics; their high bits are not interpreted as POSIX permission declarations. Reject archive.has_duplicate_names() before iterating: the upstream metadata index collapses identical raw names, so the per-path set alone cannot detect those original duplicates. Retain the per-path set for different raw encodings that decode to the same filesystem path. + +Update the locked affected TLS/PKI/QUIC/constant-time libraries to patched compatible releases: rustls≥0.23.46, rustls-webpki≥0.103.16, both WebPKI root crates≥1.0.10, quinn-proto≥0.11.18, crypto-bigint≥0.7.6 and ctutils≥0.4.4. These are minimum security floors, not invented direct dependencies. Verify actual resolved features and audit output rather than assuming the top-level manifest excludes vulnerable transitive versions. Retain AWS-LC TLS selection and existing SDK HTTP/Tokio features. + +The workspace SDK S3/EventBridge/STS clients explicitly enabled optional SigV4a despite the local gateway's [existing fail-closed ECDSA profile](./rustack-http-hardening-design.md). They serve regional local clients; enabling multi-region AWS signing is not a workflow/runtime acceptance requirement. Remove those three unused SigV4a feature selections while retaining SigV4 and the current SDK model versions. This does not change the SDK/runtime bundled inside user Lambda images. The current upstream signer1.6.3 still directly requires crypto-bigint0.5 under SigV4a (inspected released Cargo.toml), which has no patched compatible0.5 version for RUSTSEC-2026-0387. Do not downgrade the security claim, add an ignore or vendor a cryptographic fork merely to retain an unused client mode. Future multi-region support requires an upstream repaired graph and explicit protocol tests. Ordinary regional SDK CRUD, Lambda deployment/Invoke, KMS cryptography, Native ZIP execution and the strict boundary suite must still pass. Record resolved versions, removed features and remaining advisory outcomes in the review; B01 stays open until all required security gates pass. + ## 8. Cross-references - ← [PRD](./rustack-workflows-00-prd.md), [existing runtime spike](../docs/research/spike-cargo-lambda-runtime-execution.md) diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 44f0286a..e484db3c 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -92,9 +92,11 @@ Retained deployment ownership (§2.11) now has19 additional actor/provider cases `make test-lambda-validation-wire SMOKE_ENDPOINT=` explicitly runs SDK/HTTP code DryRun with inline nonarchive bytes and a nonexistent S3 source; both must succeed without extraction/fetch, revision/hash/size/configuration changes or publication. Unsupported Create KMSKeyArn, UpdateCode SourceKMSKeyArn and UpdateConfiguration KMSKeyArn decode as InvalidParameterValueException; rejected creation must leave no resource. The test is in the explicit service smoke manifest. Image DryRun separately uses the qualified scripted daemon/preparation owner and a declared test executor: old full snapshot and tag count remain unchanged, with no new removal. This is not Image Invoke acceptance. -The deployment decoder (§2.12) has exact-limit, escaped/duplicate-key, wrong-path blob, malformed/unknown-field and depth/member/string/key/metadata tests. Inline base64 covers canonical padding and the exact50MiB decoded boundary without allocation of decoded bytes. Capacity remains owned after waiter cancellation and after a result becomes ready; four concurrent admitted inline validators reuse permits. Worker panic releases capacity and surfaces a bounded error; capacity429 must not be downgraded to400. Current395 core tests and101 doctests pass; workspace2,072 tests pass,401 resource-gated cases remain separately selected. +The deployment decoder (§2.12) has exact-limit, escaped/duplicate-key, wrong-path blob, malformed/unknown-field and depth/member/string/key/metadata tests. Inline base64 covers canonical padding and the exact50MiB decoded boundary without allocation of decoded bytes. Capacity remains owned after waiter cancellation and after a result becomes ready; four concurrent admitted inline validators reuse permits. Worker panic releases capacity and surfaces a bounded error; capacity429 must not be downgraded to400. At B04 closure,395 core tests and101 doctests pass; workspace2,072 tests pass,401 resource-gated cases remain separately selected. -`make test-lambda-boundaries` runs unit/integration/doctest regressions and strict production-library Clippy for authentication, shared core, Lambda model/HTTP/core and shared runtime. Require pedantic, missing Errors docs, doc formatting, unwrap/expect, indexing/slicing and panic checks without new ignores. Current catalog has659 unit tests and165 doctests (824 passes); seven Docker cases remain resource-gated and must run via their explicit real-daemon targets for executor qualification. Four fixed RFC2202/RFC4231 HMAC vectors complement existing signature/query/streaming regressions; changing signing helpers to Result must preserve every computed output and propagate provider errors rather than treating them as mismatches. This closes B03 for these production libraries, not every repository test or service. +`make test-lambda-boundaries` runs unit/integration/doctest regressions and strict production-library Clippy for authentication, shared core, Lambda model/HTTP/core and shared runtime. Require pedantic, missing Errors docs, doc formatting, unwrap/expect, indexing/slicing and panic checks without new ignores. The B03 closure catalog has659 unit tests and165 doctests (824 passes); seven Docker cases remain resource-gated and must run via their explicit real-daemon targets for executor qualification. Four fixed RFC2202/RFC4231 HMAC vectors complement existing signature/query/streaming regressions; changing signing helpers to Result must preserve every computed output and propagate provider errors rather than treating them as mismatches. This closes B03 for these production libraries, not every repository test or service. + +Dependency security closure (§7.2/D27) additionally requires current cargo audit and all cargo-deny checks without new ignores; confirm the regional SDK feature graph has no unused SigV4a/old bigint0.5 branch. ZIP9 regressions encode original external attributes and malformed flag/name bytes directly: symlink/SUID/SGID/sticky, invalid flagged UTF-8, raw/decoded name caps, DOS compatibility, existing traversal/CRC/expansion and retained executable modes. Assert rejection leaves no new artifact, old code remains readable and an outside sentinel stays unchanged. Rerun actual SDK/HTTP validation and revision tests plus Native provider execution after the dependency migration; library parsing alone does not establish platform/handler execution acceptance. Current B01 closure has404 core tests/101 doctests,2,083 workspace tests and833 boundary unit/doctest passes (668+165), with unchanged401 workspace/seven boundary resource-gated cases. Standard and strict gates, both selected regional SDK/HTTP tests, Native provider regressions and all audit/deny checks pass. The Squib selection in the Native target is gated and remains excluded as VM evidence. ## 4. Security acceptance diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index e8b93a7f..b149b252 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -140,9 +140,9 @@ The complete implementation objective includes P0–P5. This ledger records evid | 0.2 | Evidence complete | 48 independent semantic/protocol expectations; four offline SDK serializer/parser checks; pinned AWS-validated nested shapes and error corrections. No interpreter/cloud-differential execution is claimed. | | 0.3 | Evidence complete | Model commit/hash and 37 operations pinned; six generated files compile in the standalone experiment. Numeric timestamp failure independently reproduced and mandatory generator remediation assigned task 2.1. Rust updated to current stable 1.99.1. | | 0.4 | Complete | Independent final recheck confirms W07–W09 fixed, zero owned containers on both daemons, and no further P0 correctness finding. W01–W06 contracts are corrected; production consumers remain in their assigned phases. Full workspace build/test/fmt/Clippy/rustdoc pass; unchanged baseline audit/deny advisory failures are B01 and must be fixed in P1 dependency work. | -| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. B01 remains an open P1 dependency gate; B03 is now closed by the six-library strict pass in §7.1. | +| 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. Baseline B01/B03 are now closed by the security and six-library strict passes in §§7.1–7.2; remaining P1 runtime acceptance still applies. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | -| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. B01 dependency closure is implemented/reviewed (§7.2/D27, W57–W59): nine ZIP regressions, workspace2,083 tests,833 boundary tests/doctests, green standard/strict gates, clean audit and all deny checks. Regional SDK/HTTP and Native provider regressions pass. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B03 now passes the six-library strict boundary gate after typed MAC-error propagation and guarded-index/Errors-doc corrections; W54–W56 are fixed/rechecked. B01 dependency security still must close before P1; B03/B04 closure does not enable Docker execution. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 844ea44a..6b838e23 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -257,11 +257,27 @@ Self-review also corrected two provider descriptions: layer permission principal Final-source workspace build/test/fmt/Clippy/rustdoc gates pass with2,074 tests, zero failed and401 resource-gated cases. `make test-lambda-boundaries` passes659 unit tests and165 doctests (824 total), seven separately gated Docker cases, and strict production Clippy for all six libraries without new ignores. Core remains395 tests/101 doctests; authentication is76 tests/12 doctests. The Native provider target again passes six actual Native/validation scenarios, excluding its gated Squib selection as VM evidence. The most recent audit/deny retains16 B01 advisories and passing bans/licenses/sources; this change has no manifest/version/lock change. Directory ownership/reconciliation, allowed pulls, real Docker pool/executor and startup/configuration/probes still block full P1/M0, and P2–P5 remain required. +## P1 B01 dependency security and ZIP9 review + +The dependency update resolves ZIP9.1.1, crypto-bigint0.7.10, ctutils0.4.6, quinn-proto0.11.19, rustls0.23.46, rustls-webpki0.103.16 and both WebPKI root crates1.0.11. ZIP adds nix0.31.3 and typed-path0.13; deny license/source/ban checks accept the graph without new exceptions. SDK model versions remain unchanged. Removing the three unused local SDK SigV4a selections removes crypto-bigint0.5 and its old P256/ECDSA/HMAC graph; regional SigV4/HTTP/Tokio remain enabled. The inspected latest signer1.6.3 still directly requires optional bigint0.5, so a compatible patch was unavailable for that unused mode. User-image SDKs remain independent. + +Fresh cargo audit scans474 dependencies without vulnerabilities. Cargo deny passes advisories, bans, licenses and sources; preexisting policy notices about duplicate packages and unmatched license allowances/advisory ignores remain, with no new exception or ignore. The resolved feature tree has no SigV4a and retains AWS-LC for the SDK. Historical reqwest clients still also enable ring; this change neither introduces a TLS transport nor claims to remove every historical provider selection. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W57 | P2; self-review, ZIP9 unix_mode migration | The new accessor clears special bits before the old guard sees them. Validate original high attributes for Unix-like origins; retain DOS-specific semantics | Four direct central-header symlink/SUID/SGID/sticky fixtures reject without replacing old artifacts or touching an outside sentinel; a DOS compatibility case succeeds without special permissions | +| W58 | P2; self-review, ZIP9 name migration | name() is fallible but can fall back to CP437 despite bit11. Bound raw names before decoding, enforce flagged UTF-8, propagate decoding rejection statically, then check decoded bytes/path | Invalid UTF-8, raw-name overflow and CP437 decoded-byte overflow reject before artifact publication; existing traversal, CRC, expansion and executable-mode regressions remain active | +| W59 | P2; self-review, ZIP9 metadata index | Identical original raw names collapse in the upstream index, so iterating with a path set cannot detect those duplicates; by_index does not enable header-consistency enforcement by default. Reject has_duplicate_names before traversal, retaining the path set for decoded collisions | A crafted two-entry central/local-header fixture demonstrates len1 plus duplicate metadata, then requires rejection without artifacts/directories; independent source recheck finds no issue | + +The independent read-only review verifies the released ZIP9/source semantics, raw fixture offsets, DOS guard, feature-pruning scope and lock delta; the final recheck has no in-slice findings. This is security/profile validation, not full Docker executor or P1/M0 acceptance. + +Final-source workspace build/test/fmt/Clippy/rustdoc pass with2,083 tests, zero failed and401 resource-gated cases. Core has404 tests (nine new ZIP cases) and101 doctests; `make test-lambda-boundaries` passes668 unit tests/165 doctests (833 total), seven resource-gated Docker cases, and all six strict production-library checks. The Native provider target passes six actual Native/validation scenarios; its gated Squib selection is excluded as VM evidence. Post-upgrade real Rust SDK/HTTP validation and revision/412 tests each pass with zero ignored, and the owned test gateway exits normally. Audit/deny pass against the same committed dependency graph. B01/B03/B04 are closed; persistent directory ownership/reconciliation, allowed pulls, Docker environment/pool executor, startup/configuration/probes and full M0 acceptance remain required. P2–P5 also remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| -| B01 | P1; `Cargo.lock:1462`, `:1474`, `:4308`, `:5893` | Baseline cargo audit finds 16 advisories across crypto-bigint 0.5.5/0.7.5, ctutils, quinn-proto, rustls/webpki roots and ZIP 8.6.0. cargo-deny fails its advisory gate; bans/licenses/sources pass. Standalone probe audit is clean | P1 dependency work: upgrade affected dependencies, including ZIP major and the SDK's SigV4a chain; rerun meaningful regressions and audit/deny. Do not add ignores to make the gate green | +| B01 | P1; `Cargo.toml:171`, `Cargo.lock`, runtime §7.2/D27 | **Closed:** all16 original advisory findings resolved; current cargo audit has no vulnerabilities and deny passes all four checks, without new ignores. Patched compatible dependencies and ZIP9 replace affected versions; unused local SDK SigV4a crypto graph is removed | Retain audit/deny, hostile ZIP and regional SDK regressions through P1 acceptance; this is not full executor/platform acceptance | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; Lambda model/HTTP/core and shared auth/core/runtime production libraries | **Closed:** the six-library strict target passes without new ignores after correcting123 fresh baseline findings, including typed MAC initialization errors, guarded indexing and method-specific Errors contracts | Retain `make test-lambda-boundaries` through P1 acceptance; this is not global workspace strict-lint or Docker executor acceptance | | B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 4e5d0995..11b32eae 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -215,6 +215,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Pinned by:** [runtime §7.1](./rustack-workflows-20-lambda-runtime-design.md#71-strict-production-boundary-closure-b03), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-25. This is an internal-library API migration for direct consumers, not a new AWS wire mode or a dependency upgrade. +## D27 — Align SDK signing features with the regional local compatibility profile + +- **Context:** B01 includes crypto-bigint0.5 through optional AWS SDK SigV4a; the latest released signer inspected (1.6.3) still directly requires0.5, while the patched bigint floor is0.7.6. +- **Alternatives considered:** an advisory ignore leaves the security gate false; an unreviewed crypto fork adds a maintenance/supply-chain burden; removing supported regional SigV4 clients would break the workflow contract. +- **Decision:** remove the unused explicit S3/EventBridge/STS SDK SigV4a features, retain their regional SigV4/HTTP/Tokio behavior and current model versions, and upgrade affected compatible libraries plus ZIP9.1. This selects the actual local client profile already enforced by the gateway; user-image SDKs remain independent. +- **Why:** the existing gateway fails closed for unimplemented ECDSA signing, and workflow acceptance has no multi-region AWS signing requirement. Security closure must prove the resolved graph, not merely rename or ignore a vulnerable version. Future multi-region support requires a repaired upstream signer and its own tests. +- **Pinned by:** [runtime §7.2](./rustack-workflows-20-lambda-runtime-design.md#72-dependency-security-closure-b01), [HTTP authentication profile](./rustack-http-hardening-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-25. SDK model versions and supported AWS wire modes are preserved. + ## Glossary | Term | Meaning in this specification | From 7731e263c8cc361254d6af96c6922d6568dc7c05 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 00:27:17 -0700 Subject: [PATCH 020/106] phase 1: retain exclusive directory leases through image cleanup Implement runtime section 4.1 and D28 with private descriptor-relative namespace IO, permanent OS lock ownership, bounded checksummed identity publication, trusted ancestor validation and cancellation-safe worker capacity. Image scopes, allocated pins and cleanup recovery retain the actual directory capability. Independent review W60-W62 fixes are verified. Workspace 2122 tests and all standard gates pass; six-library strict boundary 874 unit/doc passes, audit and deny pass. Both real wire suites pass arm64 VZ and amd64/QEMU. The exception at docker/ownership.rs:5 permits standard-file synchronous IO only on bounded blocking workers because Tokio files have no OS advisory-lock API. P1/M0 remains open for historical-effect reconciliation, pulls, log/network/probe consumers and environment/pool executor; P2-P5 remain required. --- Cargo.lock | 1 + Cargo.toml | 2 + Makefile | 7 + crates/rustack-lambda-core/Cargo.toml | 1 + .../src/docker/lease_tests.rs | 56 +- crates/rustack-lambda-core/src/docker/mod.rs | 2 + .../src/docker/ownership.rs | 533 ++++++++++++++++ .../src/docker/ownership_tests.rs | 567 ++++++++++++++++++ crates/rustack-lambda-core/src/docker/pin.rs | 49 +- .../rustack-lambda-core/tests/docker_wire.rs | 34 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 32 +- .../rustack-workflows-60-operations-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 18 + specs/rustack-workflows-99-decisions.md | 9 + 17 files changed, 1290 insertions(+), 38 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/ownership.rs create mode 100644 crates/rustack-lambda-core/src/docker/ownership_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 18b6d00e..25b2e407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3678,6 +3678,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "nix", "parking_lot", "percent-encoding", "rstest", diff --git a/Cargo.toml b/Cargo.toml index 7f010637..322f44c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,6 +130,8 @@ zstd = { version = "0.13.3", default-features = false } # Error handling anyhow = "1.0" thiserror = "2.0" +# Safe relative-descriptor ownership IO; also resolved by ZIP9. +nix = { version = "0.31.3", default-features = false, features = ["fs", "user"] } # Concurrent data structures dashmap = "6.2" diff --git a/Makefile b/Makefile index bece6c84..00c80f78 100644 --- a/Makefile +++ b/Makefile @@ -299,6 +299,13 @@ test-lambda-boundaries: .PHONY: test-lambda-boundaries +# Descriptor lifetime, persisted namespace identity and cleanup ownership. +test-lambda-ownership: + @cargo test -p rustack-lambda-core --lib docker::ownership::tests + @cargo test -p rustack-lambda-core --lib test_should_retain_the_directory_lock_through_abandoned_image_owner_cleanup + +.PHONY: test-lambda-ownership + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 586f766e..64fbb620 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -31,6 +31,7 @@ chrono = { workspace = true } uuid = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } +nix = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } sha2 = { workspace = true } diff --git a/crates/rustack-lambda-core/src/docker/lease_tests.rs b/crates/rustack-lambda-core/src/docker/lease_tests.rs index a26d9ea3..f803df32 100644 --- a/crates/rustack-lambda-core/src/docker/lease_tests.rs +++ b/crates/rustack-lambda-core/src/docker/lease_tests.rs @@ -11,7 +11,7 @@ use tokio::{ use super::*; use crate::docker::{ - DockerWire, + DirectoryLease, DirectoryLeaseError, DockerWire, tests::{Daemon, IncomingRequest}, }; @@ -22,7 +22,7 @@ fn deadline() -> Instant { Instant::now() + Duration::from_secs(10) } fn scope() -> ImagePinScope { - ImagePinScope::new(Uuid::new_v4(), Uuid::new_v4()).unwrap() + ImagePinScope::testing(Uuid::new_v4(), Uuid::new_v4()).unwrap() } fn config(pins: u16, jobs: u8) -> ImageLeaseConfig { ImageLeaseConfig::new(pins, jobs, Duration::from_secs(1)).unwrap() @@ -320,6 +320,56 @@ impl Drop for FakePins { } } +#[tokio::test] +async fn test_should_retain_the_directory_lock_through_abandoned_image_owner_cleanup() { + let fake = FakePins::new(Mode::HoldDelete).await; + let directory = tempfile::tempdir().unwrap(); + let lease = DirectoryLease::acquire(directory.path().into(), deadline()) + .await + .unwrap(); + let identity = lease.owner_id(); + let owner = ImageLeaseManager::start(fake.wire.port(), ImagePinScope::new(lease), config(2, 1)) + .unwrap(); + let image = owner + .port() + .prepare( + &"worker:v1".parse().unwrap(), + Architecture::Arm64, + None, + deadline(), + ) + .await + .unwrap(); + drop(image); + fake.stats + .until(|| owner.port().status().quarantined > 0) + .await; + drop(owner); + assert!(matches!( + DirectoryLease::acquire(directory.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + fake.commands.send(Command::AllowDeletes).await.unwrap(); + timeout(Duration::from_secs(10), async { + loop { + match DirectoryLease::acquire(directory.path().into(), deadline()).await { + Ok(lease) => { + assert_eq!(lease.owner_id(), identity); + break; + } + Err(DirectoryLeaseError::InUse) => { + tokio::time::sleep(Duration::from_millis(5)).await; + } + other => panic!("unexpected lease result: {other:?}"), + } + } + }) + .await + .unwrap(); + assert_eq!(fake.stats.aliases.load(Ordering::Acquire), 0); + fake.close().await; +} + #[tokio::test] async fn test_should_pin_old_code_across_tag_retargeting_and_keep_cloned_artifacts_live() { let fake = FakePins::new(Mode::Normal).await; @@ -576,7 +626,7 @@ async fn test_should_join_after_a_shutdown_waiter_is_dropped_and_cache_root_fail #[test] fn test_should_reject_invalid_pin_scope_and_preparation_configuration() { - assert!(ImagePinScope::new(Uuid::nil(), Uuid::new_v4()).is_err()); + assert!(ImagePinScope::testing(Uuid::nil(), Uuid::new_v4()).is_err()); let pin = scope().allocate().unwrap(); assert!( pin.reference() diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 91d61a89..94431d31 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -7,6 +7,7 @@ mod connection; mod framing; mod image; mod lease; +mod ownership; mod pin; mod types; @@ -32,6 +33,7 @@ use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, }; +pub use ownership::{DirectoryLease, DirectoryLeaseError}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; pub use pin::{ImagePin, ImagePinScope}; use rustack_core::json::{JsonLimits, from_slice}; diff --git a/crates/rustack-lambda-core/src/docker/ownership.rs b/crates/rustack-lambda-core/src/docker/ownership.rs new file mode 100644 index 00000000..71ac927f --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership.rs @@ -0,0 +1,533 @@ +//! Persistent, descriptor-relative ownership of a Docker resource namespace. +//! +//! Standard files are required for OS advisory locks. All synchronous IO runs +//! on bounded blocking workers; lease clones perform no IO and retain the lock. +#![allow(clippy::disallowed_types, clippy::disallowed_methods)] + +use std::{ + ffi::OsString, + fmt, + fs::{File, Metadata, TryLockError, canonicalize, symlink_metadata}, + io::{self, Read, Write}, + os::unix::fs::MetadataExt, + path::{Component, Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use nix::{ + errno::Errno, + fcntl::{OFlag, open, openat, renameat}, + sys::stat::{Mode, mkdirat}, + unistd::geteuid, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + runtime::Handle, + sync::{Semaphore, SemaphorePermit}, + task::{JoinHandle, spawn_blocking}, + time::{Instant, timeout_at}, +}; +use uuid::Uuid; + +const NAMESPACE: &str = ".rustack-lambda-docker"; +const LOCK: &str = "lease.lock"; +const OWNER: &str = "owner.v1"; +const PENDING: &str = "owner.pending"; +const MAGIC: &[u8; 8] = b"RSDKOWN1"; +const OWNER_BYTES: usize = 56; +static WORKERS: Semaphore = Semaphore::const_new(4); + +/// Directory acquisition failures; diagnostics never include operator paths. +#[derive(thiserror::Error)] +#[non_exhaustive] +pub enum DirectoryLeaseError { + /// Invalid operator directory or acquisition budget. + #[error("invalid Docker ownership configuration")] + Configuration, + /// Another descriptor/process owns this data directory. + #[error("Docker data directory is already owned")] + InUse, + /// Unsafe file kind/permissions/links or damaged authoritative identity. + #[error("invalid Docker ownership state")] + InvalidState, + /// The absolute acquisition deadline elapsed. + #[error("Docker ownership deadline exceeded")] + Deadline, + /// The requesting acquisition was abandoned. + #[error("Docker ownership acquisition cancelled")] + Cancelled, + /// No runtime or the blocking worker failed. + #[error("Docker ownership worker failed")] + Worker, + /// A filesystem operation failed, preserving its source without displaying it. + #[error("Docker ownership IO failed")] + Io(#[source] io::Error), +} +impl fmt::Debug for DirectoryLeaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} +impl From for DirectoryLeaseError { + fn from(source: io::Error) -> Self { + Self::Io(source) + } +} +impl From for DirectoryLeaseError { + fn from(source: Errno) -> Self { + match source { + Errno::ELOOP | Errno::ENOTDIR => Self::InvalidState, + _ => Self::Io(source.into()), + } + } +} + +/// Shared proof of an exclusive, durably identified Docker data directory. +#[derive(Clone)] +#[non_exhaustive] +pub struct DirectoryLease(Arc); +struct Lease { + owner: Uuid, + run: Uuid, + _guard: LeaseGuard, +} +enum LeaseGuard { + Held { + _directory: File, + _lock: File, + }, + #[cfg(test)] + Testing, +} +impl fmt::Debug for DirectoryLease { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DirectoryLease()") + } +} +impl DirectoryLease { + /// Acquire a private namespace, retaining exclusion until the last clone drops. + /// + /// # Errors + /// Rejects invalid paths/budgets, competing owners, unsafe/corrupt state, + /// cancellation/deadline, filesystem failures and unavailable/failed workers. + /// Acquisition does not establish previous-run reconciliation or readiness. + /// + /// ```no_run + /// # async fn acquire() -> Result<(), rustack_lambda_core::docker::DirectoryLeaseError> { + /// use rustack_lambda_core::docker::DirectoryLease; + /// let lease = DirectoryLease::acquire( + /// "/tmp/rustack-data".into(), + /// tokio::time::Instant::now() + std::time::Duration::from_secs(5), + /// ).await?; + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(lease); + /// # Ok(()) } + /// ``` + pub async fn acquire( + directory: PathBuf, + deadline: Instant, + ) -> Result { + let directory = DataDirectory::try_from(directory)?; + run(&WORKERS, deadline, move |control| { + acquire(&directory, control) + }) + .await + } + + /// Persistent UUID used for resource labels and private image aliases. + /// + /// ```no_run + /// # fn owner(lease: &rustack_lambda_core::docker::DirectoryLease) { + /// let owner = lease.owner_id(); + /// assert!(!owner.is_nil()); + /// # } + /// ``` + #[must_use] + pub fn owner_id(&self) -> Uuid { + self.0.owner + } + + /// Fresh UUID distinguishing this acquisition from prior runs. + /// + /// ```no_run + /// # fn run(lease: &rustack_lambda_core::docker::DirectoryLease) { + /// let run = lease.run_id(); + /// assert!(!run.is_nil()); + /// # } + /// ``` + #[must_use] + pub fn run_id(&self) -> Uuid { + self.0.run + } + + #[cfg(test)] + pub(super) fn testing(owner: Uuid, run: Uuid) -> Self { + Self(Arc::new(Lease { + owner, + run, + _guard: LeaseGuard::Testing, + })) + } +} + +struct DataDirectory(PathBuf); +impl TryFrom for DataDirectory { + type Error = DirectoryLeaseError; + + fn try_from(value: PathBuf) -> Result { + let encoded = value.as_os_str().as_encoded_bytes(); + if encoded.len() > 4096 + || encoded.contains(&0) + || value.components().count() > 128 + || !value.is_absolute() + || value + .components() + .any(|part| matches!(part, Component::ParentDir)) + { + return Err(DirectoryLeaseError::Configuration); + } + Ok(Self(value.components().collect())) + } +} + +struct Control { + deadline: Instant, + cancelled: Arc, +} +impl Control { + fn check(&self) -> Result<(), DirectoryLeaseError> { + if self.cancelled.load(Ordering::Acquire) { + Err(DirectoryLeaseError::Cancelled) + } else if Instant::now() >= self.deadline { + Err(DirectoryLeaseError::Deadline) + } else { + Ok(()) + } + } +} +struct CancelOnDrop(Arc); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +async fn run( + pool: &'static Semaphore, + deadline: Instant, + work: impl FnOnce(&Control) -> Result + Send + 'static, +) -> Result { + Handle::try_current().map_err(|_| DirectoryLeaseError::Worker)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or(DirectoryLeaseError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DirectoryLeaseError::Configuration); + } + let permit = timeout_at(deadline, pool.acquire()) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + let cancelled = Arc::new(AtomicBool::new(false)); + let _cancel = CancelOnDrop(cancelled.clone()); + let task = spawn_work( + permit, + Control { + deadline, + cancelled, + }, + work, + ); + let (value, _permit) = timeout_at(deadline, task) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)??; + if Instant::now() >= deadline { + return Err(DirectoryLeaseError::Deadline); + } + Ok(value) +} + +fn spawn_work( + permit: SemaphorePermit<'static>, + control: Control, + work: impl FnOnce(&Control) -> Result + Send + 'static, +) -> JoinHandle), DirectoryLeaseError>> { + spawn_blocking(move || { + control.check()?; + let value = work(&control)?; + control.check()?; + Ok((value, permit)) + }) +} + +fn acquire( + directory: &DataDirectory, + control: &Control, +) -> Result { + control.check()?; + let root = open_root(&directory.0, control)?; + match mkdirat(&root, NAMESPACE, Mode::from_bits_truncate(0o700)) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(error.into()), + } + control.check()?; + let namespace = File::from(openat( + &root, + NAMESPACE, + OFlag::O_RDONLY | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC, + Mode::empty(), + )?); + let metadata = namespace.metadata()?; + if !metadata.is_dir() || !private(&metadata) { + return Err(DirectoryLeaseError::InvalidState); + } + let lock = managed_file(&namespace, LOCK, OFlag::O_RDWR | OFlag::O_CREAT, 0)?; + match lock.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => return Err(DirectoryLeaseError::InUse), + Err(TryLockError::Error(error)) => return Err(error.into()), + } + control.check()?; + let owner = match read_owner(&namespace)? { + Some(owner) => owner, + None => publish_owner(&namespace, control)?, + }; + namespace.sync_all()?; + root.sync_all()?; + control.check()?; + Ok(DirectoryLease(Arc::new(Lease { + owner, + run: Uuid::new_v4(), + _guard: LeaseGuard::Held { + _directory: namespace, + _lock: lock, + }, + }))) +} + +fn open_root(path: &Path, control: &Control) -> Result { + open_root_with_sync(path, control, File::sync_all) +} + +fn open_root_with_sync( + path: &Path, + control: &Control, + sync: impl FnMut(&File) -> io::Result<()>, +) -> Result { + let (anchor, missing) = root_anchor(path, control)?; + let mut root = File::from(open(&anchor, directory_flags(), Mode::empty())?); + trusted_directory(&root.metadata()?)?; + for component in missing.into_iter().rev() { + control.check()?; + match mkdirat( + &root, + component.as_os_str(), + Mode::from_bits_truncate(0o700), + ) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(error.into()), + } + let child = File::from(openat( + &root, + component.as_os_str(), + directory_flags(), + Mode::empty(), + )?); + trusted_directory(&child.metadata()?)?; + root = child; + } + let resolved = DataDirectory::try_from(canonicalize(path)?)?; + trusted_chain(&resolved.0, control)?; + let observed = File::from(open(&resolved.0, directory_flags(), Mode::empty())?); + let original = root.metadata()?; + let current = observed.metadata()?; + if original.dev() != current.dev() || original.ino() != current.ino() { + return Err(DirectoryLeaseError::InvalidState); + } + sync_ancestors(&resolved.0, control, sync)?; + Ok(root) +} + +fn root_anchor( + path: &Path, + control: &Control, +) -> Result<(PathBuf, Vec), DirectoryLeaseError> { + let mut missing = Vec::new(); + let mut ancestor = path; + loop { + control.check()?; + match symlink_metadata(ancestor) { + Ok(metadata) => { + if ancestor == path && metadata.file_type().is_symlink() { + return Err(DirectoryLeaseError::InvalidState); + } + break; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + missing.push( + ancestor + .file_name() + .ok_or(DirectoryLeaseError::Configuration)? + .to_owned(), + ); + ancestor = ancestor + .parent() + .ok_or(DirectoryLeaseError::Configuration)?; + } + Err(error) => return Err(error.into()), + } + } + trusted_chain(ancestor, control)?; + let resolved = DataDirectory::try_from(canonicalize(ancestor)?)?; + trusted_chain(&resolved.0, control)?; + Ok((resolved.0, missing)) +} + +fn directory_flags() -> OFlag { + OFlag::O_RDONLY | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC +} + +fn trusted_directory(metadata: &Metadata) -> Result<(), DirectoryLeaseError> { + if !metadata.is_dir() + || !trusted_uid(metadata) + || (metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0) + { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(()) +} + +fn trusted_uid(metadata: &Metadata) -> bool { + metadata.uid() == geteuid().as_raw() || metadata.uid() == 0 +} + +fn trusted_chain(path: &Path, control: &Control) -> Result<(), DirectoryLeaseError> { + for ancestor in path.ancestors() { + control.check()?; + let metadata = symlink_metadata(ancestor)?; + if metadata.file_type().is_symlink() { + if !trusted_uid(&metadata) { + return Err(DirectoryLeaseError::InvalidState); + } + } else { + trusted_directory(&metadata)?; + } + } + Ok(()) +} + +fn sync_ancestors( + path: &Path, + control: &Control, + mut sync: impl FnMut(&File) -> io::Result<()>, +) -> Result<(), DirectoryLeaseError> { + let mut device = None; + for ancestor in path.ancestors() { + control.check()?; + let directory = File::from(open(ancestor, directory_flags(), Mode::empty())?); + let metadata = directory.metadata()?; + trusted_directory(&metadata)?; + if device.is_some_and(|expected| expected != metadata.dev()) { + break; + } + device = Some(metadata.dev()); + sync(&directory)?; + } + Ok(()) +} + +fn private(metadata: &Metadata) -> bool { + metadata.uid() == geteuid().as_raw() && metadata.mode() & 0o7077 == 0 +} + +fn managed_file( + directory: &File, + name: &str, + flags: OFlag, + maximum: u64, +) -> Result { + let file = File::from(openat( + directory, + name, + flags | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NONBLOCK | OFlag::O_NOCTTY, + Mode::from_bits_truncate(0o600), + )?); + let metadata = file.metadata()?; + if !metadata.is_file() + || !private(&metadata) + || metadata.nlink() != 1 + || metadata.len() > maximum + { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(file) +} + +fn read_owner(directory: &File) -> Result, DirectoryLeaseError> { + let mut file = match managed_file(directory, OWNER, OFlag::O_RDONLY, OWNER_BYTES as u64) { + Ok(file) => file, + Err(DirectoryLeaseError::Io(error)) if error.kind() == io::ErrorKind::NotFound => { + return Ok(None); + } + Err(error) => return Err(error), + }; + if file.metadata()?.len() != OWNER_BYTES as u64 { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = [0; OWNER_BYTES]; + file.read_exact(&mut bytes)?; + let payload = bytes.get(..24).ok_or(DirectoryLeaseError::InvalidState)?; + let checksum = Sha256::digest(payload); + let expected: &[u8] = checksum.as_ref(); + if bytes.get(..8) != Some(MAGIC.as_slice()) || bytes.get(24..) != Some(expected) { + return Err(DirectoryLeaseError::InvalidState); + } + let owner = Uuid::from_slice(bytes.get(8..24).ok_or(DirectoryLeaseError::InvalidState)?) + .map_err(|_| DirectoryLeaseError::InvalidState)?; + if owner.is_nil() { + return Err(DirectoryLeaseError::InvalidState); + } + Ok(Some(owner)) +} + +fn publish_owner(directory: &File, control: &Control) -> Result { + control.check()?; + let owner = Uuid::new_v4(); + let mut bytes = [0; OWNER_BYTES]; + bytes + .get_mut(..8) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(MAGIC); + bytes + .get_mut(8..24) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(owner.as_bytes()); + let checksum = Sha256::digest(bytes.get(..24).ok_or(DirectoryLeaseError::InvalidState)?); + bytes + .get_mut(24..) + .ok_or(DirectoryLeaseError::InvalidState)? + .copy_from_slice(checksum.as_ref()); + let mut pending = managed_file( + directory, + PENDING, + OFlag::O_RDWR | OFlag::O_CREAT, + OWNER_BYTES as u64, + )?; + control.check()?; + pending.set_len(0)?; + pending.write_all(&bytes)?; + pending.sync_all()?; + control.check()?; + renameat(directory, PENDING, directory, OWNER)?; + directory.sync_all()?; + Ok(owner) +} + +#[cfg(test)] +#[path = "ownership_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership_tests.rs b/crates/rustack-lambda-core/src/docker/ownership_tests.rs new file mode 100644 index 00000000..7390b59a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership_tests.rs @@ -0,0 +1,567 @@ +use std::{ + error::Error, + os::unix::fs::{PermissionsExt, symlink}, + process::Stdio, + sync::mpsc, +}; + +use nix::unistd::mkfifo; +use tempfile::TempDir; +use tokio::{ + process::Command, + sync::oneshot, + time::{sleep, timeout}, +}; + +use super::*; +use crate::docker::ImagePinScope; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn lease(root: &TempDir) -> DirectoryLease { + DirectoryLease::acquire(root.path().to_path_buf(), deadline()) + .await + .unwrap() +} +fn managed(root: &TempDir, name: &str) -> PathBuf { + root.path().join(NAMESPACE).join(name) +} + +#[tokio::test] +async fn test_should_persist_owner_and_refresh_run_after_last_clone_closes() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let (owner, run) = (first.owner_id(), first.run_id()); + assert!(!owner.is_nil() && !run.is_nil()); + let clone = first.clone(); + drop(first); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(clone); + let second = lease(&root).await; + assert_eq!(second.owner_id(), owner); + assert_ne!(second.run_id(), run); + let bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + assert_eq!(bytes.len(), OWNER_BYTES); + assert_eq!(bytes.get(..8), Some(MAGIC.as_slice())); + assert!( + tokio::fs::try_exists(managed(&root, PENDING)) + .await + .is_ok_and(|exists| !exists) + ); +} + +#[tokio::test] +async fn test_should_hold_the_directory_through_allocated_pins() { + let root = tempfile::tempdir().unwrap(); + let owner = lease(&root).await; + let id = owner.owner_id(); + let scope = ImagePinScope::new(owner); + let pin = scope.allocate().unwrap(); + let copy = pin.clone(); + drop(scope); + drop(pin); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(copy); + assert_eq!(lease(&root).await.owner_id(), id); +} + +#[rstest::rstest] +#[case(PathBuf::new())] +#[case(PathBuf::from("relative"))] +#[case(PathBuf::from("/tmp/../outside"))] +#[case(PathBuf::from("/tmp/\0bad"))] +#[case(PathBuf::from(format!("/{}", "a".repeat(4096))))] +#[tokio::test] +async fn test_should_reject_invalid_directories_without_io(#[case] path: PathBuf) { + assert!(matches!( + DirectoryLease::acquire(path, deadline()).await, + Err(DirectoryLeaseError::Configuration) + )); +} + +#[tokio::test] +async fn test_should_create_missing_roots_and_keep_descriptor_files_private() { + let parent = tempfile::tempdir().unwrap(); + let root = parent.path().join("new/data"); + let owner = DirectoryLease::acquire(root.clone(), deadline()) + .await + .unwrap(); + for (path, expected) in [ + (root.join(NAMESPACE), 0o700), + (root.join(NAMESPACE).join(LOCK), 0o600), + (root.join(NAMESPACE).join(OWNER), 0o600), + ] { + let metadata = tokio::fs::metadata(path).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, expected); + assert_eq!(metadata.uid(), geteuid().as_raw()); + } + assert_eq!(format!("{owner:?}"), "DirectoryLease()"); +} + +#[rstest::rstest] +#[case("")] +#[case("/")] +#[case("/.")] +#[case("//./")] +#[tokio::test] +async fn test_should_reject_a_symlink_root_without_touching_the_destination(#[case] suffix: &str) { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let link = root.path().join("link"); + symlink(outside.path(), &link).unwrap(); + let spelling = PathBuf::from(format!("{}{suffix}", link.display())); + assert!(matches!( + DirectoryLease::acquire(spelling, deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!( + !tokio::fs::try_exists(outside.path().join(NAMESPACE)) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn test_should_reject_untrusted_rename_permissions_and_accept_a_sticky_ancestor() { + let root = tempfile::tempdir().unwrap(); + let parent = root.path().join("shared"); + tokio::fs::create_dir(&parent).await.unwrap(); + tokio::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o777)) + .await + .unwrap(); + let path = parent.join("new/data"); + assert!(matches!( + DirectoryLease::acquire(path.clone(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert!(!tokio::fs::try_exists(parent.join("new")).await.unwrap()); + tokio::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)) + .await + .unwrap(); + let owner = DirectoryLease::acquire(path.clone(), deadline()) + .await + .unwrap(); + let id = owner.owner_id(); + drop(owner); + assert_eq!( + DirectoryLease::acquire(path, deadline()) + .await + .unwrap() + .owner_id(), + id + ); +} + +#[tokio::test] +async fn test_should_repeat_the_ancestor_sync_barrier_after_interrupted_root_creation() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("new/parent/data"); + let test = tokio::task::spawn_blocking(move || { + let control = Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + }; + let mut attempts = 0; + let first = open_root_with_sync(&path, &control, |directory| { + attempts += 1; + if attempts == 2 { + return Err(io::Error::other("interrupted parent sync")); + } + directory.sync_all() + }); + assert!(matches!(first, Err(DirectoryLeaseError::Io(_)))); + assert!(path.is_dir()); + let parent = symlink_metadata(path.parent().unwrap()).unwrap(); + let mut observed = Vec::new(); + let reopened = open_root_with_sync(&path, &control, |directory| { + let metadata = directory.metadata()?; + observed.push((metadata.dev(), metadata.ino())); + directory.sync_all() + }) + .unwrap(); + assert!( + observed.contains(&(parent.dev(), parent.ino())), + "retry must not skip the already-existing parent" + ); + drop(reopened); + let owner = acquire(&DataDirectory::try_from(path).unwrap(), &control).unwrap(); + assert!(!owner.owner_id().is_nil()); + }); + timeout(Duration::from_secs(10), test) + .await + .unwrap() + .unwrap(); +} + +#[rstest::rstest] +#[case::namespace(NAMESPACE)] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_managed_symlinks_without_modifying_their_targets(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let sentinel = outside.path().join("sentinel"); + tokio::fs::write(&sentinel, b"sentinel").await.unwrap(); + if name == NAMESPACE { + symlink(outside.path(), root.path().join(NAMESPACE)).unwrap(); + } else { + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, name)).await.unwrap(); + symlink(&sentinel, managed(&root, name)).unwrap(); + } + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(&sentinel).await.unwrap(), b"sentinel"); +} + +#[rstest::rstest] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_hardlinked_managed_files(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let before = tokio::fs::read(managed(&root, name)).await.unwrap(); + let link = root.path().join("hardlink"); + tokio::fs::hard_link(managed(&root, name), &link) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(link).await.unwrap(), before); +} + +#[rstest::rstest] +#[case::namespace(NAMESPACE)] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_group_or_other_access(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let path = if name == NAMESPACE { + root.path().join(name) + } else { + managed(&root, name) + }; + tokio::fs::set_permissions( + &path, + std::fs::Permissions::from_mode(if name == NAMESPACE { 0o755 } else { 0o644 }), + ) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[rstest::rstest] +#[case::lock(LOCK)] +#[case::identity(OWNER)] +#[tokio::test] +async fn test_should_reject_fifos_without_waiting_for_a_peer(#[case] name: &str) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, name)).await.unwrap(); + mkfifo(&managed(&root, name), Mode::from_bits_truncate(0o600)).unwrap(); + let result = timeout( + Duration::from_secs(2), + DirectoryLease::acquire(root.path().into(), deadline()), + ) + .await + .unwrap(); + assert!(matches!(result, Err(DirectoryLeaseError::InvalidState))); +} + +#[rstest::rstest] +#[case::empty(Vec::new())] +#[case::partial(vec![0; 24])] +#[case::oversized(vec![0; OWNER_BYTES + 1])] +#[case::magic(vec![0; OWNER_BYTES])] +#[tokio::test] +async fn test_should_reject_damaged_published_identity_without_regenerating_it( + #[case] damaged: Vec, +) { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::write(managed(&root, OWNER), &damaged) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!( + tokio::fs::read(managed(&root, OWNER)).await.unwrap(), + damaged + ); +} + +#[tokio::test] +async fn test_should_reject_a_nonnil_identity_bit_flip_and_a_checksummed_nil_identity() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + let mut bytes = tokio::fs::read(managed(&root, OWNER)).await.unwrap(); + *bytes.get_mut(8).unwrap() ^= 1; + tokio::fs::write(managed(&root, OWNER), &bytes) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + bytes.get_mut(8..24).unwrap().fill(0); + let checksum = Sha256::digest(bytes.get(..24).unwrap()); + bytes + .get_mut(24..) + .unwrap() + .copy_from_slice(checksum.as_ref()); + tokio::fs::write(managed(&root, OWNER), &bytes) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[tokio::test] +async fn test_should_recover_only_a_bounded_pending_initialization() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + tokio::fs::write(managed(&root, PENDING), b"partial") + .await + .unwrap(); + tokio::fs::set_permissions( + managed(&root, PENDING), + std::fs::Permissions::from_mode(0o600), + ) + .await + .unwrap(); + let recovered = lease(&root).await; + assert!(!recovered.owner_id().is_nil()); + assert!( + !tokio::fs::try_exists(managed(&root, PENDING)) + .await + .unwrap() + ); + drop(recovered); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + tokio::fs::write(managed(&root, PENDING), vec![0; OWNER_BYTES + 1]) + .await + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); +} + +#[tokio::test] +async fn test_should_reject_unsafe_pending_initialization_without_touching_its_target() { + let root = tempfile::tempdir().unwrap(); + drop(lease(&root).await); + tokio::fs::remove_file(managed(&root, OWNER)).await.unwrap(); + let sentinel = root.path().join("sentinel"); + tokio::fs::write(&sentinel, b"outside").await.unwrap(); + symlink(&sentinel, managed(&root, PENDING)).unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InvalidState) + )); + assert_eq!(tokio::fs::read(sentinel).await.unwrap(), b"outside"); +} + +#[tokio::test] +async fn test_should_bound_deadlines_and_keep_diagnostics_redacted() { + let root = tempfile::tempdir().unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), Instant::now()).await, + Err(DirectoryLeaseError::Deadline) + )); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), Instant::now() + Duration::from_secs(61)).await, + Err(DirectoryLeaseError::Configuration) + )); + let error = DirectoryLeaseError::Io(io::Error::other("secret operator path")); + assert!(!format!("{error:?} {error}").contains("secret")); + assert!(error.source().is_some()); +} + +#[rstest::rstest] +#[case::aborted(false)] +#[case::deadline(true)] +#[tokio::test] +async fn test_should_retain_worker_capacity_and_lock_after_the_waiter_ends(#[case] expires: bool) { + let root = tempfile::tempdir().unwrap(); + let path = root.path().to_path_buf(); + let pool = Box::leak(Box::new(Semaphore::new(1))); + let (ready, acquired) = oneshot::channel(); + let (release, held) = mpsc::channel(); + let budget = if expires { + Duration::from_secs(2) + } else { + Duration::from_secs(10) + }; + let task = tokio::spawn(run(pool, Instant::now() + budget, move |control| { + let lease = acquire(&DataDirectory::try_from(path)?, control)?; + ready.send(()).unwrap(); + held.recv_timeout(Duration::from_secs(10)).unwrap(); + Ok(lease) + })); + acquired.await.unwrap(); + if expires { + assert!(matches!( + timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(), + Err(DirectoryLeaseError::Deadline) + )); + } else { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + } + assert_eq!(pool.available_permits(), 0); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + release.send(()).unwrap(); + timeout(Duration::from_secs(5), async { + loop { + match DirectoryLease::acquire(root.path().into(), deadline()).await { + Ok(lease) => { + drop(lease); + break; + } + Err(DirectoryLeaseError::InUse) => sleep(Duration::from_millis(5)).await, + other => panic!("unexpected reacquisition result: {other:?}"), + } + } + }) + .await + .unwrap(); + assert_eq!(pool.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_release_worker_capacity_after_a_panicking_worker() { + let pool = Box::leak(Box::new(Semaphore::new(1))); + let result: Result<(), _> = run(pool, deadline(), |_| panic!("worker failure")).await; + assert!(matches!(result, Err(DirectoryLeaseError::Worker))); + assert_eq!(pool.available_permits(), 1); +} + +#[tokio::test] +async fn test_should_retain_capacity_with_a_completed_but_unconsumed_lease_result() { + let root = tempfile::tempdir().unwrap(); + let held = lease(&root).await; + let pool: &'static Semaphore = Box::leak(Box::new(Semaphore::new(1))); + let task = spawn_work( + pool.try_acquire().unwrap(), + Control { + deadline: deadline(), + cancelled: Arc::new(AtomicBool::new(false)), + }, + |_| Ok(held), + ); + timeout(Duration::from_secs(5), async { + while !task.is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(pool.available_permits(), 0); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + drop(task); + assert_eq!(pool.available_permits(), 1); + drop(lease(&root).await); +} + +#[tokio::test] +async fn test_should_expire_worker_admission_without_starting_io() { + let pool: &'static Semaphore = Box::leak(Box::new(Semaphore::new(1))); + let _held = pool.acquire().await.unwrap(); + let started = Arc::new(AtomicBool::new(false)); + let observed = started.clone(); + let result = run( + pool, + Instant::now() + Duration::from_millis(20), + move |_| { + started.store(true, Ordering::Release); + Ok(()) + }, + ) + .await; + assert!(matches!(result, Err(DirectoryLeaseError::Deadline))); + assert!(!observed.load(Ordering::Acquire)); +} + +#[tokio::test] +async fn test_should_enforce_exclusion_and_stable_identity_across_processes() { + const PATH: &str = "RUSTACK_DIRECTORY_LEASE_CHILD_PATH"; + const MODE: &str = "RUSTACK_DIRECTORY_LEASE_CHILD_MODE"; + if let Some(path) = std::env::var_os(PATH) { + let result = DirectoryLease::acquire(path.into(), deadline()).await; + if std::env::var(MODE).unwrap() == "busy" { + assert!(matches!(result, Err(DirectoryLeaseError::InUse))); + } else { + let lease = result.unwrap(); + assert_eq!( + lease.owner_id().to_string(), + std::env::var("RUSTACK_DIRECTORY_LEASE_CHILD_OWNER").unwrap() + ); + assert_ne!( + lease.run_id().to_string(), + std::env::var("RUSTACK_DIRECTORY_LEASE_CHILD_RUN").unwrap() + ); + } + return; + } + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let owner = first.owner_id(); + let run = first.run_id(); + assert_child(&root, "busy", owner, run).await; + drop(first); + assert_child(&root, "available", owner, run).await; +} + +async fn assert_child(root: &TempDir, mode: &str, owner: Uuid, run: Uuid) { + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "docker::ownership::tests::test_should_enforce_exclusion_and_stable_identity_across_processes", "--test-threads=1"]) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_PATH", root.path()) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_MODE", mode) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_OWNER", owner.to_string()) + .env("RUSTACK_DIRECTORY_LEASE_CHILD_RUN", run.to_string()) + .stdout(Stdio::null()).kill_on_drop(true).spawn().unwrap(); + let status = if let Ok(status) = timeout(Duration::from_secs(10), child.wait()).await { + status.unwrap() + } else { + child.start_kill().unwrap(); + timeout(Duration::from_secs(5), child.wait()) + .await + .unwrap() + .unwrap(); + panic!("ownership child timed out"); + }; + assert!(status.success()); +} diff --git a/crates/rustack-lambda-core/src/docker/pin.rs b/crates/rustack-lambda-core/src/docker/pin.rs index bf111d1b..9b05e91c 100644 --- a/crates/rustack-lambda-core/src/docker/pin.rs +++ b/crates/rustack-lambda-core/src/docker/pin.rs @@ -8,56 +8,56 @@ use tokio::time::Instant; use uuid::Uuid; use super::{ - DockerError, DockerPort, Operation, decode, escaped, + DirectoryLease, DockerError, DockerPort, Operation, decode, escaped, models::{ImageDeleteResponseItem, ImageInspect}, unit, }; use crate::artifact::{ImageReference, Sha256Digest}; /// Trusted runtime owner/run identity; never populated from a Lambda request. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct ImagePinScope { - owner: Uuid, - run: Uuid, + lease: DirectoryLease, } impl ImagePinScope { - /// Validate the scope supplied by the directory-lock owner. + /// Retain a real directory lease for this namespace and its allocated pins. /// - /// # Errors - /// Rejects nil UUIDs. This does not acquire the runtime's directory lock. - /// - /// ``` - /// let scope = rustack_lambda_core::docker::ImagePinScope::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; - /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) + /// ```no_run + /// # fn scope(lease: rustack_lambda_core::docker::DirectoryLease) { + /// let scope = rustack_lambda_core::docker::ImagePinScope::new(lease); + /// # } /// ``` - pub fn new(owner: Uuid, run: Uuid) -> Result { + #[must_use] + pub fn new(lease: DirectoryLease) -> Self { + Self { lease } + } + + #[cfg(test)] + pub(super) fn testing(owner: Uuid, run: Uuid) -> Result { if owner.is_nil() || run.is_nil() { return Err(DockerError::Configuration); } - Ok(Self { owner, run }) + Ok(Self::new(DirectoryLease::testing(owner, run))) } /// Allocate a fresh private alias before issuing any Docker mutation. /// /// # Errors /// Returns validation failure if the generated reference cannot validate. - /// - /// ``` - /// let scope = rustack_lambda_core::docker::ImagePinScope::new(uuid::Uuid::new_v4(),uuid::Uuid::new_v4())?; - /// let pin = scope.allocate()?; - /// assert!(pin.reference().as_str().starts_with("rustack-internal/")); - /// # Ok::<(),rustack_lambda_core::docker::DockerError>(()) - /// ``` - pub fn allocate(self) -> Result { + pub(super) fn allocate(&self) -> Result { let id = Uuid::new_v4(); let reference = format!( "rustack-internal/lambda-{}/{}:{}", - self.owner.simple(), - self.run.simple(), + self.lease.owner_id().simple(), + self.lease.run_id().simple(), id.simple() ) .parse()?; - Ok(ImagePin { id, reference }) + Ok(ImagePin { + id, + reference, + _scope: self.clone(), + }) } } @@ -67,6 +67,7 @@ impl ImagePinScope { pub struct ImagePin { pub(super) id: Uuid, reference: ImageReference, + _scope: ImagePinScope, } impl fmt::Debug for ImagePin { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs index 5877770b..6107ac15 100644 --- a/crates/rustack-lambda-core/tests/docker_wire.rs +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -11,8 +11,8 @@ use http::StatusCode; use rustack_lambda_core::{ artifact::{Architecture as ImageArchitecture, ImageReference, PinnedImage}, docker::{ - DockerError, DockerKey, DockerPort, DockerSocket, DockerWire, ImageLeaseConfig, - ImageLeaseManager, ImagePinScope, OutputChannel, OwnerFilter, + DirectoryLease, DirectoryLeaseError, DockerError, DockerKey, DockerPort, DockerSocket, + DockerWire, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OutputChannel, OwnerFilter, models::{ContainerCreateBody, HostConfig}, }, }; @@ -23,6 +23,7 @@ use rustack_lambda_runtime::{ session::{Invocation, RuntimeResult}, }; use serde_json::{Value, json}; +use tempfile::TempDir; use tokio::{ io::AsyncWriteExt, sync::watch, @@ -107,6 +108,7 @@ struct Environment { runtime: Option, images: ImageLeaseManager, artifacts: Vec, + directory: Option, } impl Environment { async fn new() -> Result { @@ -114,13 +116,22 @@ impl Environment { .context("explicit test Docker socket required")?; let socket: DockerSocket = format!("unix://{socket}").parse()?; let wire = DockerWire::connect(socket, deadline()).await?; - let run = Uuid::new_v4(); - let owner = Uuid::new_v4(); + let directory = tempfile::tempdir()?; + let lease = DirectoryLease::acquire(directory.path().to_path_buf(), deadline()).await?; + let run = lease.run_id(); + let owner = lease.owner_id(); let images = ImageLeaseManager::start( wire.port(), - ImagePinScope::new(owner, run)?, + ImagePinScope::new(lease), ImageLeaseConfig::default(), )?; + ensure!( + matches!( + DirectoryLease::acquire(directory.path().to_path_buf(), deadline()).await, + Err(DirectoryLeaseError::InUse) + ), + "image owner did not retain directory exclusion" + ); Ok(Self { wire, names: Names { @@ -133,6 +144,7 @@ impl Environment { runtime: None, images, artifacts: Vec::new(), + directory: Some(directory), }) } @@ -343,12 +355,20 @@ impl Environment { } else { Ok(()) }; - function + let result = function .and(runtime) .and(relay) .and(listed) .and(images) - .and(joined) + .and(joined); + if result.is_err() { + // Uncertain cleanup still owns the namespace; retain its files for + // recovery rather than unlinking the live lock inode on test Drop. + if let Some(directory) = self.directory.take() { + let _retained = directory.keep(); + } + } + result } } diff --git a/specs/index.md b/specs/index.md index 528c12f2..45d84e02 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Persistent descriptor-backed directory ownership and pin/cleanup retention are implemented and independently reviewed (§4.1/D28); prior-run reconciliation remains pending. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 13e2527f..50030921 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -158,7 +158,7 @@ Fingerprint a versioned, explicitly ordered serialization of captured addressabl ### 2.9 Owned image pins and deployment preparation -ImageLeaseManager is a retained actor above DockerWire. ImageLeaseConfig validates1..8192 total pins (default1024),1..32 accepted preparation jobs (default8,≤pin capacity) and1..60s independent Docker IO budget (default5s). These map to validated operator budgets; no process environment is mutated. ImagePinScope contains nonnil owner/run UUIDs supplied by the runtime's directory-lock owner. Each pin allocates a fresh UUID before any mutation; its only possible alias is rustack-internal/lambda-/:, with lowercase simple UUIDs. ImagePin has no arbitrary-string constructor/deserializer. Wire tag/remove APIs accept this type; removal never targets an operator source reference, bare content ID or a generic image filter. Request removal uses force=false/noprune=true. Daemon collection of content with no remaining references is separate from Rustack deleting operator tags or pruning other images. +ImageLeaseManager is a retained actor above DockerWire. ImageLeaseConfig validates1..8192 total pins (default1024),1..32 accepted preparation jobs (default8,≤pin capacity) and1..60s independent Docker IO budget (default5s). These map to validated operator budgets; no process environment is mutated. ImagePinScope retains the actual DirectoryLease defined in [§4.1](#41-persistent-directory-lease-and-namespace-capability), which supplies nonnil owner/run UUIDs and holds namespace exclusion. Only the internal image owner allocates pins. Each pin allocates a fresh UUID before any mutation; its only possible alias is rustack-internal/lambda-/:, with lowercase simple UUIDs. ImagePin has no arbitrary-string constructor/deserializer. Wire tag/remove APIs accept this type; removal never targets an operator source reference, bare content ID or a generic image filter. Request removal uses force=false/noprune=true. Daemon collection of content with no remaining references is separate from Rustack deleting operator tags or pruning other images. The cloneable preparation port accepts a validated reference, architecture, optional ImageConfig and absolute deadline. Reserve both a preparation slot and a pin slot before bounded mailbox admission; accepted-plus-running preparations cannot exceed their configured budget. A cancelled/expired/closed request before mutation releases those reservations. Workers resolve read-only metadata, record the expected addressable parent ID, then issue tag under an independent bounded IO budget retained across caller cancellation. Validate the private alias resolves to that exact parent. Measure unpacked size before publishing a PinnedImage. Recheck caller deadline/cancellation before success. This is preparation only: the deployment actor still owns Pending/Active/update status, resource/revision CAS, allowed pulls and rollback on stale results. @@ -454,6 +454,36 @@ Cleanup ordering: invalidate attempt → cancel relay job → stop/kill function Labels include persistent data-directory ownerId, runId, environmentId and function resource UUID. Startup may reclaim old runs only for the same owner after acquiring its directory's exclusive lock. Never delete containers belonging to another Rustack instance based on a generic Lambda label; never delete user images. Keep revision/artifact references alive until invocation cleanup finishes. +### 4.1 Persistent directory lease and namespace capability + +`docker::DirectoryLease::acquire(data_directory, absolute_deadline)` issues the persistent owner UUID and a fresh run UUID only after an exclusive OS file lock and durable identity validation succeed. Use the pinned stable `File::try_lock` API on a retained standard File; all synchronous filesystem/locking work runs on at most four blocking workers. Waiting for a worker and receiving its result share the caller's absolute deadline (positive and at most60s). A ready-but-unconsumed worker result retains the work permit until the result is issued/disposed; an accepted long-lived lease does not retain worker capacity. A dropped/timed-out waiter signals cancellation but cannot interrupt an already-issued filesystem operation; its worker/result retains its descriptor and work permit until completion/disposal. No Docker mutation happens in acquisition. This is an ownership capability, not a startup-reconciliation/readiness receipt. + +Validate the operator path before IO: absolute, nonempty,≤4,096 encoded OS bytes, no NUL or parent components, and at most128 path components. Normalize the accepted component sequence before IO, removing redundant separators/current-directory components so a trailing slash or /. cannot bypass final-component NOFOLLOW. Existing ancestor aliases such as macOS /var→/private/var remain usable, but the selected root itself must not be a symlink. Validate both logical and canonical existing ancestor chains before creating anything: directory/symlink owners must be the effective UID or root, and directories must not permit group/other writes unless protected by a trusted-owner sticky bit. Operator ACLs and namespace/ancestor entries must not grant untrusted rename/unlink access or be replaced while leases exist. Create missing components with restrictive permissions relative to the pinned trusted ancestor, validating each opened component with NOFOLLOW. Every successful acquisition syncs the resolved root-to-filesystem-root ancestor chain, even on retries through already-existing paths; stop the sync barrier when crossing a device boundary, after validating the remaining trusted path. A failed/expired prior creation therefore cannot hide an unsynced parent on retry. Pin the root descriptor, create/open `.rustack-lambda-docker` relative to it, and require an actual directory owned by the effective UID with no group/other permissions. Use the already-resolved nix0.31.3 safe `openat`/`mkdirat`/`renameat` wrappers with only fs/user features; no project unsafe or raw-FD conversion. NOFOLLOW/DIRECTORY/CLOEXEC and NONBLOCK where appropriate prevent symlink/FIFO/device redirection. The managed namespace is exclusively Rustack-controlled; operators must not replace its files while leases exist. This does not promise isolation from an adversary with the same UID or daemon access. + +The namespace has three fixed children: `lease.lock` (permanent, empty regular file), `owner.v1` (authoritative immutable identity), and `owner.pending` (bounded initialization staging). Managed regular files must have effective-UID ownership, link count1 and no group/other permissions; reject unexpected file kinds or oversized/malformed contents. Lock `lease.lock` without truncating or unlinking it. A competing independent descriptor/process receives InUse. Never implement exclusivity with a PID file, an in-process registry or a replaceable lock inode. + +`owner.v1` contains exactly56 bytes: eight-byte `RSDKOWN1` magic, the16 raw bytes of a nonnil UUID, and a32-byte SHA-256 checksum over the first24 bytes. The checksum detects accidental byte corruption; it is not authentication against a same-UID writer. Existing records are read through bounded descriptor IO and rejected on malformed size/magic/UUID/checksum; never regenerate a damaged published identity. When absent under the lock, validate/reuse the fixed pending regular file (≤56 bytes), write a new complete identity, sync it, atomically rename relative to the directory descriptor, and sync both the namespace and data-root directories before issuing the lease. Repeat these syncs even when the owner record already exists, so a prior failed publication sync cannot be bypassed on retry. A first-initialization crash before publication may leave only a bounded pending file; an unsuccessful acquisition never authorizes resource creation. Published identity survives ordinary restart, while run UUID changes on every new successful acquisition. The OS releases the lock after the last owning descriptor closes or the process exits. + +`ImagePinScope::new(DirectoryLease)` replaces arbitrary UUID construction; allocation becomes internal to the retained image owner. Every ImagePin retains the scope/lease. The image preparation port, queued pin envelope, live record and emergency recovery therefore keep directory exclusion through uncertain mutations and cleanup joins. Dropping the top-level directory handle cannot release a lock still retained by any of those owners. Unit fake-daemon fixtures may use a private test-only identity constructor; production and real wire fixtures require actual leases. Read-only OwnerFilter UUID construction does not grant mutation/reclamation authority. + +```text +Coordinator Filesystem worker Image owner / cleanup + │ acquire(path, deadline) │ │ + │───────────────────────────────▶│ open root + private directory │ + │ │ NOFOLLOW, UID/mode/kind checks │ + │ │ try_lock(permanent inode) │ + │ │ read or publish+sync owner.v1 │ + │◀──── DirectoryLease ───────────│ │ + │ owner_id + fresh run_id │ │ + │ ImagePinScope(lease clone) ─────────────────────────────────────▶│ + │ drop coordinator handle │ queued/live/recovery │ + │ │◀──── descriptor still owned ───│ + │ │ last clean owner drop │ + │ │◀──── close releases lock ──────│ +``` + +Errors are typed Configuration/InUse/InvalidState/Deadline/Cancelled/Worker/Io with source IO errors, static bounded Display/Debug and no path/identity bytes in diagnostics. Apply AGENTS.md boundary lints and doc/example requirements. Tests prove stable owner/fresh run, independent-process contention/reacquisition, clone/pin lifetime, malformed/oversized/nil identity, symlink/hardlink/FIFO/mode rejection, bounded pending-file recovery, deadline/cancelled-worker retention and image-manager cleanup ownership. Real wire fixtures keep a temp data directory and use its actual lease. Production Docker selection stays unavailable until the separate startup reconciliation, historical uncertain-effect handling, pool/executor and probe consumers are implemented; a directory lease alone must not advertise a ready backend. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 144a76e6..8902a7d1 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -81,6 +81,14 @@ Deployment JSON/base64 workers have a fixed aggregate ceiling of four, retained Registry/execution mailbox capacities and per-tick quantum are compile-time implementation constants from the ASL design, not runtime tuning knobs. Function init/idle settings reuse existing validated Lambda configuration. CPU and log limits use the exact environment keys above. Effective memory admission may allow fewer than 32 environments. Lower quotas must reject admission or fail the affected execution explicitly, never silently drop data/work. +### 1.3 Persistent Docker namespace ownership + +Acquire the existing configured data directory before constructing a mutating image owner. [Runtime §4.1](./rustack-workflows-20-lambda-runtime-design.md#41-persistent-directory-lease-and-namespace-capability) defines its descriptor lease, immutable checksummed owner UUID, fresh run UUID and filesystem trust rules. Acquisition has a positive absolute deadline≤60s and a fixed four-worker IO ceiling. Require an operator-controlled local filesystem with advisory locking, atomic rename and directory sync support; an unsupported filesystem fails acquisition rather than falling back to an unlocked namespace. ACLs must not grant untrusted namespace/ancestor replacement, and operators must not unlink/replace live lock files or data roots. Same-UID/root or Docker-daemon adversaries remain outside this local execution isolation promise. + +The permanent lease.lock inode is never truncated, replaced or deleted during release. Last-descriptor closure releases exclusion; image scopes, pins, queued work and quarantined cleanup retain clones until their ownership settles. A failed/cancelled acquisition may leave bounded initialization files, which are validated on retry. A corrupt published identity fails closed and is never silently regenerated. Errors contain static classes without operator paths or identity values. No Docker effect is authorized until acquisition succeeds. + +This capability alone is not a ready backend. Startup must separately reconcile prior runs and unresolved historical effects under the same persistent owner before accepting image execution. Do not treat one empty list/404 as proof that a prior uncertain daemon mutation cannot appear later. Shutdown keeps the lease until environment/deployment/image cleanup owners are joined; unresolved cleanup prevents successful quiescence/snapshot publication. + ## 2. Deployment topology and readiness | Topology | Control path | Handler SDK path | Required test | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index e484db3c..f11eef55 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -100,6 +100,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi ## 4. Security acceptance +`make test-lambda-ownership` verifies the real directory capability independently of Docker: stable owner/fresh run, descriptor/allocated-pin lifetime, actual subprocess contention and reacquisition, bounded malformed/pending state, symlink spellings (including trailing slash/current-directory), hardlink/FIFO/permission rejection, trusted/sticky ancestor rules and repeated parent sync after interrupted creation. Worker tests cover admission expiry, running deadline/cancelled waiter retention, completed-but-unconsumed result capacity and panic release. A scripted-daemon integration additionally holds a real directory lease through abandoned image-owner quarantine and delayed deletion, requiring exclusion until cleanup settles. Production wire fixtures acquire actual directory leases and retain their namespace files on uncertain cleanup. These checks do not establish prior-run reconciliation, startup readiness or full M0 acceptance. + | Boundary | Required proof | |---|---| | HTTP/ASL/archives | Byte/count/depth bounds before expansion; duplicate-key rejection; invalid values cannot reach unchecked engine indexing | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index b149b252..ef7af329 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, existing gateway/YAML/runtime/snapshot infrastructure. Missing: provider asynchronous image publication/status/CAS, leased invocation capture, Docker environment/pool executor, startup ownership/reconciliation/probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker environment/pool executor, startup ownership/reconciliation/probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -143,6 +143,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.1 | Foundation implemented and reviewed; P1 gates remain open | Resource/revision capture, closed artifact/architecture types, shared ZIP ownership, validated immutable image metadata, early settings preflight and transactional staging rollback are implemented. Arc ExecutionRevision and cooperative InvocationHandle/WorkSupervisor now preserve attempts, deadlines, cleanup ownership and bounded admission across Native/Squib. Independent W13–W15 findings fixed/rechecked. Workspace:1,767 passing tests; Lambda:215 unit tests, five real Native fixture tests and native provider/DryRun regressions pass. Actual Squib VM regression is not claimed. Baseline B01/B03 are now closed by the security and six-library strict passes in §§7.1–7.2; remaining P1 runtime acceptance still applies. | | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | | 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. B01 dependency closure is implemented/reviewed (§7.2/D27, W57–W59): nine ZIP regressions, workspace2,083 tests,833 boundary tests/doctests, green standard/strict gates, clean audit and all deny checks. Regional SDK/HTTP and Native provider regressions pass. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | +| 1.3 directory namespace | Implemented/reviewed; runtime consumers remain | DirectoryLease binds permanent nonreplaced file locking, durable checksummed owner/fresh run and trusted descriptor-relative filesystem operations to all image scopes/pins/jobs/recovery. W60–W62 fixes independently rechecked; 38 ownership cases plus abandoned image-owner cleanup pass. Workspace2,122 tests, core443 tests/103 doctests, boundary874 unit/doc passes, all standard/strict/audit/deny gates pass. Both real two-case wire suites pass on arm64 VZ and amd64/QEMU; post-test owned containers/internal aliases are empty. This does not prove previous-run reconciliation, Linux-host lock qualification or Docker executor readiness. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. Next implement task1.3 Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 6b838e23..61d246c9 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -273,6 +273,24 @@ The independent read-only review verifies the released ZIP9/source semantics, ra Final-source workspace build/test/fmt/Clippy/rustdoc pass with2,083 tests, zero failed and401 resource-gated cases. Core has404 tests (nine new ZIP cases) and101 doctests; `make test-lambda-boundaries` passes668 unit tests/165 doctests (833 total), seven resource-gated Docker cases, and all six strict production-library checks. The Native provider target passes six actual Native/validation scenarios; its gated Squib selection is excluded as VM evidence. Post-upgrade real Rust SDK/HTTP validation and revision/412 tests each pass with zero ignored, and the owned test gateway exits normally. Audit/deny pass against the same committed dependency graph. B01/B03/B04 are closed; persistent directory ownership/reconciliation, allowed pulls, Docker environment/pool executor, startup/configuration/probes and full M0 acceptance remain required. P2–P5 also remain required. +## P1 task1.3 persistent namespace lease review + +Independent read-only review covers runtime §4.1/D28: bounded acquisition, private descriptor-relative files, immutable durable identity, raw-path normalization, ancestor trust, process contention and retention through image cleanup. The final recheck finds no remaining in-slice issue. This is host-directory ownership evidence, not prior-run reconciliation, Docker readiness or P1/M0 acceptance. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W60 | P2; `docker/ownership.rs`, raw root spelling | A trailing slash or /. can bypass final-component NOFOLLOW. Normalize the validated component sequence before any IO | Four root-symlink spellings reject before touching the destination | +| W61 | P2; `docker/ownership.rs`, worker result ownership | Releasing the work permit when blocking work finishes allows ready results to accumulate outside capacity. Return the permit with the result until delivery/disposal | Completed-but-unconsumed result retains capacity and the actual directory lock; abort/deadline cases retain both until worker completion | +| W62 | P2; `docker/ownership.rs`, interrupted creation retry | Syncing only newly created parents lets a retry skip a parent left unsynced by an earlier failed attempt. Repeat the same-filesystem ancestor barrier on every acquisition | Fault at the second directory sync, then successful retry through existing parents proves the parent inode is synced | + +Logical and canonical ancestor checks also reject untrusted UID or writable nonsticky rename paths before creation; sticky trusted ancestors remain supported. Safe nix0.31.3 fs/user calls reuse the already-resolved package, and standard File supplies advisory locking. The only new lint exception is ownership.rs:5 for synchronous standard-file/descriptor operations, all executed on bounded blocking workers because Tokio files do not supply OS advisory locks. No unsafe, production panic/unwrap, ignored boundary finding or dependency-security exception is added. + +Final-source standard workspace build/test/fmt/Clippy/rustdoc pass:2,122 tests, zero failed and401 separately resource-gated cases. Lambda core has443 unit tests and103 doctests; the new ownership target has38 directory cases plus one abandoned-manager cleanup case, all passing. The actual subprocess test checks contention while held and reacquisition after release with the same owner/fresh run. The six-library strict boundary target passes707 unit tests/167 doctests (874 total), seven separately gated Docker cases and all strict production checks. Fresh audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources all pass. + +Both explicit two-case Docker wire suites pass using real DirectoryLease-backed image owners on Engine29.5.2 arm64 VZ and amd64/QEMU. Afterward both daemons have no owner-labeled containers or rustack-internal/lambda aliases. The host-directory tests run on macOS; this is not native Linux directory-lock qualification. The harness retains60s stream/120s attempt stress budgets. The local Docker CLI lacks an auto-discovered buildx plugin, so these successful Make invocations explicitly set WORKFLOWS_IMAGE_BUILDER=/opt/homebrew/bin/docker-buildx. Initial failure with the default builder occurred before any test allocation; no default Docker/runtime readiness is claimed. + +Persistent namespace capability is now implemented/reviewed. Startup historical-effect reconciliation, allowed pulls, remaining log/network/probe consumers, environment/pool executor and production configuration/acceptance remain P1 requirements. P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 11b32eae..8f643fcf 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -224,6 +224,15 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Pinned by:** [runtime §7.2](./rustack-workflows-20-lambda-runtime-design.md#72-dependency-security-closure-b01), [HTTP authentication profile](./rustack-http-hardening-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-25. SDK model versions and supported AWS wire modes are preserved. +## D28 — Retain an OS directory lease in image namespace capabilities + +- **Context:** owner/run UUIDs alone cannot exclude another process, and a coordinator-only lock can close while retained Docker cleanup still runs. +- **Alternatives considered:** PID files/in-process maps do not establish OS exclusion; replacing a lock file splits its inode ownership; arbitrary caller-supplied UUID scopes lose the filesystem proof; following path-based symlinks can redirect managed writes. +- **Decision:** acquire a permanent advisory-locked descriptor in a private UID-checked directory, durably publish a bounded immutable owner record, and clone the lease through the scope/pin/job/recovery lifetime. Use safe nix0.31.3 relative-descriptor operations and standard File locking on bounded blocking workers; do not expose arbitrary production scope constructors. The [current nix openat API](https://docs.rs/nix/0.31.3/nix/fcntl/fn.openat.html) returns an OwnedFd with fs enabled; [standard File locking](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.try_lock) supplies nonblocking advisory exclusion. Verified 2026-10-26; no new package version is required. +- **Why:** process exclusion must last through cleanup settlement, persisted corruption must fail closed, and a failed/cancelled initialization cannot authorize Docker effects. Retaining a standard File is a deliberate lint exception because Tokio has no advisory-lock API; synchronous IO is confined to blocking workers. This capability does not prove previous-run reconciliation or backend readiness. +- **Pinned by:** [runtime §4.1](./rustack-workflows-20-lambda-runtime-design.md#41-persistent-directory-lease-and-namespace-capability), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-26. + ## Glossary | Term | Meaning in this specification | From df69fbf5a50835531dcf4caf84db240c3cf9ac7a Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 00:36:12 -0700 Subject: [PATCH 021/106] phase 1: retain bounded function logs independently of control IO Implement runtime section 3.7 and D29 using output-only live attach, separate bounded binary channel rings and combined 4KiB tail, retained partial reads across snapshot requests, fair mailbox/read service, early terminal socket closure and cancel-safe cached shutdown. Independent review W63-W64 corrections are verified. All 18 focused cases, 2140 workspace tests, 896 strict boundary unit/doc cases and standard build/fmt/Clippy/rustdoc gates pass. Both two-case real wire suites capture handler stdout/stderr separately from computed responses on arm64 VZ and amd64/QEMU and leave no owned resources. No new dependency or lint exception. Environment integration, historical reconciliation, pulls, network/probes, production executor and P2-P5 remain required; M0 is not closed. --- Makefile | 6 + crates/rustack-lambda-core/src/docker/logs.rs | 345 +++++++++++++++++ .../src/docker/logs_tests.rs | 359 ++++++++++++++++++ crates/rustack-lambda-core/src/docker/mod.rs | 33 +- .../rustack-lambda-core/tests/docker_wire.rs | 40 +- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 10 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 3 +- specs/rustack-workflows-93-review.md | 15 + specs/rustack-workflows-99-decisions.md | 9 + tools/workflows-spikes/handler.py | 4 + 12 files changed, 822 insertions(+), 6 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/logs.rs create mode 100644 crates/rustack-lambda-core/src/docker/logs_tests.rs diff --git a/Makefile b/Makefile index 00c80f78..5287a4b6 100644 --- a/Makefile +++ b/Makefile @@ -306,6 +306,12 @@ test-lambda-ownership: .PHONY: test-lambda-ownership +# Bounded function output, snapshot/read fairness and joined attachment cleanup. +test-lambda-logs: + @cargo test -p rustack-lambda-core --lib docker::logs::tests + +.PHONY: test-lambda-logs + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: diff --git a/crates/rustack-lambda-core/src/docker/logs.rs b/crates/rustack-lambda-core/src/docker/logs.rs new file mode 100644 index 00000000..a2914a60 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs.rs @@ -0,0 +1,345 @@ +//! Independent function output capture; socket reads never wait for a consumer. + +use std::{collections::VecDeque, fmt, time::Duration}; + +use bytes::Bytes; +use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use tokio::{ + io::AsyncRead, + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use super::{ + DockerAttachment, DockerError, DockerOutputReader, OutputChannel, OutputChunk, check_deadline, +}; + +const TAIL: usize = 4096; +const SILENCE: Duration = Duration::from_secs(3600); + +/// Bounded binary API output and capture status; Debug never prints user bytes. +#[derive(Clone)] +#[non_exhaustive] +pub struct LogSnapshot { + /// The final 4 KiB in observed stdout/stderr chunk order, without UTF-8 conversion. + pub tail: Bytes, + /// Old stdout bytes overwritten in its independent ring. + pub stdout_discarded: u64, + /// Old stderr bytes overwritten in its independent ring. + pub stderr_discarded: u64, + /// The attachment reached EOF; container exit remains independently owned. + pub closed: bool, + /// A retained framing/IO/deadline failure; joined shutdown preserves it. + pub failure: Option, +} +impl fmt::Debug for LogSnapshot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("LogSnapshot") + .field("bytes", &self.tail.len()) + .field("stdout_discarded", &self.stdout_discarded) + .field("stderr_discarded", &self.stderr_discarded) + .field("closed", &self.closed) + .field("failure", &self.failure) + .finish() + } +} + +struct SnapshotRequest { + deadline: Instant, + reply: oneshot::Sender>, +} + +/// Retained log actor; its environment owns the independent container lifetime. +#[non_exhaustive] +pub struct FunctionLogs { + requests: mpsc::Sender, + stop: watch::Sender, + task: Option>>, + settled: Option>, +} +impl fmt::Debug for FunctionLogs { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FunctionLogs") + .field("stopping", &*self.stop.borrow()) + .field("settled", &self.settled) + .finish_non_exhaustive() + } +} +impl FunctionLogs { + /// Start output capture with a4KiB..1MiB ring for each channel. + /// + /// # Errors + /// Rejects invalid ring/record budgets, unavailable runtime or allocation. + /// + /// ```no_run + /// # fn start(attachment: rustack_lambda_core::docker::DockerAttachment) -> Result<(), rustack_lambda_core::docker::DockerError> { + /// let logs = rustack_lambda_core::docker::FunctionLogs::start( + /// attachment, std::time::Duration::from_secs(5), 262_144)?; + /// # Ok(()) } + /// ``` + pub fn start( + attachment: DockerAttachment, + io_budget: Duration, + ring_bytes: usize, + ) -> Result { + let reader = DockerOutputReader::new(attachment.0, io_budget)?; + Self::start_reader(reader, ring_bytes) + } + + fn start_reader( + reader: DockerOutputReader, + ring_bytes: usize, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| DockerError::Worker)?; + if !(TAIL..=1024 * 1024).contains(&ring_bytes) { + return Err(DockerError::Configuration); + } + let state = Capture::new(ring_bytes)?; + let (requests, received) = mpsc::channel(8); + let (stop, stopped) = watch::channel(false); + let task = runtime + .spawn(capture(reader, state, received, stopped).with_subscriber(runtime_dispatch())); + Ok(Self { + requests, + stop, + task: Some(task), + settled: None, + }) + } + + /// Read a bounded tail/status without interrupting a partial record. + /// + /// # Errors + /// Returns expired/invalid budget, full mailbox, stopped actor or worker failure. + /// + /// ```no_run + /// # async fn snapshot(logs: &rustack_lambda_core::docker::FunctionLogs) { + /// let snapshot = logs.snapshot(tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn snapshot(&self, deadline: Instant) -> Result { + check_deadline(deadline)?; + if let Some(result) = &self.settled { + return result.clone(); + } + if *self.stop.borrow() { + return Err(DockerError::Stopped); + } + let (reply, response) = oneshot::channel(); + self.requests + .try_send(SnapshotRequest { deadline, reply }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => DockerError::Busy, + mpsc::error::TrySendError::Closed(_) => DockerError::Worker, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| DockerError::Deadline)? + .map_err(|_| DockerError::Worker)?; + if Instant::now() >= deadline { + return Err(DockerError::Deadline); + } + result + } + + /// Stop and join socket ownership, preserving failure across cancelled waiters. + /// + /// # Errors + /// Returns a retained parser/IO/deadline failure or failed actor join. + /// Container cleanup must be awaited separately by the environment owner. + /// + /// ```no_run + /// # async fn stop(logs: &mut rustack_lambda_core::docker::FunctionLogs) { + /// let final_tail = logs.shutdown().await; + /// # } + /// ``` + pub async fn shutdown(&mut self) -> Result { + self.stop.send_replace(true); + if let Some(result) = &self.settled { + return result.clone(); + } + let result = match &mut self.task { + Some(task) => task + .await + .map_err(|_| DockerError::Worker) + .and_then(|result| result), + None => Err(DockerError::Worker), + }; + self.task.take(); + self.settled = Some(result.clone()); + result + } +} +impl Drop for FunctionLogs { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +struct Ring { + bytes: VecDeque, + capacity: usize, + discarded: u64, +} +impl Ring { + fn new(capacity: usize) -> Result { + let mut bytes = VecDeque::new(); + bytes + .try_reserve_exact(capacity) + .map_err(|_| DockerError::Allocation)?; + Ok(Self { + bytes, + capacity, + discarded: 0, + }) + } + fn push(&mut self, bytes: &[u8]) -> Result<(), DockerError> { + let skipped = self + .bytes + .len() + .saturating_add(bytes.len()) + .saturating_sub(self.capacity); + self.discarded = self + .discarded + .saturating_add(u64::try_from(skipped).map_err(|_| DockerError::TooLarge)?); + let remove = skipped.min(self.bytes.len()); + self.bytes.drain(..remove); + let offset = bytes.len().saturating_sub(self.capacity); + self.bytes + .extend(bytes.get(offset..).ok_or(DockerError::Protocol)?); + Ok(()) + } +} +struct Capture { + stdout: Ring, + stderr: Ring, + tail: Ring, + closed: bool, + failure: Option, +} +impl Capture { + fn new(capacity: usize) -> Result { + Ok(Self { + stdout: Ring::new(capacity)?, + stderr: Ring::new(capacity)?, + tail: Ring::new(TAIL)?, + closed: false, + failure: None, + }) + } + fn push(&mut self, chunk: &OutputChunk) -> Result<(), DockerError> { + match chunk.channel { + OutputChannel::Stdout => self.stdout.push(&chunk.bytes)?, + OutputChannel::Stderr => self.stderr.push(&chunk.bytes)?, + } + self.tail.push(&chunk.bytes) + } + fn snapshot(&self) -> LogSnapshot { + LogSnapshot { + tail: Bytes::from(self.tail.bytes.iter().copied().collect::>()), + stdout_discarded: self.stdout.discarded, + stderr_discarded: self.stderr.discarded, + closed: self.closed, + failure: self.failure.clone(), + } + } + fn reply(&self, request: SnapshotRequest) { + if request.reply.is_closed() { + return; + } + let result = if Instant::now() >= request.deadline { + Err(DockerError::Deadline) + } else { + Ok(self.snapshot()) + }; + let _ = request.reply.send(result); + } +} + +async fn capture( + reader: DockerOutputReader, + mut state: Capture, + mut requests: mpsc::Receiver, + mut stop: watch::Receiver, +) -> Result { + let mut reader = Some(reader); + let mut quantum = 0_u8; + loop { + if *stop.borrow() { + break; + } + if state.closed || state.failure.is_some() { + tokio::select! { + biased; + () = super::connection::stopped(&mut stop) => break, + request = requests.recv() => match request { Some(request) => state.reply(request), None => break }, + } + continue; + } + let result = match &mut reader { + Some(reader) => read_next(reader, &state, &mut requests, &mut stop).await, + None => return Err(DockerError::Worker), + }; + match result { + None => break, + Some(Ok(None)) => state.closed = true, + Some(Err(error)) => state.failure = Some(error), + Some(Ok(Some(chunk))) => { + if let Err(error) = state.push(&chunk) { + state.failure = Some(error); + } + quantum = quantum.saturating_add(1); + if quantum == 64 { + tokio::task::yield_now().await; + quantum = 0; + } + } + } + if state.closed || state.failure.is_some() { + reader.take(); + } + } + drop(reader); + requests.close(); + while let Ok(request) = requests.try_recv() { + state.reply(request); + } + match state.failure.clone() { + Some(error) => Err(error), + None => Ok(state.snapshot()), + } +} + +async fn read_next( + reader: &mut DockerOutputReader, + state: &Capture, + requests: &mut mpsc::Receiver, + stop: &mut watch::Receiver, +) -> Option, DockerError>> { + // This future survives snapshot traffic. Only terminal stop drops it. + let next = reader.next(Instant::now() + SILENCE); + tokio::pin!(next); + loop { + if *stop.borrow() { + return None; + } + if let Ok(request) = requests.try_recv() { + state.reply(request); + } + tokio::select! { + biased; + () = super::connection::stopped(stop) => return None, + result = &mut next => return Some(result), + request = requests.recv() => match request { Some(request) => state.reply(request), None => return None }, + } + } +} + +#[cfg(test)] +#[path = "logs_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/logs_tests.rs b/crates/rustack-lambda-core/src/docker/logs_tests.rs new file mode 100644 index 00000000..bedf0534 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs_tests.rs @@ -0,0 +1,359 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll, Waker}, +}; + +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, DuplexStream, ReadBuf, duplex}, + time::{advance, timeout}, +}; + +use super::*; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} +fn logs(reader: DuplexStream) -> FunctionLogs { + FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + TAIL, + ) + .unwrap() +} +fn record(channel: u8, bytes: &[u8]) -> Vec { + let mut frame = vec![channel, 0, 0, 0]; + frame.extend_from_slice(&u32::try_from(bytes.len()).unwrap().to_be_bytes()); + frame.extend_from_slice(bytes); + frame +} +async fn closed(logs: &FunctionLogs) -> LogSnapshot { + timeout(Duration::from_secs(5), async { + loop { + let snapshot = logs.snapshot(deadline()).await.unwrap(); + if snapshot.closed || snapshot.failure.is_some() { + break snapshot; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap() +} + +#[tokio::test] +async fn test_should_capture_binary_channels_in_observed_order_and_retain_tail_after_eof() { + let (mut writer, reader) = duplex(1024); + let mut logs = logs(reader); + writer.write_all(&record(1, b"stdout\0\xff")).await.unwrap(); + writer.write_all(&record(2, b"stderr\n")).await.unwrap(); + writer.shutdown().await.unwrap(); + let snapshot = closed(&logs).await; + assert_eq!(snapshot.tail.as_ref(), b"stdout\0\xffstderr\n"); + assert_eq!(snapshot.stdout_discarded, 0); + assert_eq!(snapshot.stderr_discarded, 0); + assert!(snapshot.failure.is_none()); + assert!(!format!("{snapshot:?} {logs:?}").contains("stdout\\0")); + assert!(!format!("{snapshot:?}").contains("stderr\\n")); + assert_eq!(logs.snapshot(deadline()).await.unwrap().tail, snapshot.tail); + assert_eq!(logs.shutdown().await.unwrap().tail, snapshot.tail); + assert_eq!(logs.shutdown().await.unwrap().tail, snapshot.tail); +} + +#[tokio::test] +async fn test_should_bound_independent_rings_and_saturate_discard_counts() { + let mut capture = Capture::new(TAIL).unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stdout, + bytes: Bytes::from(vec![1; TAIL + 17]), + }) + .unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stderr, + bytes: Bytes::from(vec![2; TAIL + 9]), + }) + .unwrap(); + capture + .push(&OutputChunk { + channel: OutputChannel::Stdout, + bytes: Bytes::from_static(b"final"), + }) + .unwrap(); + assert_eq!(capture.stdout.bytes.len(), TAIL); + assert_eq!(capture.stderr.bytes.len(), TAIL); + assert_eq!(capture.stdout.discarded, 22); + assert_eq!(capture.stderr.discarded, 9); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.tail.len(), TAIL); + assert!( + snapshot + .tail + .get(..TAIL - 5) + .unwrap() + .iter() + .all(|byte| *byte == 2) + ); + assert_eq!(snapshot.tail.get(TAIL - 5..), Some(b"final".as_slice())); + capture.stdout.discarded = u64::MAX - 1; + capture.stdout.push(b"more").unwrap(); + assert_eq!(capture.stdout.discarded, u64::MAX); +} + +#[rstest::rstest] +#[case(0)] +#[case(TAIL - 1)] +#[case(1024 * 1024 + 1)] +#[tokio::test] +async fn test_should_reject_invalid_log_capacity(#[case] capacity: usize) { + let (_, reader) = duplex(1); + assert!(matches!( + FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + capacity + ), + Err(DockerError::Configuration) + )); +} + +#[tokio::test] +async fn test_should_keep_a_fragmented_record_alive_through_cancelled_and_completed_snapshots() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + let frame = record(1, b"secret-frame"); + writer.write_all(frame.get(..4).unwrap()).await.unwrap(); + for _ in 0..16 { + assert!(logs.snapshot(deadline()).await.unwrap().tail.is_empty()); + } + { + let snapshot = logs.snapshot(deadline()); + tokio::pin!(snapshot); + assert!( + snapshot + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + } + writer.write_all(frame.get(4..).unwrap()).await.unwrap(); + writer.shutdown().await.unwrap(); + assert_eq!(closed(&logs).await.tail.as_ref(), b"secret-frame"); + assert!(logs.shutdown().await.is_ok()); +} + +#[tokio::test(start_paused = true)] +async fn test_should_not_extend_the_partial_record_deadline_with_snapshot_traffic() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + writer.write_all(&[1, 0, 0, 0]).await.unwrap(); + for _ in 0..12 { + let _ = logs.snapshot(deadline()).await.unwrap(); + advance(Duration::from_millis(100)).await; + } + let failed = logs.snapshot(deadline()).await.unwrap(); + assert!(matches!(failed.failure, Some(DockerError::Deadline))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); +} + +#[tokio::test(start_paused = true)] +async fn test_should_bound_first_byte_silence_independently_of_snapshot_requests() { + let (_writer, reader) = duplex(1); + let mut logs = logs(reader); + logs.snapshot(deadline()).await.unwrap(); + advance(SILENCE + Duration::from_secs(1)).await; + assert!(matches!( + logs.snapshot(deadline()).await.unwrap().failure, + Some(DockerError::Deadline) + )); + assert!(matches!(logs.shutdown().await, Err(DockerError::Deadline))); +} + +#[tokio::test] +async fn test_should_retain_shutdown_ownership_after_its_waiter_is_cancelled() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + { + let shutdown = logs.shutdown(); + tokio::pin!(shutdown); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + } + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); + assert!(writer.write_all(b"x").await.is_err()); +} + +#[tokio::test] +async fn test_should_signal_quiet_socket_cleanup_on_drop_outside_an_entered_runtime() { + let (mut writer, reader) = duplex(1); + let logs = logs(reader); + let thread = std::thread::spawn(move || { + assert!(Handle::try_current().is_err()); + drop(logs); + }); + let task = tokio::task::spawn_blocking(move || thread.join().unwrap()); + timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn test_should_serve_snapshots_and_stop_while_output_floods() { + let (mut writer, reader) = duplex(64 * 1024); + let mut logs = logs(reader); + let producer = tokio::spawn(async move { + let frame = record(1, &vec![b'x'; 16 * 1024]); + while writer.write_all(&frame).await.is_ok() { + tokio::task::yield_now().await; + } + }); + timeout(Duration::from_secs(5), async { + loop { + let snapshot = logs.snapshot(deadline()).await.unwrap(); + assert!(snapshot.tail.len() <= TAIL); + if snapshot.stdout_discarded > 64 * 1024 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); + timeout(Duration::from_secs(1), producer) + .await + .unwrap() + .unwrap(); +} + +#[rstest::rstest] +#[case::bad_channel(vec![3, 0, 0, 0, 0, 0, 0, 0])] +#[case::reserved(vec![1, 1, 0, 0, 0, 0, 0, 0])] +#[case::truncated_header(vec![1, 0])] +#[case::truncated_payload(vec![1, 0, 0, 0, 0, 0, 0, 2, 1])] +#[tokio::test] +async fn test_should_preserve_malformed_stream_failure_through_shutdown(#[case] frame: Vec) { + let (mut writer, reader) = duplex(64); + let mut logs = logs(reader); + writer.write_all(&frame).await.unwrap(); + writer.shutdown().await.unwrap(); + assert!(closed(&logs).await.failure.is_some()); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} + +struct Panics; +impl AsyncRead for Panics { + fn poll_read( + self: Pin<&mut Self>, + _: &mut Context<'_>, + _: &mut ReadBuf<'_>, + ) -> Poll> { + panic!("log worker failure") + } +} +#[tokio::test] +async fn test_should_cache_log_actor_join_failure() { + let mut logs = FunctionLogs::start_reader( + DockerOutputReader::new(Panics, Duration::from_secs(1)).unwrap(), + TAIL, + ) + .unwrap(); + while !logs.task.as_ref().unwrap().is_finished() { + tokio::task::yield_now().await; + } + assert!(matches!( + logs.snapshot(deadline()).await, + Err(DockerError::Worker) + )); + assert!(matches!(logs.shutdown().await, Err(DockerError::Worker))); + assert!(matches!(logs.shutdown().await, Err(DockerError::Worker))); +} + +#[tokio::test] +async fn test_should_bound_snapshot_admission_and_discard_abandoned_requests() { + let (_writer, reader) = duplex(1); + let mut logs = logs(reader); + { + let mut requests = Vec::new(); + for _ in 0..8 { + let mut request = Box::pin(logs.snapshot(deadline())); + assert!( + request + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + requests.push(request); + } + assert!(matches!( + logs.snapshot(deadline()).await, + Err(DockerError::Busy) + )); + drop(requests); + } + assert!( + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .is_ok() + ); +} + +#[tokio::test] +async fn test_should_use_output_only_attach_without_replay_and_transfer_driver_ownership() { + let mut daemon = super::super::tests::Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let attach = tokio::spawn(async move { + port.attach_logs(&"function".parse().unwrap(), deadline()) + .await + }); + let request = daemon.next().await; + assert!(request.head.starts_with( + "POST /v1.53/containers/function/attach?stdin=false&stdout=true&stderr=true&stream=true&\ + logs=false HTTP/1.1\r\n" + )); + let mut reply = + b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n".to_vec(); + reply.extend_from_slice(&record(2, b"wire-secret-output")); + request.raw(&reply).await; + let mut logs = + FunctionLogs::start(attach.await.unwrap().unwrap(), Duration::from_secs(1), TAIL).unwrap(); + wire.shutdown().await.unwrap(); + assert_eq!(closed(&logs).await.tail.as_ref(), b"wire-secret-output"); + assert!(logs.shutdown().await.is_ok()); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 94431d31..2652a6ac 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -7,6 +7,7 @@ mod connection; mod framing; mod image; mod lease; +mod logs; mod ownership; mod pin; mod types; @@ -29,6 +30,7 @@ use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; pub use image::ImageResolver; pub use lease::{ImageLeaseConfig, ImageLeaseManager, ImageLeasePort, ImageLeaseStatus}; +pub use logs::{FunctionLogs, LogSnapshot}; use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, @@ -514,15 +516,42 @@ impl DockerPort { &self, relay: &DockerKey, deadline: Instant, + ) -> Result { + self.attach_inner(relay, true, deadline).await + } + + /// Attach only live function output before start, without stdin or replay. + /// + /// # Errors + /// Returns admission, upgrade, status, deadline or joined transport failure. + /// + /// ```no_run + /// # async fn logs(port: &rustack_lambda_core::docker::DockerPort, function: &rustack_lambda_core::docker::DockerKey) { + /// let attached = port.attach_logs(function, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn attach_logs( + &self, + function: &DockerKey, + deadline: Instant, + ) -> Result { + self.attach_inner(function, false, deadline).await + } + + async fn attach_inner( + &self, + key: &DockerKey, + stdin: bool, + deadline: Instant, ) -> Result { let reply = self .call( Operation::Attach, Method::POST, format!( - "/containers/{}/attach?stdin=true&stdout=true&stderr=true&stream=true&\ + "/containers/{}/attach?stdin={stdin}&stdout=true&stderr=true&stream=true&\ logs=false", - escaped(relay.as_str()) + escaped(key.as_str()) ), Bytes::new(), StatusCode::SWITCHING_PROTOCOLS, diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs index 6107ac15..2f585fab 100644 --- a/crates/rustack-lambda-core/tests/docker_wire.rs +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -12,7 +12,8 @@ use rustack_lambda_core::{ artifact::{Architecture as ImageArchitecture, ImageReference, PinnedImage}, docker::{ DirectoryLease, DirectoryLeaseError, DockerError, DockerKey, DockerPort, DockerSocket, - DockerWire, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OutputChannel, OwnerFilter, + DockerWire, FunctionLogs, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, + OutputChannel, OwnerFilter, models::{ContainerCreateBody, HostConfig}, }, }; @@ -106,6 +107,7 @@ struct Environment { wire: DockerWire, names: Names, runtime: Option, + logs: Option, images: ImageLeaseManager, artifacts: Vec, directory: Option, @@ -142,6 +144,7 @@ impl Environment { environment: EnvironmentId::default(), }, runtime: None, + logs: None, images, artifacts: Vec::new(), directory: Some(directory), @@ -248,6 +251,11 @@ impl Environment { deadline(), ) .await?; + self.logs = Some(FunctionLogs::start( + port.attach_logs(&self.names.function, deadline()).await?, + IO, + 262_144, + )?); port.start_container(&self.names.function, deadline()) .await?; ensure!( @@ -325,6 +333,11 @@ impl Environment { async fn cleanup(&mut self) -> Result<()> { let port = self.wire.port(); let function = remove(&port, &self.names.function).await; + let logs = if let Some(logs) = &mut self.logs { + logs.shutdown().await.map(drop).map_err(anyhow::Error::from) + } else { + Ok(()) + }; let runtime = if let Some(runtime) = &mut self.runtime { runtime.shutdown().await } else { @@ -356,6 +369,7 @@ impl Environment { Ok(()) }; let result = function + .and(logs) .and(runtime) .and(relay) .and(listed) @@ -403,7 +417,8 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra let mut environment = Environment::new().await?; let result = async { environment.start().await?; - let event = json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE}); + let event = + json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE,"logs":true}); let computed = environment.call(&event, false).await?; ensure!( computed["computed"] == 42 @@ -434,6 +449,27 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra failure["errorType"] == "ValueError", "function error mismatch" ); + let logs = environment.logs.as_ref().context("function log capture")?; + timeout_at(deadline(), async { + loop { + let snapshot = logs.snapshot(deadline()).await?; + ensure!(snapshot.failure.is_none(), "log capture failed"); + if snapshot + .tail + .windows(19) + .any(|part| part == b"WIRE_STDOUT_CAPTURE") + && snapshot + .tail + .windows(19) + .any(|part| part == b"WIRE_STDERR_CAPTURE") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("function log receipt deadline")??; Ok(()) } .await; diff --git a/specs/index.md b/specs/index.md index 45d84e02..7af1dcdc 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Persistent descriptor-backed directory ownership and pin/cleanup retention are implemented and independently reviewed (§4.1/D28); prior-run reconciliation remains pending. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Persistent descriptor-backed directory ownership and pin/cleanup retention are implemented and independently reviewed (§4.1/D28); prior-run reconciliation remains pending. Independent bounded function log capture and output-only live attach are also implemented/reviewed (§3.7/D29), with actual stdout/stderr receipt in both real wire suites. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 50030921..70e0bf75 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -444,6 +444,16 @@ Function logs use the same strict outer framing with their own retained owner an Mandatory task-bound diagnostic filtering covers Hyper/hyper_util and Bollard DEBUG/TRACE during construction and polling, including every spawned IO task. Keep the model-only production path and its filter explicit; the P0/full-client compatibility probe may still use Bollard and must not reintroduce verbose payload logs. This refines W01 rather than waiving it. Fake local Unix daemons must test oversized/chunked bodies, duplicate/recursive metadata, headers/trailers, malformed multiplexed lengths/reserved bytes/channels, truncated EOF, slow partial records, dropped waiters and TRACE-secret capture. Real daemon tests must prove 6 MiB-class control payload fragmentation, normal Close/Ack and no owned allocation left behind. Startup/limit enforcement and actual SDK executor acceptance remain task1.3–1.4 gates. +### 3.7 Retained function log capture + +The function's stdout/stderr use a separate output-only Docker attachment, established before container start with stdin=false, stdout/stderr/stream=true and logs=false. `DockerPort::attach_logs` uses the same strict upgrade and joined driver as control attach, releases the short management slot after handoff and never replays previous output. Its live owner is bounded by the environment reservation; it is not a control-stream producer. This implements continuous streaming without a second HTTP-body framing parser or the daemon's persistent log API. + +`FunctionLogs::start(attachment, partial_io_budget, ring_bytes)` validates record IO (positive≤60s) and ring capacity (4,096..1,048,576 bytes per channel, default operator budget262,144), reserves storage fallibly, then starts one retained actor. It owns the DockerOutputReader, separate stdout/stderr byte rings and a combined4,096-byte tail preserving observed chunk order. Excess output overwrites oldest bytes with saturating per-channel discard counters; never decode UTF-8, split lines, trace raw bytes or block waiting for a log consumer. A snapshot exposes only the combined tail, counters and closed/failure classification; Debug reports sizes/classes. Tail bytes are intentional Lambda API output and may contain arbitrary binary/user secrets. + +The actor's snapshot mailbox holds at most8 requests. Every envelope carries an absolute deadline; reject expired requests before copying a tail. `snapshot(deadline)` waits cancel-safely; abandoned read-only requests grant no mutation authority. Keep one partial-record read future pinned across snapshot traffic: cancelling it merely to answer a snapshot would poison/lose framing. Service at most one already-queued snapshot before polling the read, then select stop, the retained read and further snapshots, so neither log flooding nor snapshot flooding starves the other. Explicitly yield after64 chunks. First-byte silence has a fixed one-hour safety ceiling; partial-record IO keeps the reader's short deadline, independent of snapshots. Environment idle/attempt/cleanup deadlines remain authoritative and normally end a silent attachment earlier. + +Stop has a separate priority watch signal. `shutdown(&mut self)` signals stop and awaits the stored join by mutable borrow, retaining ownership if the waiter is cancelled; cache either the final snapshot or failure so repeated shutdown cannot report false success. Drop signals the same actor, which drops its owned socket without requiring an entered runtime. EOF marks capture closed; parser/IO/deadline failure is retained and observable through snapshots and fails joined shutdown. Drop the socket immediately on EOF/parser failure, but keep the actor available for final snapshots until owner stop, rather than losing already-captured bytes. Closing this log stream never proves container termination; environment cleanup must independently stop/wait/remove the function and relay. Unit tests cover binary order/tails, independent channel bounds/discards, record fragmentation under snapshot traffic, flood/stop responsiveness, quiet cancellation/EOF, malformed input, cancelled snapshot/shutdown waiters and cached supervision errors. Real wire qualification attaches to the function before start, captures explicit handler stdout/stderr, verifies its computed response and joins logs separately from container cleanup. This is a consumer primitive; it does not enable the production executor. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index f11eef55..7df728b9 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -100,6 +100,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi ## 4. Security acceptance +`make test-lambda-logs` verifies the independent function output owner: binary stdout/stderr ordering, separate bounded overwrite rings and saturating discard counts, combined4KiB tail, EOF/failure retention, socket closure on framing failure, snapshot admission bounds, partial-frame reads retained through completed/abandoned snapshot requests, fixed partial/silence deadlines, output-flood responsiveness, cancelled shutdown join, off-runtime Drop and cached actor panic. A scripted daemon pins output-only/no-replay attach query and driver handoff. The real wire suite additionally attaches before function start, enables explicit handler stdout/stderr markers, requires both markers without affecting computed results, and joins logs independently after function removal. These tests qualify capture, not complete per-invocation log attribution or the still-pending executor. + `make test-lambda-ownership` verifies the real directory capability independently of Docker: stable owner/fresh run, descriptor/allocated-pin lifetime, actual subprocess contention and reacquisition, bounded malformed/pending state, symlink spellings (including trailing slash/current-directory), hardlink/FIFO/permission rejection, trusted/sticky ancestor rules and repeated parent sync after interrupted creation. Worker tests cover admission expiry, running deadline/cancelled waiter retention, completed-but-unconsumed result capacity and panic release. A scripted-daemon integration additionally holds a real directory lease through abandoned image-owner quarantine and delayed deletion, requiring exclusion until cleanup settles. Production wire fixtures acquire actual directory leases and retain their namespace files on uncertain cleanup. These checks do not establish prior-run reconciliation, startup readiness or full M0 acceptance. | Boundary | Required proof | diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index ef7af329..79c74da6 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -144,6 +144,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.2 | Runtime contracts implemented/reviewed; Docker consumer/probe gates remain | Protocol4da7df3, shared actor2c50c2d, Native migration95282c4 and relay6779f93 are committed. The ordered host ControlClient now shares retained stream workers with the relay, validates causal attempt/cancel/close messages and poisons uncertain sends. Workspace:1,867 tests; runtime:97 tests/47 doctests and strict boundary Clippy; actual Linux entry-point Clippy passes. Five explicit Docker qualification tests pass on each arm64 VZ and amd64/QEMU daemon, including production ControlClient→real relay→official RIC computed warm calls, FunctionError, busy cancellation and independent stream/container cleanup. W27–W29 are fixed/rechecked. Startup deployment/resource probes still require the task1.3–1.4 Docker consumer; no production executor/provider acceptance yet. | | 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. B01 dependency closure is implemented/reviewed (§7.2/D27, W57–W59): nine ZIP regressions, workspace2,083 tests,833 boundary tests/doctests, green standard/strict gates, clean audit and all deny checks. Regional SDK/HTTP and Native provider regressions pass. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.3 directory namespace | Implemented/reviewed; runtime consumers remain | DirectoryLease binds permanent nonreplaced file locking, durable checksummed owner/fresh run and trusted descriptor-relative filesystem operations to all image scopes/pins/jobs/recovery. W60–W62 fixes independently rechecked; 38 ownership cases plus abandoned image-owner cleanup pass. Workspace2,122 tests, core443 tests/103 doctests, boundary874 unit/doc passes, all standard/strict/audit/deny gates pass. Both real two-case wire suites pass on arm64 VZ and amd64/QEMU; post-test owned containers/internal aliases are empty. This does not prove previous-run reconciliation, Linux-host lock qualification or Docker executor readiness. | +| 1.3 function logs | Implemented/reviewed; environment integration remains | Output-only attach and FunctionLogs retain partial framing across snapshot traffic, bounded independent rings/combined4KiB binary tail, fair service, early socket closure and cached joined shutdown (§3.7/D29).18 focused tests pass; W63/W64 fixes independently rechecked. Core461 tests/107 doctests, workspace2,140 tests, boundary896 unit/doc passes and all standard/strict gates pass. Both real two-case wire suites receive deliberate handler stdout/stderr markers independently from computed results and join logs during cleanup. No new dependency/version. Per-invocation log attribution, environment integration and complete executor acceptance remain pending. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining stream/network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining network/probe consumers plus function-log/environment integration, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 61d246c9..a8de054b 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -291,6 +291,21 @@ Both explicit two-case Docker wire suites pass using real DirectoryLease-backed Persistent namespace capability is now implemented/reviewed. Startup historical-effect reconciliation, allowed pulls, remaining log/network/probe consumers, environment/pool executor and production configuration/acceptance remain P1 requirements. P2–P5 remain required. +## P1 task1.3 function output capture review + +Independent read-only review covers §3.7/D29: output-only live attach, bounded binary rings/tail, fair snapshot/read service, retained partial framing, early socket closure, priority stop and cached join/failure ownership. Its final recheck finds no remaining in-slice issue. Capture is a runtime consumer primitive, not production executor readiness or complete per-invocation log attribution. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W63 | P3; `docker/logs_tests.rs`, cancelled snapshot proof | Dropping the Pin reference produced by tokio::pin does not drop its backing future. Use a lexical scope around future/pin/poll to actually abandon its receiver | A fragmented record completes after both16 completed snapshots and a genuinely abandoned snapshot; shutdown succeeds | +| W64 | P2; self-review, `docker/logs.rs`, terminal reader ownership | Keeping the idle actor alive after EOF/framing failure also retained its socket. Drop the reader immediately but preserve capture state and failure for final snapshots/join | All four malformed stream cases observe peer EOF before owner shutdown, then retain failure across repeated shutdown | + +Eighteen focused cases pass, including independent channel overwrite/saturating counters, binary combined tail, bounded mailbox8 admission, output flooding, quiet off-runtime Drop, partial/silence deadline independence, cancelled shutdown join, actor panic and a scripted-daemon output-only/no-replay upgrade. The latter shuts down DockerWire after handoff and still consumes the separately owned capture, proving no management slot/driver remains needed for the transferred stream. Debug exposes sizes/error classes, never user bytes. No new dependency/version or lint exception is introduced. + +Final-source workspace build/test/fmt/Clippy/rustdoc pass with2,140 tests, zero failures and401 resource-gated cases. Core has461 tests/107 doctests; the six-library strict boundary target passes725 unit tests/171 doctests (896 total), with seven separately gated Docker cases. The unchanged dependency graph retains the current turn's clean474-dependency audit and passing deny advisories/bans/licenses/sources from the directory slice. + +Both explicit two-case Docker wire suites pass on Engine29.5.2 arm64 VZ and amd64/QEMU. The official RIC handler prints deliberate stdout/stderr markers only for the requested fixture event; the test requires both independently from its computed near6MiB response and warm/error outcomes, then joins capture after function removal. After both suites, ownership queries show no Rustack containers or internal aliases. These are60s stream/120s attempt stress qualifications, not default executor latency or native x86/Desktop evidence. Historical reconciliation, pulls, network/probe consumers, environment/pool actors, production wiring and P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 8f643fcf..5febdf03 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -254,3 +254,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - ← [PRD](./rustack-workflows-00-prd.md) - ↔ [roadmap](./rustack-workflows-80-roadmap.md), [implementation plan](./rustack-workflows-90-impl-plan.md), [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) + +## D29 — Retain function output framing independently of snapshot traffic + +- **Context:** real Docker handlers can produce binary/flooding stdout/stderr while the control channel carries invocation results. Log snapshots must not cancel a partially consumed outer Docker record or retain unbounded output. +- **Alternatives considered:** ordinary buffered Docker logs can accumulate response bodies; sharing control transport introduces output backpressure into invocation delivery; cancelling next() for every snapshot poisons its partial parser; blocking consumers on a full ring can block the handler. +- **Decision:** attach live output only before function start using the existing joined upgrade transport; one retained actor owns framing and bounded channel/tail rings, preserving its pinned read future across read-only snapshot requests and prioritizing stop. Close failed/EOF sockets immediately while retaining final snapshots and join failures. +- **Why:** snapshot and log flooding have bounded service quanta, user bytes never enter infrastructure diagnostics, and environment cleanup can join this socket independently of function/relay termination. The capture alone does not prove container exit or exact per-invocation log attribution. +- **Pinned by:** [runtime §3.7](./rustack-workflows-20-lambda-runtime-design.md#37-retained-function-log-capture), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-26. No new dependency or version is introduced. diff --git a/tools/workflows-spikes/handler.py b/tools/workflows-spikes/handler.py index 807a0a3b..af8d191e 100644 --- a/tools/workflows-spikes/handler.py +++ b/tools/workflows-spikes/handler.py @@ -1,6 +1,7 @@ """Real official-RIC handler used by the isolated P0 transport experiment.""" import os +import sys from pathlib import Path import time @@ -10,6 +11,9 @@ def handler(event, context): global counter counter += 1 + if event.get("logs"): + print("WIRE_STDOUT_CAPTURE", flush=True) + print("WIRE_STDERR_CAPTURE", file=sys.stderr, flush=True) if event.get("fail"): raise ValueError("intentional P0 failure") if event.get("sleep"): From 9c8f23d106c67cb431d0526adff2bc99b6beb6b1 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 00:57:19 -0700 Subject: [PATCH 022/106] phase 1: bind container allocation to immutable runtime policy Implement runtime section 4.2 and D30 with retained revision/image/directory ownership, fixed allocation identities/platforms, validated network/user/relay budgets and resource/swap/tmpfs/health/log/restart limits. Compose unique environment keys with explicit image/captured/runtime precedence and deletion markers. Scope the wire serializer before await so supervised create futures are Send. Independent W65-W67 fixes are rechecked. 54 new cases pass, workspace 2194 tests and strict boundary 959 unit/doc cases pass, and standard build/fmt/Clippy/rustdoc plus audit/deny are green. Both real two-case wire suites consume allocation plans, verify cgroups/tmpfs/privileges/env and actual empty ImageConfig execution; all five shared runtime Docker regressions pass per daemon. Native provider regressions pass without claiming the gated Squib case as VM evidence. Environment/pool lifecycle, reconciliation, pulls, network/probes, production executor and P2-P5 remain required; M0 remains open. --- Makefile | 5 + crates/rustack-lambda-core/src/docker/mod.rs | 16 +- crates/rustack-lambda-core/src/docker/plan.rs | 607 ++++++++++++++++++ .../src/docker/plan_tests.rs | 569 ++++++++++++++++ .../src/function_config.rs | 14 +- .../rustack-lambda-core/tests/docker_wire.rs | 255 ++++---- specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 14 + .../rustack-workflows-60-operations-design.md | 2 +- .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 18 + specs/rustack-workflows-99-decisions.md | 9 + tools/workflows-spikes/Handler.Dockerfile | 4 + tools/workflows-spikes/handler.py | 17 + 15 files changed, 1392 insertions(+), 149 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/plan.rs create mode 100644 crates/rustack-lambda-core/src/docker/plan_tests.rs diff --git a/Makefile b/Makefile index 5287a4b6..b80ad339 100644 --- a/Makefile +++ b/Makefile @@ -312,6 +312,11 @@ test-lambda-logs: .PHONY: test-lambda-logs +test-lambda-environment-plans: + @cargo test -p rustack-lambda-core --lib docker::plan::tests + +.PHONY: test-lambda-environment-plans + # Real-execution Lambda invoke tests (native backend). # Builds a Rust bootstrap fixture and invokes it through the rustack provider. test-lambda-invoke-native: diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 2652a6ac..614516e2 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -10,6 +10,7 @@ mod lease; mod logs; mod ownership; mod pin; +mod plan; mod types; use std::{ @@ -38,6 +39,7 @@ use models::{ pub use ownership::{DirectoryLease, DirectoryLeaseError}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; pub use pin::{ImagePin, ImagePinScope}; +pub use plan::{ContainerPolicy, EnvironmentPlan, NetworkSelection, RelayBudgets}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; @@ -407,12 +409,14 @@ impl DockerPort { architecture: Option, deadline: Instant, ) -> Result { - let mut query = form_urlencoded::Serializer::new(String::new()); - query.append_pair("name", name.as_str()); - if let Some(architecture) = architecture { - query.append_pair("platform", architecture.docker_platform()); - } - let query = query.finish(); + let query = { + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("name", name.as_str()); + if let Some(architecture) = architecture { + query.append_pair("platform", architecture.docker_platform()); + } + query.finish() + }; decode( self.call( Operation::Create, diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs new file mode 100644 index 00000000..6bb7b305 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -0,0 +1,607 @@ +//! Immutable, bounded container creation policy; lifetime ownership remains above IO. + +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + str::FromStr, + sync::Arc, + time::Duration, +}; + +use rustack_lambda_model::types::ImageConfig; +use rustack_lambda_runtime::protocol::EnvironmentId; +use tokio::time::Instant; +use uuid::Uuid; + +use super::{ + DirectoryLease, DockerError, DockerKey, DockerPort, + models::{ + ContainerCreateBody, ContainerCreateResponse, HealthConfig, HostConfig, + HostConfigLogConfig, RestartPolicy, RestartPolicyNameEnum, + }, +}; +use crate::{ + artifact::{DeploymentArtifact, PinnedImage}, + executor::{ExecutionRevision, ExecutorBackend}, + function_config::validate_environment_entries, +}; + +const MIB: u64 = 1024 * 1024; +const RELAY_MEMORY: u64 = 64 * MIB; + +/// Operator network selection; a bridge still requires daemon startup validation. +#[derive(Clone)] +#[non_exhaustive] +pub struct NetworkSelection(String); +impl fmt::Debug for NetworkSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("NetworkSelection()") + } +} +impl FromStr for NetworkSelection { + type Err = DockerError; + fn from_str(value: &str) -> Result { + if value == "none" { + return Ok(Self(value.into())); + } + if matches!(value, "host" | "bridge" | "default") { + return Err(DockerError::Configuration); + } + value.parse::()?; + Ok(Self(value.into())) + } +} + +/// Integral relay session budgets independent of handler and deployment budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct RelayBudgets { + io: Duration, + idle: Duration, + retirement: Duration, +} +impl Default for RelayBudgets { + fn default() -> Self { + Self { + io: Duration::from_secs(5), + idle: Duration::from_secs(600), + retirement: Duration::from_secs(10), + } + } +} +impl RelayBudgets { + /// Validate whole-second IO/retirement1..60s and idle1..86400s budgets. + /// + /// # Errors + /// Rejects zero, fractional or excessive durations. + /// + /// ``` + /// let budgets = rustack_lambda_core::docker::RelayBudgets::new( + /// std::time::Duration::from_secs(5), std::time::Duration::from_secs(600), + /// std::time::Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new(io: Duration, idle: Duration, retirement: Duration) -> Result { + for (duration, maximum) in [(io, 60), (idle, 86400), (retirement, 60)] { + if duration.subsec_nanos() != 0 || !(1..=maximum).contains(&duration.as_secs()) { + return Err(DockerError::Configuration); + } + } + Ok(Self { + io, + idle, + retirement, + }) + } +} + +/// Validated operator limits, independent of user function configuration. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ContainerPolicy { + network: NetworkSelection, + uid: u32, + gid: u32, + nano_cpus: i64, + relay: RelayBudgets, +} +impl ContainerPolicy { + /// Validate nonzero signed32-bit UID/GID and0.1..16 CPU allocation. + /// + /// # Errors + /// Rejects invalid IDs or CPU values, preserving bounded operator policy. + /// + /// ``` + /// let policy = rustack_lambda_core::docker::ContainerPolicy::new( + /// "none".parse()?, 1000, 1000, 1_000_000_000, + /// rustack_lambda_core::docker::RelayBudgets::default())?; + /// # Ok::<(), rustack_lambda_core::docker::DockerError>(()) + /// ``` + pub fn new( + network: NetworkSelection, + uid: u32, + gid: u32, + nano_cpus: u64, + relay: RelayBudgets, + ) -> Result { + if !(1..=2_147_483_647).contains(&uid) + || !(1..=2_147_483_647).contains(&gid) + || !(100_000_000..=16_000_000_000).contains(&nano_cpus) + { + return Err(DockerError::Configuration); + } + Ok(Self { + network, + uid, + gid, + nano_cpus: i64::try_from(nano_cpus).map_err(|_| DockerError::Configuration)?, + relay, + }) + } +} + +struct Identity { + environment: EnvironmentId, + relay: DockerKey, + function: DockerKey, +} +struct Bodies { + relay: ContainerCreateBody, + function: ContainerCreateBody, + memory: u64, +} + +/// Fixed create requests retaining revision, image availability and namespace leases. +#[non_exhaustive] +pub struct EnvironmentPlan { + _directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + identity: Identity, + bodies: Bodies, +} +impl fmt::Debug for EnvironmentPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPlan") + .field("environment", &self.identity.environment) + .field("resource", &self.revision.resource_id) + .field("memory_bytes", &self.bodies.memory) + .finish_non_exhaustive() + } +} +impl EnvironmentPlan { + /// Allocate identities and validate both immutable create bodies before IO. + /// + /// # Errors + /// Rejects unavailable/mismatched artifacts, invalid captured settings or + /// effective image configuration. This does not reserve environment capacity. + /// + /// ```no_run + /// # fn plan(directory: rustack_lambda_core::docker::DirectoryLease, relay: rustack_lambda_core::artifact::PinnedImage, revision: std::sync::Arc, policy: rustack_lambda_core::docker::ContainerPolicy) { + /// let plan = rustack_lambda_core::docker::EnvironmentPlan::new(directory, relay, revision, policy); + /// # } + /// ``` + pub fn new( + directory: DirectoryLease, + relay: PinnedImage, + revision: Arc, + policy: ContainerPolicy, + ) -> Result { + validate_revision(&revision)?; + let function = image(&revision)?; + if !relay.is_live() || relay.resolved().metadata().architecture != revision.architecture { + return Err(DockerError::Configuration); + } + let environment = EnvironmentId::default(); + let identity = Identity { + environment, + relay: format!("rustack-relay-{environment}").parse()?, + function: format!("rustack-function-{environment}").parse()?, + }; + let labels = HashMap::from([ + ( + "rustack.lambda.ownerId".into(), + directory.owner_id().to_string(), + ), + ( + "rustack.lambda.runId".into(), + directory.run_id().to_string(), + ), + ( + "rustack.lambda.environmentId".into(), + environment.to_string(), + ), + ( + "rustack.lambda.functionResourceId".into(), + revision.resource_id.to_string(), + ), + ]); + let user = format!("{}:{}", policy.uid, policy.gid); + let mut relay_body = body(&relay, None, &user, &labels, "relay")?; + relay_body.env = Some(vec![ + format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={environment}"), + format!( + "RUSTACK_LAMBDA_RELAY_IO_SECONDS={}", + policy.relay.io.as_secs() + ), + format!( + "RUSTACK_LAMBDA_RELAY_IDLE_SECONDS={}", + policy.relay.idle.as_secs() + ), + format!( + "RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS={}", + policy.relay.retirement.as_secs() + ), + ]); + relay_body.attach_stdin = Some(true); + relay_body.open_stdin = Some(true); + relay_body.stdin_once = Some(true); + relay_body.host_config = Some(limits( + policy.network.0, + RELAY_MEMORY, + 100_000_000, + 64, + MIB, + )?); + let selected_user = image_user( + &function.resolved().metadata().execution.metadata().user, + &user, + )?; + let mut function_body = body( + function, + revision.image_config.as_ref(), + &selected_user, + &labels, + "function", + )?; + function_body.env = Some(function_environment(&revision, environment)?); + let memory = u64::from(revision.memory_mb) + .checked_mul(MIB) + .ok_or(DockerError::Configuration)?; + function_body.host_config = Some(limits( + format!("container:{}", identity.relay.as_str()), + memory, + policy.nano_cpus, + 256, + u64::from(revision.ephemeral_storage_mb) + .checked_mul(MIB) + .ok_or(DockerError::Configuration)?, + )?); + let memory = memory + .checked_add(RELAY_MEMORY) + .ok_or(DockerError::Configuration)?; + Ok(Self { + _directory: directory, + relay, + revision, + identity, + bodies: Bodies { + relay: relay_body, + function: function_body, + memory, + }, + }) + } + + /// Fresh environment UUID used by both labels and control protocol. + /// + /// ```no_run + /// # fn id(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let id = plan.environment_id(); + /// # } + /// ``` + #[must_use] + pub fn environment_id(&self) -> EnvironmentId { + self.identity.environment + } + + /// Preallocated relay identity retained through uncertain allocation cleanup. + /// + /// ```no_run + /// # fn key(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let name = plan.relay_key(); + /// # } + /// ``` + #[must_use] + pub fn relay_key(&self) -> &DockerKey { + &self.identity.relay + } + + /// Preallocated function identity; never populated from invocation input. + /// + /// ```no_run + /// # fn key(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let name = plan.function_key(); + /// # } + /// ``` + #[must_use] + pub fn function_key(&self) -> &DockerKey { + &self.identity.function + } + + /// Checked function plus relay memory required before allocation admission. + /// + /// ```no_run + /// # fn memory(plan: &rustack_lambda_core::docker::EnvironmentPlan) { + /// let bytes = plan.memory_bytes(); + /// # } + /// ``` + #[must_use] + pub fn memory_bytes(&self) -> u64 { + self.bodies.memory + } + + /// Create the relay after its owner has recorded uncertain allocation. + /// + /// # Errors + /// Returns revoked availability, deadline, wire/status or schema failure. + /// Every uncertain outcome still requires owner reconciliation. + /// + /// ```no_run + /// # async fn create(plan: &rustack_lambda_core::docker::EnvironmentPlan, port: &rustack_lambda_core::docker::DockerPort) { + /// let created = plan.create_relay(port, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_relay( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + self.live()?; + port.create_container_on_platform( + &self.identity.relay, + &self.bodies.relay, + self.revision.architecture, + deadline, + ) + .await + } + + /// Create the function only after its own relay is verified ready. + /// + /// # Errors + /// Returns revoked availability, deadline, wire/status or schema failure. + /// Caller owns startup ordering and every uncertain allocation outcome. + /// + /// ```no_run + /// # async fn create(plan: &rustack_lambda_core::docker::EnvironmentPlan, port: &rustack_lambda_core::docker::DockerPort) { + /// let created = plan.create_function(port, tokio::time::Instant::now() + std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn create_function( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + self.live()?; + port.create_container_on_platform( + &self.identity.function, + &self.bodies.function, + self.revision.architecture, + deadline, + ) + .await + } + fn live(&self) -> Result<(), DockerError> { + match &self.revision.artifact { + DeploymentArtifact::Image(function) if self.relay.is_live() && function.is_live() => { + Ok(()) + } + DeploymentArtifact::Image(_) => Err(DockerError::Stopped), + DeploymentArtifact::Zip(_) => Err(DockerError::Configuration), + } + } +} + +fn image(revision: &ExecutionRevision) -> Result<&PinnedImage, DockerError> { + match &revision.artifact { + DeploymentArtifact::Image(image) + if image.is_live() + && image.resolved().metadata().architecture == revision.architecture => + { + Ok(image) + } + _ => Err(DockerError::Configuration), + } +} +fn validate_revision(revision: &ExecutionRevision) -> Result<(), DockerError> { + if revision.revision_id.len() != 36 { + return Err(DockerError::Configuration); + } + let id = Uuid::parse_str(&revision.revision_id).map_err(|_| DockerError::Configuration)?; + let version = &revision.qualifier; + let numeric_version = version.len() <= 20 + && version + .parse::() + .is_ok_and(|number| number > 0 && number.to_string() == *version); + if revision.backend != ExecutorBackend::Docker + || revision.runtime.is_some() + || revision.handler.is_some() + || id.is_nil() + || id.to_string() != revision.revision_id + || !(1..=64).contains(&revision.function_name.len()) + || !revision + .function_name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte)) + || (version != "$LATEST" && !numeric_version) + || !(1..=900).contains(&revision.timeout.as_secs()) + || revision.timeout.subsec_nanos() != 0 + || !(128..=10240).contains(&revision.memory_mb) + || !(512..=10240).contains(&revision.ephemeral_storage_mb) + { + return Err(DockerError::Configuration); + } + image(revision)?; + Ok(()) +} + +fn body( + image: &PinnedImage, + overrides: Option<&ImageConfig>, + user: &str, + labels: &HashMap, + role: &str, +) -> Result { + let plan = image.resolved().metadata().execution.plan(overrides)?; + let mut labels = labels.clone(); + labels.insert("rustack.lambda.role".into(), role.into()); + Ok(ContainerCreateBody { + image: Some(image.resolved().metadata().image_id.as_str().into()), + entrypoint: Some(plan.entrypoint().to_vec()), + cmd: Some(plan.command().to_vec()), + working_dir: Some(plan.working_directory().into()), + user: Some(user.into()), + labels: Some(labels), + tty: Some(false), + attach_stdin: Some(false), + attach_stdout: Some(true), + attach_stderr: Some(true), + open_stdin: Some(false), + stdin_once: Some(false), + healthcheck: Some(HealthConfig { + test: Some(vec!["NONE".into()]), + ..Default::default() + }), + ..Default::default() + }) +} + +fn limits( + network: String, + memory: u64, + cpu: i64, + pids: i64, + temporary: u64, +) -> Result { + let memory = i64::try_from(memory).map_err(|_| DockerError::Configuration)?; + Ok(HostConfig { + network_mode: Some(network), + memory: Some(memory), + memory_swap: Some(memory), + oom_kill_disable: Some(false), + nano_cpus: Some(cpu), + pids_limit: Some(pids), + readonly_rootfs: Some(true), + cap_drop: Some(vec!["ALL".into()]), + security_opt: Some(vec!["no-new-privileges:true".into()]), + privileged: Some(false), + publish_all_ports: Some(false), + auto_remove: Some(false), + restart_policy: Some(RestartPolicy { + name: Some(RestartPolicyNameEnum::NO), + maximum_retry_count: Some(0), + }), + log_config: Some(HostConfigLogConfig { + typ: Some("none".into()), + config: None, + }), + tmpfs: Some(HashMap::from([( + "/tmp".into(), + format!("rw,nosuid,nodev,size={temporary},mode=1777"), + )])), + ..Default::default() + }) +} + +fn image_user(value: &str, fallback: &str) -> Result { + if value.is_empty() { + return Ok(fallback.into()); + } + let mut parts = value.split(':'); + let user = parts.next().ok_or(DockerError::Configuration)?; + let group = parts.next(); + if parts.next().is_some() || !account(user) || group.is_some_and(|group| !account(group)) { + return Err(DockerError::Configuration); + } + if user == "root" || user.parse::().is_ok_and(|uid| uid == 0) { + Ok(fallback.into()) + } else { + Ok(value.into()) + } +} +fn account(value: &str) -> bool { + if value.is_empty() || value.len() > 64 { + return false; + } + if value.bytes().all(|byte| byte.is_ascii_digit()) { + return value.parse::().is_ok_and(|id| id <= 2_147_483_647); + } + value + .as_bytes() + .first() + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) +} + +fn function_environment( + revision: &ExecutionRevision, + environment: EnvironmentId, +) -> Result, DockerError> { + if revision.environment.len() > 130 { + return Err(DockerError::Configuration); + } + validate_environment_entries( + revision + .environment + .iter() + .filter(|(key, _)| !matches!(key.as_str(), "AWS_REGION" | "AWS_DEFAULT_REGION")), + ) + .map_err(|_| DockerError::Configuration)?; + let region = revision + .environment + .get("AWS_REGION") + .ok_or(DockerError::Configuration)?; + if !(1..=64).contains(®ion.len()) + || !region + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + || revision.environment.get("AWS_DEFAULT_REGION") != Some(region) + { + return Err(DockerError::Configuration); + } + let image = image(revision)?; + let mut output: BTreeMap> = BTreeMap::new(); + for entry in &image.resolved().metadata().execution.metadata().environment { + let (key, value) = entry + .split_once('=') + .map_or((entry.as_str(), None), |(key, value)| { + (key, Some(value.to_owned())) + }); + output.insert(key.to_owned(), value); + } + for (key, value) in &revision.environment { + output.insert(key.clone(), Some(value.clone())); + } + let authoritative = [ + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001".into(), + format!("AWS_LAMBDA_FUNCTION_NAME={}", revision.function_name), + format!("AWS_LAMBDA_FUNCTION_VERSION={}", revision.qualifier), + format!("AWS_LAMBDA_FUNCTION_MEMORY_SIZE={}", revision.memory_mb), + format!("AWS_LAMBDA_FUNCTION_TIMEOUT={}", revision.timeout.as_secs()), + "AWS_LAMBDA_INITIALIZATION_TYPE=on-demand".into(), + format!( + "AWS_LAMBDA_LOG_GROUP_NAME=/aws/lambda/{}", + revision.function_name + ), + format!("AWS_LAMBDA_LOG_STREAM_NAME=rustack/{environment}"), + "LAMBDA_TASK_ROOT=/var/task".into(), + ]; + for entry in authoritative { + let (key, value) = entry.split_once('=').ok_or(DockerError::Configuration)?; + output.insert(key.to_owned(), Some(value.to_owned())); + } + Ok(output + .into_iter() + .map(|(key, value)| match value { + Some(value) => format!("{key}={value}"), + None => key, + }) + .collect()) +} + +#[cfg(test)] +#[path = "plan_tests.rs"] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/plan_tests.rs b/crates/rustack-lambda-core/src/docker/plan_tests.rs new file mode 100644 index 00000000..f05062c2 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/plan_tests.rs @@ -0,0 +1,569 @@ +use std::{collections::HashSet, sync::atomic::AtomicBool}; + +use tokio::{ + sync::watch, + time::{Instant, timeout}, +}; + +use super::*; +use crate::{ + artifact::{ + Architecture, ImageExecutionMetadata, ImageMetadata, ResolvedImage, ValidatedArtifact, + }, + identity::FunctionResourceId, +}; + +fn directory() -> DirectoryLease { + DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()) +} +fn policy() -> ContainerPolicy { + ContainerPolicy::new( + "none".parse().unwrap(), + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap() +} +fn image_with(user: &str, environment: Vec) -> PinnedImage { + let mut metadata = ResolvedImage::fixture(Architecture::Arm64) + .metadata() + .clone(); + metadata.execution = ImageExecutionMetadata::builder() + .entrypoint(vec!["/bootstrap".into()]) + .command(vec!["handler".into()]) + .working_directory("/var/task".into()) + .environment(environment) + .user(user.into()) + .build() + .try_into() + .unwrap(); + metadata.inspection_hash = ImageMetadata::execution_fingerprint( + &metadata.image_id, + metadata.selected_manifest.as_ref(), + metadata.architecture, + metadata.size_bytes, + &metadata.execution, + ) + .unwrap(); + PinnedImage::fixture(metadata.try_into().unwrap()) +} +fn revision() -> ExecutionRevision { + ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("policy-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(Architecture::Arm64) + .artifact(DeploymentArtifact::Image(image_with( + "root", + vec!["BASE=value".into()], + ))) + .image_config(None) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ("TEST_VALUE".into(), "secret-value".into()), + ])) + .timeout(Duration::from_secs(3)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build() +} +fn plan(revision: ExecutionRevision) -> Result { + EnvironmentPlan::new( + directory(), + image_with("0", vec![]), + Arc::new(revision), + policy(), + ) +} +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(5) +} + +#[test] +fn test_should_capture_fixed_images_identities_labels_and_complete_resource_policy() { + let directory = directory(); + let owner = directory.owner_id(); + let run = directory.run_id(); + let revision = revision(); + let resource = revision.resource_id; + let image = image(&revision) + .unwrap() + .resolved() + .metadata() + .image_id + .clone(); + let plan = EnvironmentPlan::new( + directory, + image_with("0", vec![]), + Arc::new(revision), + policy(), + ) + .unwrap(); + assert_ne!(plan.relay_key().as_str(), plan.function_key().as_str()); + assert_eq!(plan.memory_bytes(), 256 * MIB); + for (body, role, memory, cpu, pids, tmp) in [ + (&plan.bodies.relay, "relay", 64 * MIB, 100_000_000, 64, MIB), + ( + &plan.bodies.function, + "function", + 192 * MIB, + 1_000_000_000, + 256, + 512 * MIB, + ), + ] { + assert_eq!(body.image.as_deref(), Some(image.as_str())); + assert_eq!(body.tty, Some(false)); + assert_eq!(body.user.as_deref(), Some("1000:1000")); + assert_eq!( + body.healthcheck.as_ref().unwrap().test.as_ref().unwrap(), + &["NONE"] + ); + let labels = body.labels.as_ref().unwrap(); + assert_eq!(labels["rustack.lambda.ownerId"], owner.to_string()); + assert_eq!(labels["rustack.lambda.runId"], run.to_string()); + assert_eq!( + labels["rustack.lambda.environmentId"], + plan.environment_id().to_string() + ); + assert_eq!( + labels["rustack.lambda.functionResourceId"], + resource.to_string() + ); + assert_eq!(labels["rustack.lambda.role"], role); + let limits = body.host_config.as_ref().unwrap(); + assert_eq!(limits.memory, Some(i64::try_from(memory).unwrap())); + assert_eq!(limits.memory_swap, limits.memory); + assert_eq!(limits.oom_kill_disable, Some(false)); + assert_eq!(limits.nano_cpus, Some(cpu)); + assert_eq!(limits.pids_limit, Some(pids)); + assert_eq!(limits.readonly_rootfs, Some(true)); + assert_eq!(limits.cap_drop.as_ref().unwrap(), &["ALL"]); + assert_eq!( + limits.security_opt.as_ref().unwrap(), + &["no-new-privileges:true"] + ); + assert_eq!(limits.privileged, Some(false)); + assert_eq!(limits.auto_remove, Some(false)); + assert_eq!(limits.publish_all_ports, Some(false)); + assert_eq!( + limits.restart_policy.as_ref().unwrap().name, + Some(RestartPolicyNameEnum::NO) + ); + assert_eq!( + limits.log_config.as_ref().unwrap().typ.as_deref(), + Some("none") + ); + assert_eq!( + limits.tmpfs.as_ref().unwrap()["/tmp"], + format!("rw,nosuid,nodev,size={tmp},mode=1777") + ); + assert!( + limits.binds.is_none() && limits.devices.is_none() && limits.port_bindings.is_none() + ); + } + assert_eq!(plan.bodies.relay.attach_stdin, Some(true)); + assert_eq!(plan.bodies.relay.open_stdin, Some(true)); + assert_eq!(plan.bodies.relay.stdin_once, Some(true)); + assert_eq!(plan.bodies.function.attach_stdin, Some(false)); + assert_eq!( + plan.bodies + .function + .host_config + .as_ref() + .unwrap() + .network_mode + .as_deref(), + Some(format!("container:{}", plan.relay_key().as_str()).as_str()) + ); + assert!(!format!("{plan:?}").contains("secret-value")); +} + +#[test] +fn test_should_preserve_explicit_image_overrides_and_authoritative_environment_order() { + let mut captured = revision(); + captured.qualifier = "27".into(); + captured.artifact = DeploymentArtifact::Image(image_with( + "1234:4321", + vec![ + "TEST_VALUE=old".into(), + "AWS_LAMBDA_RUNTIME_API=outside:9001".into(), + "DELETED".into(), + ], + )); + captured.image_config = Some(ImageConfig { + entry_point: Some(vec![]), + command: Some(vec!["/actual".into(), "$(not-a-shell)".into()]), + working_directory: Some("/app".into()), + }); + let plan = plan(captured).unwrap(); + let body = &plan.bodies.function; + assert_eq!(body.user.as_deref(), Some("1234:4321")); + assert_eq!(body.entrypoint.as_ref().unwrap(), &[""]); + assert_eq!(body.cmd.as_ref().unwrap(), &["/actual", "$(not-a-shell)"]); + assert_eq!(body.working_dir.as_deref(), Some("/app")); + let env = body.env.as_ref().unwrap(); + assert!(env.iter().any(|entry| entry == "DELETED")); + let keys: HashSet<_> = env + .iter() + .map(|entry| entry.split('=').next().unwrap()) + .collect(); + assert_eq!(keys.len(), env.len()); + assert_eq!( + env.iter() + .find(|entry| entry.starts_with("AWS_LAMBDA_RUNTIME_API=")) + .unwrap(), + "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001" + ); + assert!( + !env.iter() + .any(|entry| entry.contains("outside:9001") || entry == "TEST_VALUE=old") + ); + assert!(env.contains(&"TEST_VALUE=secret-value".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_VERSION=27".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192".into())); + assert!(env.contains(&"AWS_LAMBDA_FUNCTION_TIMEOUT=3".into())); + assert!(env.contains(&"LAMBDA_TASK_ROOT=/var/task".into())); + let relay = plan.bodies.relay.env.as_ref().unwrap(); + assert_eq!(relay.len(), 4); + assert!(!relay.iter().any(|entry| entry.contains("secret-value"))); +} + +#[rstest::rstest] +#[case::absent("", "1000:1000")] +#[case::root("root", "1000:1000")] +#[case::numeric_root("0:4321", "1000:1000")] +#[case::padded_root("0000", "1000:1000")] +#[case::numeric("1234", "1234")] +#[case::named("nobody:nogroup", "nobody:nogroup")] +#[case::named_numeric_group("app_user:0", "app_user:0")] +fn test_should_select_nonroot_container_users(#[case] user: &str, #[case] expected: &str) { + assert_eq!(image_user(user, "1000:1000").unwrap(), expected); +} +#[rstest::rstest] +#[case(":1000")] +#[case("1000:")] +#[case("root:group:extra")] +#[case("+0")] +#[case("4294967295")] +#[case("account name")] +#[case("/root")] +fn test_should_reject_malformed_image_users(#[case] user: &str) { + assert!(image_user(user, "1000:1000").is_err()); +} + +#[rstest::rstest] +#[case("host")] +#[case("bridge")] +#[case("default")] +#[case("container:other")] +#[case("")] +#[case("bad/network")] +fn test_should_reject_unsafe_network_selections(#[case] value: &str) { + assert!(value.parse::().is_err()); +} +#[test] +fn test_should_accept_disconnected_and_named_operator_networks_without_claiming_probe_success() { + for name in ["none", "rustack-test", "ab0123"] { + assert!(name.parse::().is_ok()); + } + for (uid, gid, cpu) in [ + (0, 1000, 1_000_000_000), + (1000, 0, 1_000_000_000), + (u32::MAX, 1000, 1_000_000_000), + (1000, 1000, 0), + (1000, 1000, 99_999_999), + (1000, 1000, 16_000_000_001), + ] { + assert!( + ContainerPolicy::new( + "none".parse().unwrap(), + uid, + gid, + cpu, + RelayBudgets::default() + ) + .is_err() + ); + } + for cpu in [100_000_000, 16_000_000_000] { + assert!( + ContainerPolicy::new( + "none".parse().unwrap(), + 1, + 2_147_483_647, + cpu, + RelayBudgets::default() + ) + .is_ok() + ); + } + for durations in [ + ( + Duration::ZERO, + Duration::from_secs(600), + Duration::from_secs(10), + ), + ( + Duration::from_secs(61), + Duration::from_secs(600), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_secs(86401), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_millis(1500), + Duration::from_secs(10), + ), + ( + Duration::from_secs(5), + Duration::from_secs(600), + Duration::from_secs(61), + ), + ] { + assert!(RelayBudgets::new(durations.0, durations.1, durations.2).is_err()); + } +} + +#[rstest::rstest] +#[case("backend")] +#[case("zip")] +#[case("runtime")] +#[case("handler")] +#[case("nil_revision")] +#[case("revision_case")] +#[case("name")] +#[case("name_length")] +#[case("alias")] +#[case("zero_version")] +#[case("padded_version")] +#[case("overflow_version")] +#[case("timeout_zero")] +#[case("timeout_excess")] +#[case("timeout_fraction")] +#[case("memory_low")] +#[case("memory_high")] +#[case("tmp_low")] +#[case("tmp_high")] +#[case("architecture")] +#[case("region_missing")] +#[case("region_mismatch")] +#[case("region_invalid")] +#[case("reserved_env")] +#[case("environment_bytes")] +#[case("environment_nul")] +#[case("empty_argv")] +fn test_should_reject_invalid_captured_revisions_before_any_daemon_operation( + #[case] scenario: &str, +) { + let mut revision = revision(); + match scenario { + "backend" => revision.backend = ExecutorBackend::Native, + "zip" => revision.artifact = DeploymentArtifact::Zip(ValidatedArtifact::fixture()), + "runtime" => revision.runtime = Some("provided.al2023".into()), + "handler" => revision.handler = Some("bootstrap".into()), + "nil_revision" => revision.revision_id = Uuid::nil().to_string(), + "revision_case" => revision.revision_id = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA".into(), + "name" => revision.function_name = "bad/name".into(), + "name_length" => revision.function_name = "x".repeat(65), + "alias" => revision.qualifier = "prod".into(), + "zero_version" => revision.qualifier = "0".into(), + "padded_version" => revision.qualifier = "01".into(), + "overflow_version" => revision.qualifier = "18446744073709551616".into(), + "timeout_zero" => revision.timeout = Duration::ZERO, + "timeout_excess" => revision.timeout = Duration::from_secs(901), + "timeout_fraction" => revision.timeout = Duration::from_millis(1500), + "memory_low" => revision.memory_mb = 127, + "memory_high" => revision.memory_mb = 10241, + "tmp_low" => revision.ephemeral_storage_mb = 511, + "tmp_high" => revision.ephemeral_storage_mb = 10241, + "architecture" => revision.architecture = Architecture::X86_64, + "region_missing" => { + revision.environment.remove("AWS_REGION"); + } + "region_mismatch" => { + revision + .environment + .insert("AWS_DEFAULT_REGION".into(), "different".into()); + } + "region_invalid" => { + revision + .environment + .insert("AWS_REGION".into(), "outside\nregion".into()); + } + "reserved_env" => { + revision + .environment + .insert("AWS_LAMBDA_RUNTIME_API".into(), "outside:9001".into()); + } + "environment_bytes" => { + revision + .environment + .insert("TOO_LARGE".into(), "x".repeat(4096)); + } + "environment_nul" => { + revision.environment.insert("BAD_VALUE".into(), "\0".into()); + } + "empty_argv" => { + revision.image_config = Some(ImageConfig { + entry_point: Some(vec![]), + command: Some(vec![]), + working_directory: None, + }); + } + _ => panic!("unknown fixture scenario"), + } + assert!(plan(revision).is_err(), "{scenario}"); +} + +#[tokio::test] +async fn test_should_retain_the_real_directory_until_allocation_plan_is_released() { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let plan = EnvironmentPlan::new( + directory, + image_with("", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(super::super::DirectoryLeaseError::InUse) + )); + drop(plan); + assert!( + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .is_ok() + ); +} + +#[tokio::test] +async fn test_should_submit_exact_captured_platform_and_refuse_revoked_availability_before_io() { + let mut daemon = super::super::tests::Daemon::new(); + let mut wire = daemon.wire().await; + let plan = Arc::new(plan(revision()).unwrap()); + let worker = { + let plan = plan.clone(); + let port = wire.port(); + tokio::spawn(async move { plan.create_relay(&port, deadline()).await }) + }; + let request = daemon.next().await; + assert!(request.head.contains("platform=linux%2Farm64")); + request + .reply(201, b"{\"Id\":\"created\",\"Warnings\":[]}") + .await; + assert_eq!(worker.await.unwrap().unwrap().id, "created"); + let (release, _) = watch::channel(()); + let (stop, stopping) = watch::channel(false); + let revoked = PinnedImage::new( + image_with("", vec![]).resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopping, + ); + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(revoked); + let plan = EnvironmentPlan::new( + directory(), + image_with("", vec![]), + Arc::new(revision), + policy(), + ) + .unwrap(); + stop.send_replace(true); + assert!( + plan.create_function(&wire.port(), deadline()) + .await + .is_err() + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_normalize_inherited_duplicates_empty_values_and_deletion_markers_once() { + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(image_with( + "1000", + vec![ + "INHERITED=first".into(), + "INHERITED=last".into(), + "REMOVED=old".into(), + "REMOVED".into(), + "EMPTY=".into(), + "TEST_VALUE=old".into(), + "TEST_VALUE".into(), + ], + )); + let plan = plan(revision).unwrap(); + let env = plan.bodies.function.env.as_ref().unwrap(); + for (key, expected) in [ + ("INHERITED", "INHERITED=last"), + ("REMOVED", "REMOVED"), + ("EMPTY", "EMPTY="), + ("TEST_VALUE", "TEST_VALUE=secret-value"), + ] { + let matching: Vec<_> = env + .iter() + .filter(|entry| entry.split('=').next() == Some(key)) + .collect(); + assert_eq!(matching, vec![expected]); + } +} + +#[tokio::test] +async fn test_should_retain_both_live_image_leases_through_allocation_plan_lifetime() { + let image = image_with("", vec![]); + let (relay_release, mut relay_released) = watch::channel(()); + let (function_release, mut function_released) = watch::channel(()); + let (_, stopped) = watch::channel(false); + let make = |release| { + PinnedImage::new( + image.resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped.clone(), + ) + }; + let relay = make(relay_release); + let function = make(function_release); + let mut revision = revision(); + revision.artifact = DeploymentArtifact::Image(function.clone()); + let plan = + EnvironmentPlan::new(directory(), relay.clone(), Arc::new(revision), policy()).unwrap(); + drop(relay); + drop(function); + assert!(!relay_released.has_changed().unwrap()); + assert!(!function_released.has_changed().unwrap()); + drop(plan); + let _ = timeout(Duration::from_secs(1), relay_released.changed()) + .await + .unwrap(); + let _ = timeout(Duration::from_secs(1), function_released.changed()) + .await + .unwrap(); + assert!(relay_released.has_changed().is_err()); + assert!(function_released.has_changed().is_err()); +} diff --git a/crates/rustack-lambda-core/src/function_config.rs b/crates/rustack-lambda-core/src/function_config.rs index f48e59e7..6a39b6d1 100644 --- a/crates/rustack-lambda-core/src/function_config.rs +++ b/crates/rustack-lambda-core/src/function_config.rs @@ -270,11 +270,19 @@ pub(crate) fn validate_image_config(config: &ImageConfig) -> Result<(), LambdaSe } fn validate_environment(environment: &HashMap) -> Result<(), LambdaServiceError> { - if environment.len() > 128 { - return Err(invalid("Environment has too many variables")); - } + validate_environment_entries(environment) +} + +pub(crate) fn validate_environment_entries<'a>( + environment: impl IntoIterator, +) -> Result<(), LambdaServiceError> { + let mut count = 0_u16; let mut bytes = 0usize; for (key, value) in environment { + count = count.saturating_add(1); + if count > 128 { + return Err(invalid("Environment has too many variables")); + } if !(2..=64).contains(&key.len()) || !key.as_bytes().first().is_some_and(u8::is_ascii_alphabetic) || !key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') diff --git a/crates/rustack-lambda-core/tests/docker_wire.rs b/crates/rustack-lambda-core/tests/docker_wire.rs index 2f585fab..51f6099d 100644 --- a/crates/rustack-lambda-core/tests/docker_wire.rs +++ b/crates/rustack-lambda-core/tests/docker_wire.rs @@ -3,20 +3,24 @@ #![cfg(unix)] #![forbid(unsafe_code)] -use std::{collections::HashMap, env, time::Duration}; +use std::{collections::HashMap, env, sync::Arc, time::Duration}; use anyhow::{Context, Result, bail, ensure}; use bytes::Bytes; use http::StatusCode; use rustack_lambda_core::{ - artifact::{Architecture as ImageArchitecture, ImageReference, PinnedImage}, + artifact::{ + Architecture as ImageArchitecture, DeploymentArtifact, ImageReference, PinnedImage, + }, docker::{ - DirectoryLease, DirectoryLeaseError, DockerError, DockerKey, DockerPort, DockerSocket, - DockerWire, FunctionLogs, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, - OutputChannel, OwnerFilter, - models::{ContainerCreateBody, HostConfig}, + ContainerPolicy, DirectoryLease, DirectoryLeaseError, DockerError, DockerKey, DockerPort, + DockerSocket, DockerWire, EnvironmentPlan, FunctionLogs, ImageLeaseConfig, + ImageLeaseManager, ImagePinScope, OutputChannel, OwnerFilter, RelayBudgets, }, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, }; +use rustack_lambda_model::types::ImageConfig; use rustack_lambda_runtime::{ client::{ClientConfig, ClientEvent, ControlClient}, metadata::local_trace_id, @@ -47,22 +51,6 @@ struct Names { run: Uuid, environment: EnvironmentId, } -impl Names { - fn labels(&self) -> HashMap { - HashMap::from([ - ("rustack.lambda.ownerId".into(), self.owner.to_string()), - ("rustack.lambda.runId".into(), self.run.to_string()), - ( - "rustack.lambda.environmentId".into(), - self.environment.to_string(), - ), - ( - "rustack.lambda.functionResourceId".into(), - self.environment.to_string(), - ), - ]) - } -} #[derive(Debug)] struct Runtime { @@ -110,6 +98,8 @@ struct Environment { logs: Option, images: ImageLeaseManager, artifacts: Vec, + plan: Option, + lease: DirectoryLease, directory: Option, } impl Environment { @@ -124,7 +114,7 @@ impl Environment { let owner = lease.owner_id(); let images = ImageLeaseManager::start( wire.port(), - ImagePinScope::new(lease), + ImagePinScope::new(lease.clone()), ImageLeaseConfig::default(), )?; ensure!( @@ -147,6 +137,8 @@ impl Environment { logs: None, images, artifacts: Vec::new(), + plan: None, + lease, directory: Some(directory), }) } @@ -168,6 +160,10 @@ impl Environment { } async fn start(&mut self) -> Result<()> { + self.start_with(None, true).await + } + + async fn start_with(&mut self, overrides: Option, bootstrap: bool) -> Result<()> { let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; let (architecture, daemon_arch) = match arch.as_str() { "arm64" => (Architecture::Arm64, "aarch64"), @@ -186,35 +182,47 @@ impl Environment { let function = self .image(&format!("rustack-p0-handler:{arch}"), &arch) .await?; - let relay_plan = relay.resolved().metadata().execution.plan(None)?; - let function_plan = function.resolved().metadata().execution.plan(None)?; - port.create_container_on_platform( - &self.names.relay, - &ContainerCreateBody { - image: Some(relay.resolved().metadata().image_id.as_str().into()), - entrypoint: Some(relay_plan.entrypoint().to_vec()), - cmd: Some(relay_plan.command().to_vec()), - working_dir: Some(relay_plan.working_directory().into()), - user: Some("1000:1000".into()), - env: Some(vec![ - format!("RUSTACK_LAMBDA_ENVIRONMENT_ID={}", self.names.environment), - "RUSTACK_LAMBDA_RELAY_IO_SECONDS=60".into(), - "RUSTACK_LAMBDA_RELAY_RETIRE_SECONDS=60".into(), - ]), - labels: Some(self.names.labels()), - attach_stdin: Some(true), - attach_stdout: Some(true), - attach_stderr: Some(true), - open_stdin: Some(true), - stdin_once: Some(true), - tty: Some(false), - host_config: Some(limits("none".into(), 64 * 1024 * 1024, 100_000_000, 64)), - ..Default::default() - }, - relay.resolved().metadata().architecture, - deadline(), - ) - .await?; + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("wire-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(function.resolved().metadata().architecture) + .artifact(DeploymentArtifact::Image(function)) + .image_config(overrides) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ("TEST_VALUE".into(), "from-captured-revision".into()), + ])) + .timeout(Duration::from_secs(120)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build(); + let plan = EnvironmentPlan::new( + self.lease.clone(), + relay, + Arc::new(revision), + ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new(IO, Duration::from_secs(600), IO)?, + )?, + )?; + self.names.relay = plan.relay_key().clone(); + self.names.function = plan.function_key().clone(); + self.names.environment = plan.environment_id(); + self.plan = Some(plan); + self.plan + .as_ref() + .context("allocation plan")? + .create_relay(&port, deadline()) + .await?; self.attach(&port, architecture).await?; port.start_container(&self.names.relay, deadline()).await?; let client = &mut self.runtime.as_mut().context("runtime owner")?.client; @@ -222,35 +230,11 @@ impl Environment { client.next(deadline()).await? == ClientEvent::RelayReady, "relay handshake mismatch" ); - port.create_container_on_platform( - &self.names.function, - &ContainerCreateBody { - image: Some(function.resolved().metadata().image_id.as_str().into()), - entrypoint: Some(function_plan.entrypoint().to_vec()), - cmd: Some(function_plan.command().to_vec()), - working_dir: Some(function_plan.working_directory().into()), - user: Some("1000:1000".into()), - labels: Some(self.names.labels()), - env: Some(vec![ - "AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001".into(), - "AWS_LAMBDA_FUNCTION_NAME=wire-fixture".into(), - "AWS_LAMBDA_FUNCTION_VERSION=$LATEST".into(), - "AWS_LAMBDA_FUNCTION_MEMORY_SIZE=192".into(), - "AWS_REGION=us-east-1".into(), - "AWS_DEFAULT_REGION=us-east-1".into(), - ]), - host_config: Some(limits( - format!("container:{}", self.names.relay.as_str()), - 192 * 1024 * 1024, - 1_000_000_000, - 256, - )), - ..Default::default() - }, - function.resolved().metadata().architecture, - deadline(), - ) - .await?; + self.plan + .as_ref() + .context("allocation plan")? + .create_function(&port, deadline()) + .await?; self.logs = Some(FunctionLogs::start( port.attach_logs(&self.names.function, deadline()).await?, IO, @@ -258,10 +242,12 @@ impl Environment { )?); port.start_container(&self.names.function, deadline()) .await?; - ensure!( - client.next(deadline()).await? == ClientEvent::BootstrapReady, - "function readiness mismatch" - ); + if bootstrap { + ensure!( + client.next(deadline()).await? == ClientEvent::BootstrapReady, + "function readiness mismatch" + ); + } Ok(()) } @@ -357,6 +343,7 @@ impl Environment { ); Ok(()) }); + self.plan.take(); self.artifacts.clear(); let images = self .images @@ -386,23 +373,6 @@ impl Environment { } } -fn limits(network_mode: String, memory: i64, nano_cpus: i64, pids_limit: i64) -> HostConfig { - HostConfig { - network_mode: Some(network_mode), - memory: Some(memory), - nano_cpus: Some(nano_cpus), - pids_limit: Some(pids_limit), - readonly_rootfs: Some(true), - cap_drop: Some(vec!["ALL".into()]), - security_opt: Some(vec!["no-new-privileges:true".into()]), - tmpfs: Some(HashMap::from([( - "/tmp".into(), - "rw,noexec,nosuid,size=536870912".into(), - )])), - ..Default::default() - } -} - async fn remove(port: &DockerPort, key: &DockerKey) -> Result<()> { match port.remove_container(key, deadline()).await { Ok(()) | Err(DockerError::Status(StatusCode::NOT_FOUND)) => Ok(()), @@ -418,7 +388,7 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra let result = async { environment.start().await?; let event = - json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE,"logs":true}); + json!({"value":35,"padding":"p".repeat(LARGE),"response_bytes":LARGE,"logs":true,"policy":true}); let computed = environment.call(&event, false).await?; ensure!( computed["computed"] == 42 @@ -439,6 +409,22 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra && computed["pidsMax"] == "256", "function resource controls differ" ); + ensure!( + computed["tmpBytes"] == 512 * 1024 * 1024_u64 + && computed["tmpMode"] == 0o1777 + && computed["swapMax"] == "0" + && computed["capEff"] == "0000000000000000" + && computed["noNewPrivs"] == "1" + && computed["taskRoot"] == "/var/task" + && computed["functionName"] == "wire-fixture" + && computed["functionVersion"] == "$LATEST" + && computed["region"] == "us-east-1" + && computed["configuredValue"] == "from-captured-revision", + "allocation policy runtime enforcement differs" + ); + let cpu = computed["cpuMax"].as_str().context("CPU quota report")?; + let cpu: Vec<_> = cpu.split_whitespace().collect(); + ensure!(matches!(cpu.as_slice(), [quota, period] if quota.parse::().is_ok_and(|quota| quota > 0 && period.parse::() == Ok(quota))), "CPU quota does not enforce one CPU"); let warm = environment.call(&json!({"value":0}), false).await?; ensure!( warm["computed"] == 7 && warm["counter"] == 2, @@ -480,49 +466,39 @@ async fn test_should_execute_real_ric_over_production_wire_with_large_binary_fra #[tokio::test] #[ignore = "requires explicit Docker wire qualification images/socket"] async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() -> Result<()> { - let mut environment = Environment::new().await?; - let result = async { - let arch = env::var("RUSTACK_RUNTIME_ARCH").context("explicit architecture required")?; - let port = environment.wire.port(); - let image = environment - .image(&format!("rustack-p0-handler:{arch}"), &arch) - .await?; - for reset in [false, true] { - let overrides = rustack_lambda_model::types::ImageConfig { + for reset in [false, true] { + let mut environment = Environment::new().await?; + let result = async { + let overrides = ImageConfig { entry_point: reset.then(Vec::new), command: Some(if reset { vec!["/bin/true".into()] } else { vec![] }), - ..Default::default() + working_directory: Some("/tmp".into()), }; - let plan = image + environment + .start_with(Some(overrides.clone()), false) + .await?; + let port = environment.wire.port(); + let expected = environment + .artifacts + .last() + .context("function artifact")? .resolved() .metadata() .execution .plan(Some(&overrides))?; - port.create_container_on_platform( - &environment.names.function, - &ContainerCreateBody { - image: Some(image.resolved().metadata().image_id.as_str().into()), - entrypoint: Some(plan.entrypoint().to_vec()), - cmd: Some(plan.command().to_vec()), - working_dir: Some(plan.working_directory().into()), - labels: Some(environment.names.labels()), - user: Some("1000:1000".into()), - host_config: Some(limits("none".into(), 192 * 1024 * 1024, 1_000_000_000, 256)), - ..Default::default() - }, - image.resolved().metadata().architecture, - deadline(), - ) - .await?; let actual = port .inspect_container(&environment.names.function, deadline()) .await? .config .context("actual create config")?; + ensure!( + actual.working_dir.as_deref() == Some("/tmp"), + "working directory override differs" + ); if reset { ensure!( actual.entrypoint.as_deref().unwrap_or_default().is_empty(), @@ -540,7 +516,7 @@ async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() actual .entrypoint .as_deref() - .is_some_and(|entrypoint| entrypoint == plan.entrypoint()), + .is_some_and(|entrypoint| entrypoint == expected.entrypoint()), "inherited entrypoint mismatch" ); ensure!( @@ -548,11 +524,18 @@ async fn test_should_preserve_empty_overrides_in_actual_docker_create_merging() "daemon restored a cleared command" ); } - remove(&port, &environment.names.function).await?; + let exit = port + .wait_container(&environment.names.function, deadline()) + .await?; + ensure!( + exit.status_code == if reset { 0 } else { 142 }, + "empty override execution exit differs" + ); + Ok(()) } - Ok(()) + .await; + let cleanup = environment.cleanup().await; + result.and(cleanup)?; } - .await; - let cleanup = environment.cleanup().await; - result.and(cleanup) + Ok(()) } diff --git a/specs/index.md b/specs/index.md index 7af1dcdc..0d7593ee 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: implementation contract dated 2026-10-25; P0 evidence and P1 invocation/artifact/cancellation plus shared Native/relay/client runtime contracts are implemented. Bounded Docker management/attach IO, platform-aware local resolution and owned image preparation/leases are also implemented and qualified against real RIC images. Typed deployment settlement, leased invocation capture and retained provider deployment ownership/API wiring are implemented. Explicit side-effect-free code DryRun and strict bounded deployment decoding are implemented and independently reviewed with live SDK/HTTP validation evidence. The authentication/shared-core/Lambda production boundary strict gate is also implemented and independently reviewed. B01 dependency security is closed with patched libraries, explicit regional SDK features and ZIP9 regressions; audit and all deny checks pass. Persistent descriptor-backed directory ownership and pin/cleanup retention are implemented and independently reviewed (§4.1/D28); prior-run reconciliation remains pending. Independent bounded function log capture and output-only live attach are also implemented/reviewed (§3.7/D29), with actual stdout/stderr receipt in both real wire suites. Provider control-plane tests use an explicit test executor capability; production Docker remains unavailable until the Docker executor/probes land. The ASL engine also remains pending; see the implementation ledger for verified scope. All documents in this set are in English. Start here for the requested ASL engine and actual Docker image Lambda execution. +Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs and immutable allocation plans. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. + +Production Docker selection remains unavailable until environment/pool lifecycle, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 70e0bf75..574bf4d2 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -494,6 +494,20 @@ Coordinator Filesystem worker Image owner / cleanup Errors are typed Configuration/InUse/InvalidState/Deadline/Cancelled/Worker/Io with source IO errors, static bounded Display/Debug and no path/identity bytes in diagnostics. Apply AGENTS.md boundary lints and doc/example requirements. Tests prove stable owner/fresh run, independent-process contention/reacquisition, clone/pin lifetime, malformed/oversized/nil identity, symlink/hardlink/FIFO/mode rejection, bounded pending-file recovery, deadline/cancelled-worker retention and image-manager cleanup ownership. Real wire fixtures keep a temp data directory and use its actual lease. Production Docker selection stays unavailable until the separate startup reconciliation, historical uncertain-effect handling, pool/executor and probe consumers are implemented; a directory lease alone must not advertise a ready backend. +### 4.2 Immutable environment allocation plan + +`EnvironmentPlan::new(directory_lease, relay_image, captured_revision, policy)` constructs the two fixed create requests before any daemon mutation. The plan retains the actual directory capability, live relay/function image leases and Arc revision until cleanup owners release it. Allocate one fresh environment UUID and preallocate distinct relay/function names before IO. Labels on both include directory owner/run/environment/function resource identity and a fixed relay/function role; never accept labels or names from invocation payloads. Use captured addressable image IDs plus explicit platforms, and recheck both leases before each create. A plan is not a reservation, reconciliation receipt, executor or readiness proof; the environment owner must register it and retain uncertain-create ownership before awaiting either create. + +`ContainerPolicy` validates UID/GID in1..2,147,483,647, CPU in100,000,000..16,000,000,000 nanocpus and whole-second relay budgets (IO/retirement1..60s, idle1..86,400s). `NetworkSelection` accepts an explicit none mode for disconnected pure-handler testing or a bounded operator user-bridge name/ID. Reject host, default bridge and arbitrary container sharing; only the plan itself constructs function `container:` sharing. Named bridges require separate daemon bridge/ownership validation at startup; parsing a selection does not prove network readiness. The default enabled topology remains a private/user bridge; none is deliberate opt-in and advertises SDK callbacks unavailable. Docker documents [none isolation](https://docs.docker.com/engine/network/drivers/none/) and [container namespace sharing](https://docs.docker.com/engine/network/). + +Revalidate externally constructible revisions: Docker backend, Image artifact, live matching Linux architectures, absent ZIP runtime/handler, canonical nonnil revision UUID, bounded function name and resolved numeric version/$LATEST, integral timeout1..900s, memory128..10,240MiB and ephemeral512..10,240MiB. Validate captured user environment under the existing4KiB/count/key rules, separately accepting only the provider's bounded AWS_REGION/AWS_DEFAULT_REGION injections; require matching nonempty region values. Compose a deterministic map with exactly one entry per key: last inherited image value/deletion marker, then captured variables, then authoritative runtime metadata. Bare-name inherited deletion markers remain explicit unless a later tier supplies that key. Do not concatenate duplicate names: process lookup of duplicate environment entries is not a portable precedence mechanism. Inject fixed loopback Runtime API, name/resolved version/memory/timeout, on-demand initialization, log group/stream and task root. Never copy process environment or host credentials. The four Rustack-supplied relay Env entries contain only its environment UUID and validated relay budgets; Docker also retains validated relay-image defaults. No captured function variables or process environment are supplied to the relay. Preserve omitted/empty ImageConfig through the already-validated execution plan, with no shell or tag re-resolution. + +Both create bodies disable inherited healthchecks and daemon persistent logging, set Tty=false, no automatic removal/restart, read-only rootfs, cap-drop ALL, no-new-privileges and privileged=false, with no host binds/devices or published ports. Function memory and swap-total are equal (no extra unaccounted swap),256 PID cap and the selected CPU cap; relay is64MiB/64PIDs/0.1CPU. Aggregate memory admission uses checked function+relay bytes. The function /tmp tmpfs has its captured ephemeral byte cap, nosuid/nodev and mode1777; relay /tmp is1MiB. Tmpfs memory still counts against each memory cap. Matching memory/swap semantics follow [Docker resource constraints](https://docs.docker.com/engine/containers/resource_constraints/). Require default seccomp/resource enforcement in the separate daemon probe. + +Absent/root/numeric-zero image USER is replaced with configured nonzero UID:GID; preserve bounded nonroot numeric/named image USER declarations and reject malformed user/group grammar rather than passing arbitrary text. Named-account resolution remains the image's own /etc/passwd contract, not a host account lookup or proof against operator-supplied hostile image aliases. The relay always uses configured nonzero UID:GID. Permission-incompatible images fail initialization. No host filesystem/user lookup or credential inheritance is allowed. + +Tests pin immutable IDs, platforms, distinct names/labels, image overrides, env ordering/authoritative metadata/redaction, user policy, all resource/swap/tmpfs/health/log/restart fields, invalid revision/policy/region rejection before mutation, and actual directory/image lease lifetime. Real wire qualification consumes the plan for both containers and verifies the handler's cgroup/user/rootfs/tmpfs output, actual argv/workdir overrides, warm/error results and owned cleanup. These are allocation policy checks; environment/pool lifecycle, historical reconciliation, named bridge validation, probes and SDK executor acceptance remain required. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 8902a7d1..74652d56 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -35,7 +35,7 @@ Start with `RUSTACK_CONFIG=./rustack.yaml rustack`; the current CLI does not hav |---|---| | LAMBDA_DOCKER_SOCKET | `unix:///var/run/docker.sock`; operator may select Desktop/rootless socket; absolute local socket path≤4,096 bytes; no TCP/SSH fallback | | LAMBDA_DOCKER_RELAY_IMAGE | Required when Docker enabled; bounded image reference, resolved and pinned at startup; no implicit pull | -| LAMBDA_DOCKER_NETWORK | Operator-owned user bridge name/ID, or a per-owner private bridge created by Rustack;≤128 ASCII bytes; never host/container arbitrary input | +| LAMBDA_DOCKER_NETWORK | Operator-owned user bridge name/ID, or a per-owner private bridge created by Rustack;≤128 ASCII bytes; explicit none opts into disconnected pure handlers with SDK callbacks unavailable; never host/default-bridge/arbitrary-container sharing | | LAMBDA_DOCKER_PULL_POLICY | never / ifMissing; default never; missing image gives actionable error | | LAMBDA_DOCKER_REGISTRIES | Required for ifMissing; comma-separated≤32 approved registry hosts, each≤253 bytes; exact host[:port] matching, no suffix matching | | LAMBDA_DOCKER_REGISTRY_AUTH_FILE | Optional operator-owned YAML credential map by approved registry host;≤64 KiB,≤32 entries, secret fields redacted and permissions checked; parse as a secret-bearing type, never execute credential helpers | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 7df728b9..2cba4e4b 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -100,6 +100,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi ## 4. Security acceptance +`make test-lambda-environment-plans` verifies §4.2/D30 before allocation: fixed image IDs/platforms, distinct names and ownership roles, relay/function resource/swap/tmpfs/health/log/restart policy, user selection/grammar, bounded revision/region/env/argv validation, single-key inherited→captured→runtime environment precedence with deletion markers, retained real-directory and both-image lease lifetime, and revoked availability before IO. A spawned create regression proves the wire future is Send. The actual two-case wire suite consumes EnvironmentPlan for both ordinary RIC execution and empty ImageConfig overrides: require handler-reported CPU/memory/no-swap/PIDs,512MiB tmpfs/mode, cap-drop/no-new-privileges/read-only rootfs, and configured metadata despite intentionally conflicting image defaults. Inspect real CMD/ENTRYPOINT/WORKDIR and require expected exits142/0 for inherited-empty and reset-entrypoint variants. Join control/log streams and remove both owned containers/aliases on all normal test outcomes. This is allocation-policy qualification, not lifecycle/pool, startup bridge/probe, historical reconciliation or SDK executor acceptance. + `make test-lambda-logs` verifies the independent function output owner: binary stdout/stderr ordering, separate bounded overwrite rings and saturating discard counts, combined4KiB tail, EOF/failure retention, socket closure on framing failure, snapshot admission bounds, partial-frame reads retained through completed/abandoned snapshot requests, fixed partial/silence deadlines, output-flood responsiveness, cancelled shutdown join, off-runtime Drop and cached actor panic. A scripted daemon pins output-only/no-replay attach query and driver handoff. The real wire suite additionally attaches before function start, enables explicit handler stdout/stderr markers, requires both markers without affecting computed results, and joins logs independently after function removal. These tests qualify capture, not complete per-invocation log attribution or the still-pending executor. `make test-lambda-ownership` verifies the real directory capability independently of Docker: stable owner/fresh run, descriptor/allocated-pin lifetime, actual subprocess contention and reacquisition, bounded malformed/pending state, symlink spellings (including trailing slash/current-directory), hardlink/FIFO/permission rejection, trusted/sticky ancestor rules and repeated parent sync after interrupted creation. Worker tests cover admission expiry, running deadline/cancelled waiter retention, completed-but-unconsumed result capacity and panic release. A scripted-daemon integration additionally holds a real directory lease through abandoned image-owner quarantine and delayed deletion, requiring exclusion until cleanup settles. Production wire fixtures acquire actual directory leases and retain their namespace files on uncertain cleanup. These checks do not establish prior-run reconciliation, startup readiness or full M0 acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 79c74da6..f0f6e7e2 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker environment/pool executor, startup ownership/reconciliation/probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs and immutable environment allocation plans, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker environment/pool executor, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -145,6 +145,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 | In progress | Pinned transport study/§3.6/D19 define generated-type reuse, bounded Unix HTTP/upgrade IO, retained handoff and reserved cleanup slots. Shared core JSON preflight is implemented/reviewed before the Docker model consumer:21 new parser tests, two512-case properties, four core doctests and strict core boundary Clippy pass. Workspace:1,888 tests; audit remains unchanged B01. DockerWire now implements typed management and strict attach with15 fake-daemon/framing tests, retained IO/driver supervision and separate lifecycle-wait capacity (W32/D20). Real near6MiB official RIC wire tests pass on arm64 VZ and amd64/QEMU; independent review finds no remaining in-slice issue. Workspace1,903 tests and standard gates pass. Read-only ImageResolver and retained validated defaults/plans now bind platform inspection to captured addressable parent IDs, preserve selected manifests, verify snapshot fingerprints and support explicit platform create.61 additional tests and corrected two-case real Docker suites pass; independent W33–W38 fixes are rechecked. Workspace1,964 tests and standard gates pass. Owned ImageLeaseManager/PinnedImage preparation now preserves private-alias availability, proves unpacked quota, retains uncertain mutations and joins phase-aware cleanup/recovery.33 additional cases pass; real two-case suites on both retained daemons consume leases through actual container execution/cleanup. Independent W39–W42 fixes and final recheck pass. Workspace1,997 tests and standard gates pass. Typed deployment state/store settlement and leased invocation capture are implemented and independently reviewed (§2.10); initial/update failures, stale resource/revision/generation results, rollback and metadata-only restore are covered by25 additional cases (346 core tests,92 doctests). Workspace2,022 tests and standard build/test/fmt/Clippy/rustdoc gates pass; Native provider/DryRun regressions pass without claiming the gated Squib case as VM evidence. W43–W46 fixes are reviewed; strict B03 and dependency B01 remain open. Retained ImageDeploymentManager and provider Image admission/status/revision wiring are implemented/reviewed (§2.11).20 core additions and one model wire-field case pass (366 core tests,101 doctests; workspace2,043 tests). Actual Rust SDK/HTTP revision/412 regression passes separately from Native provider execution tests. W47–W50 fixes pass. B04 is now closed by explicit side-effect-free code-update DryRun, strict deployment DTOs and bounded borrowed-blob JSON decoding (§2.12).29 additional core cases pass (395 core tests,101 doctests; workspace2,072 tests,401 gated cases), and actual SDK/HTTP validation and revision tests pass. W51–W53 fixes/recheck pass. B03 strict closure is independently reviewed (§7.1), with all six production libraries passing without new ignores; two added HMAC-vector tests preserve signing semantics. Final workspace2,074 tests and standard gates pass; the boundary target passes659 unit tests/165 doctests and strict Clippy for all six libraries. B01 dependency closure is implemented/reviewed (§7.2/D27, W57–W59): nine ZIP regressions, workspace2,083 tests,833 boundary tests/doctests, green standard/strict gates, clean audit and all deny checks. Regional SDK/HTTP and Native provider regressions pass. Production Docker execution is still disabled. Directory ownership/previous-run reconciliation, allowed pulls, remaining stream/network/probe consumers and environment/pool integration remain required. | | 1.3 directory namespace | Implemented/reviewed; runtime consumers remain | DirectoryLease binds permanent nonreplaced file locking, durable checksummed owner/fresh run and trusted descriptor-relative filesystem operations to all image scopes/pins/jobs/recovery. W60–W62 fixes independently rechecked; 38 ownership cases plus abandoned image-owner cleanup pass. Workspace2,122 tests, core443 tests/103 doctests, boundary874 unit/doc passes, all standard/strict/audit/deny gates pass. Both real two-case wire suites pass on arm64 VZ and amd64/QEMU; post-test owned containers/internal aliases are empty. This does not prove previous-run reconciliation, Linux-host lock qualification or Docker executor readiness. | | 1.3 function logs | Implemented/reviewed; environment integration remains | Output-only attach and FunctionLogs retain partial framing across snapshot traffic, bounded independent rings/combined4KiB binary tail, fair service, early socket closure and cached joined shutdown (§3.7/D29).18 focused tests pass; W63/W64 fixes independently rechecked. Core461 tests/107 doctests, workspace2,140 tests, boundary896 unit/doc passes and all standard/strict gates pass. Both real two-case wire suites receive deliberate handler stdout/stderr markers independently from computed results and join logs during cleanup. No new dependency/version. Per-invocation log attribution, environment integration and complete executor acceptance remain pending. | +| 1.3 allocation policy | Implemented/reviewed; lifecycle owner remains | EnvironmentPlan fixes both create bodies to captured identities/platforms/revision, retained directory/image leases, validated policy and authoritative unique-key environment (§4.2/D30).54 new cases pass; W65–W67 fixed/rechecked. Core515 tests/116 doctests, workspace2,194 tests, boundary959 unit/doc passes and standard/strict/audit/deny gates pass. Both real two-case wire suites now consume plans for normal/empty ImageConfig execution, with actual resource/tmpfs/privilege/env assertions; all five shared runtime Docker regressions also pass per daemon. No production pool/executor or historical reconciliation readiness is claimed. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining network/probe consumers plus function-log/environment integration, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) is also implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining network/probe consumers plus function-log/environment integration, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index a8de054b..5c369036 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -306,6 +306,24 @@ Final-source workspace build/test/fmt/Clippy/rustdoc pass with2,140 tests, zero Both explicit two-case Docker wire suites pass on Engine29.5.2 arm64 VZ and amd64/QEMU. The official RIC handler prints deliberate stdout/stderr markers only for the requested fixture event; the test requires both independently from its computed near6MiB response and warm/error outcomes, then joins capture after function removal. After both suites, ownership queries show no Rustack containers or internal aliases. These are60s stream/120s attempt stress qualifications, not default executor latency or native x86/Desktop evidence. Historical reconciliation, pulls, network/probe consumers, environment/pool actors, production wiring and P2–P5 remain required. +## P1 task1.3 immutable allocation policy review + +Independent read-only review covers runtime §4.2/D30: closed allocation plans retaining namespace/image/revision leases, captured IDs/platforms, environment composition, bounded user/network policy, resource/swap/tmpfs/health/log/restart fields and real-wire consumption. The final recheck finds no remaining in-slice issue. This is allocation policy, not an environment/pool lifecycle, historical reconciliation, startup probe or SDK executor acceptance. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W65 | P2; self-review, `docker/mod.rs:412`, create future | The query serializer remained in async scope across await, making container create futures non-Send and preventing supervised Tokio workers. Finish it within a lexical block before IO | A scripted-daemon test spawns EnvironmentPlan create through Tokio, checks its captured platform and successful response | +| W66 | P2; `docker/plan.rs:566`, environment precedence | Initial concatenation retained duplicate names; process lookup cannot be treated as portable last-wins. Compose a deterministic unique-key map with inherited→captured→runtime precedence and explicit bare deletion markers | Uniqueness/last-inherited/unset/empty/captured cases plus actual handler execution despite conflicting image Runtime API/name/region/default variable | +| W67 | P3; runtime §4.2, relay environment wording | Four supplied relay entries do not suppress Docker image Env inheritance. State the actual validated-image-default inheritance, while excluding captured function/process environment | Final prose/source directly rechecked; no host-credential inheritance is claimed | + +The environment rule follows [POSIX duplicate-name limitations](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/V1_chap08.html) and the inspected [Moby image merge](../vendors/moby/daemon/commit.go) / [process environment merge](../vendors/moby/container/env.go), which do not deduplicate supplied new keys. Shared validation now accepts iterated entries so captured region injections can be checked separately without first cloning invalid user data. Reserved-key, byte/count and NUL rejection remain unchanged for existing provider settings. + +Fifty-four new cases pass, including both-image lease release and actual directory exclusion through plan lifetime. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,194 tests, zero failed and401 resource-gated cases. Core has515 tests/116 doctests; the six-library strict target passes779 unit tests/180 doctests (959 total), with seven Docker cases separately gated. Fresh audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources pass. No dependency/version or lint exception is added. + +Both explicit two-case real Docker wire suites pass on Engine29.5.2 arm64 VZ and amd64/QEMU. They now consume the allocation plan for normal official RIC execution and empty ImageConfig cases, including actual ENTRYPOINT/CMD/WORKDIR inspection and exit142/0. Handler reports verify memory/PID/CPU/no-swap, read-only rootfs,512MiB tmpfs/mode1777, zero effective capabilities/no-new-privileges, metadata and captured env precedence. Bounded output capture still receives stdout/stderr with daemon logging disabled; warm/error and near6MiB results remain correct. Because fixture image defaults changed, all five shared runtime/relay/client Docker regressions also pass on each daemon. These14 selected platform cases have no skips; they remain separate from full executor/default-latency and native x86/Desktop acceptance. Post-test queries on both daemons find no owner-labeled or runtime-test containers and no internal image aliases. The Native provider target also passes six actual Native/validation scenarios; its separately gated Squib selection remains excluded as VM evidence. + +The plan deliberately cannot authorize unreserved or untracked side effects: the future environment/pool owner must register it above retained IO and reconcile every uncertain create. Named bridges still require daemon validation; explicit none provides disconnected pure-handler tests only. Production Docker remains unavailable pending lifecycle/pool, reconciliation/pulls/network/probes and final wiring, and P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 5febdf03..ffec2de0 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -263,3 +263,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why:** snapshot and log flooding have bounded service quanta, user bytes never enter infrastructure diagnostics, and environment cleanup can join this socket independently of function/relay termination. The capture alone does not prove container exit or exact per-invocation log attribution. - **Pinned by:** [runtime §3.7](./rustack-workflows-20-lambda-runtime-design.md#37-retained-function-log-capture), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-26. No new dependency or version is introduced. + +## D30 — Materialize one immutable, bounded environment allocation plan + +- **Context:** container creation must bind captured function configuration to immutable image leases, namespace ownership and operator limits before any daemon side effect. Hand-built relay/function requests otherwise drift in resource, metadata and entry semantics. +- **Alternatives considered:** per-call tag resolution breaks captured versions; assembling separate arbitrary Docker bodies permits policy drift; concatenating Env tiers leaves duplicate names with undefined process lookup precedence; inheriting daemon logging can spill unbounded output outside Rustack rings. +- **Decision:** EnvironmentPlan owns the captured revision/leases and preallocated identities, validates policy and serializes fixed relay/function bodies; only typed create methods issue them. Compose exactly one environment entry per key with inherited→captured→authoritative precedence and explicit deletion markers. Disable daemon logging, healthchecks, restart and extra swap while retaining independent live attach capture. +- **Why:** one plan supplies resource admission totals and immutable allocation inputs for the future environment/pool owner. Docker create futures must be Send for supervised Tokio workers, and raw request serializers must leave scope before awaits. A plan neither reserves capacity nor proves startup/reconciliation readiness. +- **Pinned by:** [runtime §4.2](./rustack-workflows-20-lambda-runtime-design.md#42-immutable-environment-allocation-plan), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date:** 2026-10-26. No dependency/version change is required. Network none is deliberate pure-handler opt-in; bridge and resource probes remain mandatory for production enablement. diff --git a/tools/workflows-spikes/Handler.Dockerfile b/tools/workflows-spikes/Handler.Dockerfile index 95d432e4..22ab04b8 100644 --- a/tools/workflows-spikes/Handler.Dockerfile +++ b/tools/workflows-spikes/Handler.Dockerfile @@ -1,4 +1,8 @@ FROM public.ecr.aws/lambda/python:3.13@sha256:e0237850eecd760896365e768bac1526ea01098e325fa590e5bc735b26d5dbb9 COPY handler.py /var/task/handler.py +ENV AWS_LAMBDA_RUNTIME_API=outside:9001 \ + AWS_LAMBDA_FUNCTION_NAME=image-default \ + AWS_REGION=image-region \ + TEST_VALUE=image-default USER 1000:1000 CMD ["handler.handler"] diff --git a/tools/workflows-spikes/handler.py b/tools/workflows-spikes/handler.py index af8d191e..e24b2a1c 100644 --- a/tools/workflows-spikes/handler.py +++ b/tools/workflows-spikes/handler.py @@ -33,4 +33,21 @@ def handler(event, context): response["inputBytes"] = len(event["padding"].encode("utf-8")) if event.get("response_bytes"): response["large"] = "r" * event["response_bytes"] + if event.get("policy"): + temporary = os.statvfs("/tmp") + status = dict( + line.split(":", 1) for line in Path("/proc/self/status").read_text().splitlines() + if ":" in line + ) + response["tmpBytes"] = temporary.f_blocks * temporary.f_frsize + response["tmpMode"] = os.stat("/tmp").st_mode & 0o7777 + response["swapMax"] = (cgroup / "memory.swap.max").read_text().strip() + response["cpuMax"] = (cgroup / "cpu.max").read_text().strip() + response["capEff"] = status["CapEff"].strip() + response["noNewPrivs"] = status["NoNewPrivs"].strip() + response["taskRoot"] = os.environ.get("LAMBDA_TASK_ROOT") + response["functionName"] = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") + response["functionVersion"] = os.environ.get("AWS_LAMBDA_FUNCTION_VERSION") + response["region"] = os.environ.get("AWS_REGION") + response["configuredValue"] = os.environ.get("TEST_VALUE") return response From 832d39aaa24cf84010b7121eca05f2298fd0fda6 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 01:58:19 -0700 Subject: [PATCH 023/106] phase 1: retain environment lifecycle ownership through recovery Implement runtime section 4.3 and D31: retain allocation budgets, immutable plans, container mutation receipts and IO cleanup above supervised environment workers. Resolve and independently recheck W68-W77, including ready-reply abandonment and worker-failure precedence. Workspace build/test/fmt/Clippy/rustdoc pass with 2,215 tests; six-library strict boundary checks pass with 991 unit/doc tests. Audit and deny pass. Owner, wire and shared-runtime Docker suites pass on arm64 VZ and amd64/QEMU, with no owned resources afterward. Warm pool/executor integration, delivery-based handler timeout, historical reconciliation, pulls/network/probes and P2-P5 remain required; M0 is not closed. --- Makefile | 13 + .../src/docker/connection.rs | 4 + .../src/docker/environment/actor.rs | 463 ++++++++++++ .../src/docker/environment/cleanup.rs | 223 ++++++ .../src/docker/environment/io.rs | 216 ++++++ .../src/docker/environment/mod.rs | 585 ++++++++++++++++ .../src/docker/environment/tests.rs | 656 ++++++++++++++++++ .../src/docker/environment/worker.rs | 319 +++++++++ crates/rustack-lambda-core/src/docker/logs.rs | 2 +- crates/rustack-lambda-core/src/docker/mod.rs | 37 +- .../src/docker/mutation.rs | 31 + crates/rustack-lambda-core/src/docker/plan.rs | 78 ++- .../src/docker/plan_tests.rs | 6 +- .../rustack-lambda-core/src/docker/tests.rs | 2 + .../tests/docker_environment.rs | 277 ++++++++ crates/rustack-lambda-runtime/src/client.rs | 4 +- .../src/client/tests.rs | 36 + .../rustack-lambda-runtime/src/transport.rs | 20 +- specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 34 + .../rustack-workflows-60-operations-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 4 + specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 23 + specs/rustack-workflows-99-decisions.md | 9 + 25 files changed, 3026 insertions(+), 27 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/environment/actor.rs create mode 100644 crates/rustack-lambda-core/src/docker/environment/cleanup.rs create mode 100644 crates/rustack-lambda-core/src/docker/environment/io.rs create mode 100644 crates/rustack-lambda-core/src/docker/environment/mod.rs create mode 100644 crates/rustack-lambda-core/src/docker/environment/tests.rs create mode 100644 crates/rustack-lambda-core/src/docker/environment/worker.rs create mode 100644 crates/rustack-lambda-core/src/docker/mutation.rs create mode 100644 crates/rustack-lambda-core/tests/docker_environment.rs diff --git a/Makefile b/Makefile index b80ad339..2ee9fc00 100644 --- a/Makefile +++ b/Makefile @@ -876,3 +876,16 @@ test-iam-integration: test-iam-unit test-iam-integration .PHONY: smoke smoke-services smoke-regressions + +# Explicit real two-container environment lifecycle qualification. +test-lambda-environments-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_environment -- --ignored --test-threads=1 + +.PHONY: test-lambda-environments-docker + +# Pure ownership/failure regressions plus transport failure precedence. +test-lambda-environments: + @cargo test -p rustack-lambda-core --lib docker::environment::tests + @cargo test -p rustack-lambda-runtime --lib test_should_preserve_transport_worker_failure_after_a_prior_scope_error + +.PHONY: test-lambda-environments diff --git a/crates/rustack-lambda-core/src/docker/connection.rs b/crates/rustack-lambda-core/src/docker/connection.rs index b4c16f04..c77c7f4b 100644 --- a/crates/rustack-lambda-core/src/docker/connection.rs +++ b/crates/rustack-lambda-core/src/docker/connection.rs @@ -68,6 +68,7 @@ pub(super) async fn exchange( .with_subscriber(runtime_dispatch()), ); let operation = async { + let mut mutation = request.mutation; let mut builder = Request::builder() .method(request.method) .uri(request.target) @@ -83,6 +84,9 @@ pub(super) async fn exchange( let request = builder .body(Full::new(request.body)) .map_err(|_| DockerError::Protocol)?; + if let Some(mutation) = &mut mutation { + mutation.issued(); + } let response = sender.send_request(request).await?; read_response(response, expected, deadline).await }; diff --git a/crates/rustack-lambda-core/src/docker/environment/actor.rs b/crates/rustack-lambda-core/src/docker/environment/actor.rs new file mode 100644 index 00000000..c8c90006 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/actor.rs @@ -0,0 +1,463 @@ +//! Root registry owns reservations above workers and their recovery jobs. + +use std::{ + collections::HashMap, + mem, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use rustack_lambda_runtime::{diagnostics::runtime_dispatch, protocol::EnvironmentId}; +use tokio::{ + runtime::Handle, + sync::{OwnedSemaphorePermit, Semaphore, mpsc, watch}, + task::{Id, JoinError, JoinSet}, + time::{Instant, sleep}, +}; +use tracing::instrument::WithSubscriber; +use typed_builder::TypedBuilder; + +use super::{ + Call, EnvironmentConfig, EnvironmentError, EnvironmentPhase, EnvironmentStatus, + cleanup::{Cleaner, Progress}, + worker::Worker, +}; +use crate::{ + docker::{DockerPort, EnvironmentPlan}, + executor::ExecutorError, + invocation::CleanupTask, +}; + +pub(super) struct Health { + control: watch::Sender, + pub(super) failed: AtomicBool, + pub(super) quarantined: AtomicUsize, +} +impl Default for Health { + fn default() -> Self { + Self::new(watch::channel(false).0) + } +} +impl Health { + pub(super) fn new(control: watch::Sender) -> Self { + Self { + control, + failed: AtomicBool::new(false), + quarantined: AtomicUsize::new(0), + } + } + pub(super) fn fail(&self) { + self.failed.store(true, Ordering::Release); + self.control.send_replace(true); + } +} +pub(super) struct Budget { + capacity: Arc, + maximum: usize, + pub(super) memory: AtomicU64, + limit: u64, +} +impl Budget { + pub(super) fn new(config: EnvironmentConfig) -> Self { + let maximum = usize::from(config.0.environments); + Self { + capacity: Arc::new(Semaphore::new(maximum)), + maximum, + memory: AtomicU64::new(0), + limit: config.0.memory_bytes, + } + } + pub(super) fn count(&self) -> usize { + self.maximum + .saturating_sub(self.capacity.available_permits()) + } + pub(super) fn reserve(self: &Arc, memory: u64) -> Result { + let permit = self + .capacity + .clone() + .try_acquire_owned() + .map_err(|_| EnvironmentError::Busy)?; + self.memory + .try_update(Ordering::AcqRel, Ordering::Acquire, |current| { + current + .checked_add(memory) + .filter(|total| *total <= self.limit) + }) + .map_err(|_| EnvironmentError::Busy)?; + Ok(Reservation { + _permit: permit, + memory, + budget: self.clone(), + }) + } +} +pub(super) struct Reservation { + _permit: OwnedSemaphorePermit, + memory: u64, + budget: Arc, +} +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.memory.fetch_sub(self.memory, Ordering::AcqRel); + } +} +#[derive(Clone)] +pub(super) struct Signals { + pub(super) status: watch::Sender, + pub(super) stop: watch::Sender, +} +impl Signals { + pub(super) fn new() -> Self { + Self { + status: watch::channel(EnvironmentStatus { + phase: EnvironmentPhase::Allocating, + error: None, + }) + .0, + stop: watch::channel(false).0, + } + } + pub(super) fn phase(&self, phase: EnvironmentPhase) { + self.status.send_modify(|status| status.phase = phase); + } + pub(super) fn error(&self, error: EnvironmentError) { + self.status.send_modify(|status| { + if status.error.is_none() { + status.error = Some(error); + } + }); + } + fn worker_failed(&self) { + self.stop.send_replace(true); + self.status.send_modify(|status| { + status.phase = EnvironmentPhase::Retiring; + status.error = Some(EnvironmentError::Worker); + }); + } +} +#[derive(TypedBuilder)] +pub(super) struct Envelope { + pub(super) plan: Arc, + pub(super) mailbox: mpsc::Receiver, + pub(super) signals: Signals, + pub(super) reservation: Reservation, + pub(super) done: watch::Sender>>, + pub(super) deadline: Instant, +} +pub(super) struct Record { + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) io: Arc, + pub(super) signals: Signals, + pub(super) done: watch::Sender>>, + _reservation: Reservation, +} +impl Record { + fn recovery(&self) -> Recovery { + Recovery { + plan: self.plan.clone(), + progress: self.progress.clone(), + io: self.io.clone(), + signals: self.signals.clone(), + done: self.done.clone(), + } + } +} +pub(super) struct Recovery { + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) io: Arc, + pub(super) signals: Signals, + pub(super) done: watch::Sender>>, +} +impl Recovery { + async fn run( + self, + docker: DockerPort, + config: EnvironmentConfig, + health: Arc, + cleanup: Arc, + ) { + let cleaner = Cleaner::builder() + .docker(docker) + .config(config) + .plan(self.plan) + .progress(self.progress) + .health(health) + .cleanup(cleanup) + .signals(self.signals.clone()) + .build(); + cleaner.retire(&self.io, None).await; + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Err(EnvironmentError::Worker))); + } +} + +type Job = (EnvironmentId, Result<(), EnvironmentError>); +type Joined = Result<(Id, Job), JoinError>; +#[derive(TypedBuilder)] +pub(super) struct Actor { + docker: DockerPort, + config: EnvironmentConfig, + inbox: mpsc::Receiver, + control: watch::Sender, + health: Arc, + runtime: Handle, + emergency: Arc, + #[builder(default = Arc::new(Semaphore::new(2)))] + cleanup: Arc, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + records: HashMap, + #[builder(default)] + index: HashMap, + #[builder(default)] + settled: bool, +} +impl Actor { + pub(super) async fn run(mut self) -> Result<(), EnvironmentError> { + let mut stopped = self.control.subscribe(); + let mut wire = self.docker.raw.stop.subscribe(); + loop { + if *stopped.borrow() { + self.inbox.close(); + while let Ok(envelope) = self.inbox.try_recv() { + let failed = self.health.failed.load(Ordering::Acquire); + envelope.signals.error(if failed { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + }); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope.done.send_replace(Some(if failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + for record in self.records.values() { + if self.health.failed.load(Ordering::Acquire) { + record.signals.worker_failed(); + } else { + record.signals.stop.send_replace(true); + } + } + if self.jobs.is_empty() { + break; + } + if let Some(result) = self.jobs.join_next_with_id().await { + self.finished(&result); + } + continue; + } + tokio::select! {biased; + _=wire.wait_for(|stop|*stop)=>{self.fail();}, + _=stopped.wait_for(|stop|*stop)=>{}, + result=self.jobs.join_next_with_id(),if !self.jobs.is_empty()=>if let Some(result)=result{self.finished(&result);}, + envelope=self.inbox.recv()=>match envelope{Some(envelope)=>self.start(envelope),None=>{self.control.send_replace(true);}}, + } + } + self.settled = true; + if self.health.failed.load(Ordering::Acquire) { + Err(EnvironmentError::Worker) + } else { + Ok(()) + } + } + fn start(&mut self, envelope: Envelope) { + if *self.control.borrow() + || *envelope.signals.stop.borrow() + || Instant::now() >= envelope.deadline + { + envelope.signals.error(EnvironmentError::Stopped); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope.done.send_replace(Some(Ok(()))); + return; + } + let id = envelope.plan.environment_id(); + if self.records.contains_key(&id) { + self.fail(); + envelope.signals.error(EnvironmentError::Worker); + envelope + .done + .send_replace(Some(Err(EnvironmentError::Worker))); + return; + } + let progress = Arc::new(Progress::new()); + let io = Arc::new(CleanupTask::default()); + let worker = Worker::builder() + .docker(self.docker.clone()) + .config(self.config) + .plan(envelope.plan.clone()) + .progress(progress.clone()) + .cleanup(self.cleanup.clone()) + .health(self.health.clone()) + .signals(envelope.signals.clone()) + .global(self.control.clone()) + .io(io.clone()) + .runtime(self.runtime.clone()) + .build(); + self.records.insert( + id, + Record { + plan: envelope.plan, + progress, + io, + signals: envelope.signals, + done: envelope.done, + _reservation: envelope.reservation, + }, + ); + let job = self.jobs.spawn( + async move { (id, worker.run(envelope.mailbox, envelope.deadline).await) } + .with_subscriber(runtime_dispatch()), + ); + self.index.insert(job.id(), (id, false)); + } + fn finished(&mut self, result: &Joined) { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some((id, recovering)) = self.index.remove(&task) else { + self.fail(); + return; + }; + let clean = self + .records + .get(&id) + .is_some_and(|record| record.progress.clean()); + if matches!(&result,Ok((_,(returned,Ok(())))) if *returned==id) && clean { + if let Some(record) = self.records.remove(&id) { + let failed = recovering || self.health.failed.load(Ordering::Acquire); + if failed { + record.signals.worker_failed(); + } + record.signals.phase(EnvironmentPhase::Removed); + record.done.send_replace(Some(if failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + return; + } + self.fail(); + let Some(record) = self.records.get(&id) else { + return; + }; + record.signals.worker_failed(); + let recovery = record.recovery(); + let docker = self.docker.clone(); + let config = self.config; + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + let job = self.jobs.spawn(async move { + if recovering { + sleep(Duration::from_secs(1)).await; + } + recovery.run(docker, config, health, cleanup).await; + (id, Ok(())) + }); + self.index.insert(job.id(), (id, true)); + } + fn fail(&self) { + self.health.fail(); + self.control.send_replace(true); + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + while let Ok(envelope) = self.inbox.try_recv() { + envelope.signals.worker_failed(); + envelope.signals.phase(EnvironmentPhase::Removed); + envelope + .done + .send_replace(Some(Err(EnvironmentError::Worker))); + } + if self.records.is_empty() { + return; + } + self.fail(); + for record in self.records.values() { + record.signals.worker_failed(); + } + let records = mem::take(&mut self.records); + let mut jobs = mem::take(&mut self.jobs); + jobs.abort_all(); + let docker = self.docker.clone(); + let config = self.config; + let health = self.health.clone(); + let cleanup = self.cleanup.clone(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover_all(records, docker, config, health, cleanup).await + }); + } +} +async fn recover_all( + mut records: HashMap, + docker: DockerPort, + config: EnvironmentConfig, + health: Arc, + cleanup: Arc, +) -> Result<(), ExecutorError> { + let mut jobs = JoinSet::new(); + let mut index = HashMap::new(); + for (id, record) in &records { + let id = *id; + let recovery = record.recovery(); + let docker = docker.clone(); + let health = health.clone(); + let cleanup = cleanup.clone(); + let job = jobs.spawn(async move { + recovery.run(docker, config, health, cleanup).await; + id + }); + index.insert(job.id(), id); + } + while let Some(result) = jobs.join_next_with_id().await { + let task = match &result { + Ok((task, _)) => *task, + Err(error) => error.id(), + }; + let Some(id) = index.remove(&task) else { + continue; + }; + if matches!(result,Ok((_,returned)) if returned==id) + && records + .get(&id) + .is_some_and(|record| record.progress.clean()) + { + records.remove(&id); + continue; + } + let Some(record) = records.get(&id) else { + continue; + }; + let recovery = record.recovery(); + let docker = docker.clone(); + let health = health.clone(); + let cleanup = cleanup.clone(); + let job = jobs.spawn(async move { + sleep(Duration::from_secs(1)).await; + recovery.run(docker, config, health, cleanup).await; + id + }); + index.insert(job.id(), id); + } + if records.is_empty() { + Ok(()) + } else { + Err(ExecutorError::CleanupFailed( + "Environment recovery lost ownership".into(), + )) + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/cleanup.rs b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs new file mode 100644 index 00000000..a8131a77 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs @@ -0,0 +1,223 @@ +//! Phase-aware ID-only cleanup; uncertain create absence never grants settlement. + +use std::{ + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use http::StatusCode; +use tokio::{ + sync::{Semaphore, watch}, + time::sleep, +}; +use typed_builder::TypedBuilder; + +use super::{ + EnvironmentConfig, EnvironmentError, EnvironmentPhase, + actor::{Health, Signals}, + io::IoGuard, +}; +use crate::{ + docker::{ + DockerError, DockerKey, DockerPort, EnvironmentPlan, models::ContainerInspectResponse, + mutation::MutationReceipt, plan::ContainerRole, + }, + invocation::CleanupTask, +}; + +#[derive(Debug, Clone)] +pub(super) enum Phase { + Unissued, + Uncertain(MutationReceipt), + Known(DockerKey), + Clean, +} +#[derive(Debug)] +pub(super) struct Progress { + pub(super) relay: watch::Sender, + pub(super) function: watch::Sender, +} +impl Progress { + pub(super) fn new() -> Self { + Self { + relay: watch::channel(Phase::Unissued).0, + function: watch::channel(Phase::Unissued).0, + } + } + pub(super) fn phase(&self, role: ContainerRole) -> &watch::Sender { + match role { + ContainerRole::Relay => &self.relay, + ContainerRole::Function => &self.function, + } + } + pub(super) fn clean(&self) -> bool { + [&self.relay, &self.function] + .iter() + .all(|phase| matches!(*phase.borrow(), Phase::Unissued | Phase::Clean)) + } +} + +#[derive(TypedBuilder)] +pub(super) struct Cleaner { + pub(super) docker: DockerPort, + pub(super) config: EnvironmentConfig, + pub(super) plan: Arc, + pub(super) progress: Arc, + pub(super) health: Arc, + pub(super) cleanup: Arc, + pub(super) signals: Signals, +} +impl Cleaner { + pub(super) async fn container(&self, role: ContainerRole) -> bool { + let phase = self.progress.phase(role).borrow().clone(); + let id = match phase { + Phase::Unissued | Phase::Clean => return true, + Phase::Known(id) => id, + Phase::Uncertain(receipt) => { + if receipt.definitely_unissued() { + self.progress.phase(role).send_replace(Phase::Clean); + return true; + } + + let Ok(inspected) = self + .docker + .inspect_container(self.plan.key(role), self.config.deadline()) + .await + else { + return false; + }; + let Ok(id) = self.plan.owned(role, &inspected) else { + return false; + }; + self.progress + .phase(role) + .send_replace(Phase::Known(id.clone())); + id + } + }; + let Ok(_permit) = self.cleanup.acquire().await else { + return false; + }; + let inspected = match self + .docker + .inspect_container(&id, self.config.deadline()) + .await + { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + self.progress.phase(role).send_replace(Phase::Clean); + return true; + } + Ok(inspected) => inspected, + Err(_) => return false, + }; + if self.plan.owned(role, &inspected).is_err() + || inspected.id.as_deref() != Some(id.as_str()) + { + return false; + } + let Some(state) = inspected.state else { + return false; + }; + match state.running { + Some(true) => { + let _ = self + .docker + .kill_container(&id, self.config.deadline()) + .await; + let waited = self + .docker + .wait_container(&id, self.config.deadline()) + .await; + if !matches!(waited,Ok(ref exit) if exit.error.is_none()) { + return false; + } + } + Some(false) => {} + None => return false, + } + let _ = self + .docker + .remove_container(&id, self.config.deadline()) + .await; + if matches!( + self.docker + .inspect_container(&id, self.config.deadline()) + .await, + Err(DockerError::Status(StatusCode::NOT_FOUND)) + ) { + self.progress.phase(role).send_replace(Phase::Clean); + true + } else { + false + } + } + pub(super) async fn retire(&self, io: &CleanupTask, mut streams: Option<&mut IoGuard>) { + let mut dirty = None; + let mut delay = Duration::from_secs(1); + loop { + let function = self.container(ContainerRole::Function).await; + if function { + if let Some(streams) = streams.take() { + streams.begin_cleanup(); + } + } + let joined = if function { io.wait().await } else { Ok(()) }; + let relay = if function { + self.container(ContainerRole::Relay).await + } else { + false + }; + if function && relay && joined.is_ok() { + return; + } + if joined.is_err() { + self.health.fail(); + } + if dirty.is_none() { + dirty = Some(Dirty::new(self.health.clone())); + } + self.signals.phase(EnvironmentPhase::Quarantined); + sleep(delay).await; + delay = delay.saturating_mul(2).min(Duration::from_secs(30)); + } + } + pub(super) async fn inspect_health(&self) -> Result<(), EnvironmentError> { + let phase = self.progress.function.borrow().clone(); + let Phase::Known(id) = phase else { + return Err(EnvironmentError::Stopped); + }; + let inspected = self + .docker + .inspect_container(&id, self.config.deadline()) + .await?; + if self.plan.owned(ContainerRole::Function, &inspected)? != id { + return Err(DockerError::Quarantined.into()); + } + function_health(&inspected) + } +} +pub(super) fn function_health( + inspected: &ContainerInspectResponse, +) -> Result<(), EnvironmentError> { + let state = inspected.state.as_ref().ok_or(DockerError::Protocol)?; + match state.running { + Some(true) => Ok(()), + Some(false) => Err(EnvironmentError::Exited { + exit_code: state.exit_code.ok_or(DockerError::Protocol)?, + oom: state.oom_killed.ok_or(DockerError::Protocol)?, + }), + None => Err(DockerError::Protocol.into()), + } +} +struct Dirty(Arc); +impl Dirty { + fn new(health: Arc) -> Self { + health.quarantined.fetch_add(1, Ordering::AcqRel); + Self(health) + } +} +impl Drop for Dirty { + fn drop(&mut self) { + self.0.quarantined.fetch_sub(1, Ordering::AcqRel); + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/io.rs b/crates/rustack-lambda-core/src/docker/environment/io.rs new file mode 100644 index 00000000..b5bdc871 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/io.rs @@ -0,0 +1,216 @@ +//! Worker-owned streams transfer to retained cleanup before their guard drops. + +use std::{collections::VecDeque, mem::take, sync::Arc, time::Duration}; + +use bytes::Bytes; +use rustack_lambda_runtime::{ + client::{ClientConfig, ControlClient}, + diagnostics::runtime_dispatch, + protocol::{Architecture, EnvironmentId}, + relay::RelayError, +}; +use tokio::{ + io::{AsyncWriteExt, duplex}, + runtime::Handle, + sync::watch, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use tracing::instrument::WithSubscriber; + +use super::{EnvironmentConfig, EnvironmentError, actor::Signals}; +use crate::{ + docker::{ + DockerAttachment, DockerError, FunctionLogs, OutputChannel, + connection::stopped as wait_for_stop, + }, + executor::ExecutorError, + invocation::CleanupTask, +}; + +struct Bridge { + stop: watch::Sender, + task: JoinHandle>, + status: watch::Receiver>>, +} +#[derive(Default)] +struct Streams { + client: Option, + logs: Option, + bridge: Option, +} +pub(super) struct IoGuard { + streams: Streams, + runtime: Handle, + cleanup: Arc, + config: EnvironmentConfig, + signals: Signals, + closing: bool, + logs_closed: bool, +} +impl IoGuard { + #[cfg(test)] + pub(super) fn test_streams(&mut self, client: ControlClient, logs: FunctionLogs) { + self.streams.client = Some(client); + self.streams.logs = Some(logs); + } + + pub(super) fn new( + runtime: Handle, + cleanup: Arc, + config: EnvironmentConfig, + signals: Signals, + ) -> Self { + Self { + streams: Streams::default(), + runtime, + cleanup, + config, + signals, + closing: false, + logs_closed: false, + } + } + pub(super) fn control( + &mut self, + attachment: DockerAttachment, + environment: EnvironmentId, + architecture: Architecture, + ) -> Result<(), EnvironmentError> { + let (mut output, input) = attachment.split(self.config.0.io)?; + let (mut bridge, reader) = duplex(64 * 1024); + self.streams.client = Some(ControlClient::start( + ClientConfig::new(environment, architecture, self.config.0.io)?, + reader, + input, + )?); + let (stop, mut stopped) = watch::channel(false); + let (status, observed) = watch::channel(None); + let budget = self.config.0.io; + let mut diagnostics = VecDeque::new(); + diagnostics + .try_reserve_exact(65536) + .map_err(|_| DockerError::Allocation)?; + let task=self.runtime.spawn(async move{ + let result=async{ + loop{ + let chunk=tokio::select!{biased;()=wait_for_stop(&mut stopped)=>return Ok(()),chunk=output.next(Instant::now()+Duration::from_secs(3600))=>chunk?,}; + match chunk{ + None=>return Ok(()), + Some(chunk) if chunk.channel==OutputChannel::Stdout=>{timeout_at(Instant::now()+budget,bridge.write_all(&chunk.bytes)).await.map_err(|_|DockerError::Deadline)??;}, + Some(chunk)=>{let remove=diagnostics.len().saturating_add(chunk.bytes.len()).saturating_sub(65536);diagnostics.drain(..remove.min(diagnostics.len()));diagnostics.extend(chunk.bytes.iter().copied());}, + } + } + }.await;status.send_replace(Some(result.clone()));result + }.with_subscriber(runtime_dispatch())); + self.streams.bridge = Some(Bridge { + stop, + task, + status: observed, + }); + Ok(()) + } + pub(super) fn client(&mut self) -> Result<&mut ControlClient, EnvironmentError> { + self.streams.client.as_mut().ok_or(EnvironmentError::Worker) + } + pub(super) fn logs(&mut self, attachment: DockerAttachment) -> Result<(), EnvironmentError> { + self.streams.logs = Some(FunctionLogs::start( + attachment, + self.config.0.io, + self.config.0.log_ring_bytes, + )?); + Ok(()) + } + pub(super) async fn tail(&mut self) -> Result { + let logs = self.streams.logs.as_ref().ok_or(EnvironmentError::Worker)?; + let snapshot = logs.snapshot(self.config.deadline()).await?; + if let Some(error) = snapshot.failure { + return Err(error.into()); + } + self.logs_closed = snapshot.closed; + Ok(snapshot.tail) + } + pub(super) async fn healthy(&mut self) -> Result<(), EnvironmentError> { + if let Some(bridge) = &self.streams.bridge { + if let Some(result) = bridge.status.borrow().as_ref() { + return match result { + Ok(()) => Err(DockerError::Protocol.into()), + Err(error) => Err(error.clone().into()), + }; + } + if bridge.task.is_finished() && bridge.status.borrow().is_none() { + return Err(EnvironmentError::Worker); + } + } + self.tail().await?; + if self.logs_closed { + Err(DockerError::Protocol.into()) + } else { + Ok(()) + } + } + pub(super) fn reusable(&self) -> bool { + !self.logs_closed + && self + .streams + .bridge + .as_ref() + .is_none_or(|bridge| bridge.status.borrow().is_none()) + } + pub(super) fn begin_cleanup(&mut self) { + if self.closing { + return; + } + self.closing = true; + let streams = take(&mut self.streams); + let config = self.config; + let signals = self.signals.clone(); + self.cleanup.start_on(&self.runtime, move || async move { + close(streams, config, signals).await + }); + } +} +impl Drop for IoGuard { + fn drop(&mut self) { + self.begin_cleanup(); + } +} +async fn close( + mut streams: Streams, + config: EnvironmentConfig, + signals: Signals, +) -> Result<(), ExecutorError> { + let mut failed = false; + if let Some(client) = &mut streams.client { + if let Err(error) = client.shutdown(config.deadline()).await { + failed |= matches!(error, RelayError::Worker); + signals.error(error.into()); + } + } + if let Some(bridge) = &streams.bridge { + bridge.stop.send_replace(true); + } + if let Some(logs) = &mut streams.logs { + if let Err(error) = logs.shutdown().await { + failed |= matches!(error, DockerError::Worker); + signals.error(error.into()); + } + } + if let Some(bridge) = streams.bridge { + match bridge.task.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + signals.error(error.into()); + } + Err(_) => failed = true, + } + } + if failed { + signals.error(EnvironmentError::Worker); + Err(ExecutorError::CleanupFailed( + "Environment IO supervision failed".into(), + )) + } else { + Ok(()) + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs new file mode 100644 index 00000000..f6c48dee --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -0,0 +1,585 @@ +//! Retained two-container environments; executor pooling and startup readiness are separate. + +mod actor; +mod cleanup; +mod io; +mod worker; + +use std::{ + fmt, + sync::{Arc, atomic::Ordering}, + time::Duration, +}; + +use actor::{Actor, Budget, Envelope, Health, Signals}; +use bytes::Bytes; +use rustack_lambda_runtime::{ + protocol::{EnvironmentId, Payload}, + relay::RelayError, + session::{Invocation, RuntimeResult}, +}; +use tokio::{ + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{DirectoryLease, DockerError, DockerPort, EnvironmentPlan, check_deadline}; +use crate::invocation::CleanupTask; + +/// Raw operator budgets; convert to `EnvironmentConfig` before starting ownership. +#[derive(Debug, Clone, Copy, TypedBuilder)] +#[non_exhaustive] +pub struct EnvironmentSettings { + /// Count of allocated environments, including quarantine, in1..32. + #[builder(default = 32)] + pub environments: u8, + /// Aggregate function+relay memory,256MiB..1TiB. + #[builder(default = 8 * 1024 * 1024 * 1024)] + pub memory_bytes: u64, + /// Independent Docker/control IO budget,1..60s. + #[builder(default = Duration::from_secs(5))] + pub io: Duration, + /// Cold startup budget,1..60s, also bounded by caller admission deadline. + #[builder(default = Duration::from_secs(15))] + pub start: Duration, + /// Idle lifetime,1..86400s. + #[builder(default = Duration::from_secs(600))] + pub idle: Duration, + /// Retained bytes per function-output channel,4KiB..1MiB. + #[builder(default = 262_144)] + pub log_ring_bytes: usize, +} +impl Default for EnvironmentSettings { + fn default() -> Self { + Self::builder().build() + } +} + +/// Closed, validated environment ownership budgets. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct EnvironmentConfig(EnvironmentSettings); +impl TryFrom for EnvironmentConfig { + type Error = EnvironmentError; + fn try_from(settings: EnvironmentSettings) -> Result { + if !(1..=32).contains(&settings.environments) + || !(256 * 1024 * 1024..=1024 * 1024 * 1024 * 1024).contains(&settings.memory_bytes) + || !(4096..=1024 * 1024).contains(&settings.log_ring_bytes) + { + return Err(EnvironmentError::Configuration); + } + for (duration, maximum) in [ + (settings.io, 60), + (settings.start, 60), + (settings.idle, 86400), + ] { + if duration.subsec_nanos() != 0 || !(1..=maximum).contains(&duration.as_secs()) { + return Err(EnvironmentError::Configuration); + } + } + Ok(Self(settings)) + } +} +impl EnvironmentConfig { + fn deadline(self) -> Instant { + Instant::now() + self.0.io + } +} + +/// Environment errors retain typed sources and never display peer payload bytes. +#[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] +pub enum EnvironmentError { + /// Invalid settings, scope, captured invocation or lifecycle operation. + #[error("invalid environment configuration")] + Configuration, + /// Allocation or command capacity is exhausted. + #[error("environment capacity exhausted")] + Busy, + /// Explicit cancellation or abandoned owner/caller. + #[error("environment cancelled")] + Cancelled, + /// Startup, call or waiting deadline expired. + #[error("environment deadline exceeded")] + Deadline, + /// This owner no longer accepts work. + #[error("environment is stopped")] + Stopped, + /// A worker/root/cleanup supervisor failed. + #[error("environment supervision failed")] + Worker, + /// Function reported a bounded initialization error. + #[error("function initialization failed")] + Initialization(Payload), + /// Function exited before its expected result/readiness. + #[error("function exited with status {exit_code}; oom={oom}")] + Exited { + /// Daemon exit status, without peer text. + exit_code: i64, + /// Daemon-reported OOM kill flag. + oom: bool, + }, + /// Validated local Docker transport failure. + #[error(transparent)] + Docker(#[from] DockerError), + /// Ordered relay session failure. + #[error(transparent)] + Relay(#[from] RelayError), +} + +/// Current ownership/lifecycle state, independent of external API readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum EnvironmentPhase { + /// An allocation is registered and creating its relay. + Allocating, + /// Relay/function initialization is in progress. + Initializing, + /// Ready for one invocation. + Idle, + /// An admitted invocation is active. + Busy, + /// All owned resources are being retired. + Retiring, + /// Failed/uncertain cleanup retains its allocation reservation. + Quarantined, + /// Containers and retained IO are confirmed settled. + Removed, +} +/// Redacted lifecycle projection; a business failure does not imply unowned cleanup. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EnvironmentStatus { + /// Current owner state. + pub phase: EnvironmentPhase, + /// Primary failure, retained across cleanup retries. + pub error: Option, +} +/// Allocation accounting includes every queued, live and quarantined reservation. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct EnvironmentCapacity { + /// Number of retained allocation slots. + pub allocated: usize, + /// Aggregate reserved memory bytes, including relay overhead. + pub memory_bytes: u64, + /// Environments retaining failed cleanup. + pub quarantined: usize, + /// Whether admission is open and healthy. + pub accepting: bool, + /// A supervision failure permanently closed admission. + pub worker_failed: bool, +} +/// Actual handler result and bounded optional log output; Debug redacts user bytes. +#[derive(Clone)] +#[non_exhaustive] +pub struct EnvironmentResponse { + /// Unmodified handler success or `FunctionError` payload. + pub result: RuntimeResult, + /// Combined captured final 4 KiB when requested; per-invocation attribution is separate. + pub logs: Bytes, +} +impl fmt::Debug for EnvironmentResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentResponse") + .field("result", &self.result) + .field("log_bytes", &self.logs.len()) + .finish() + } +} +struct Call { + invocation: Invocation, + capture_logs: bool, + cancellation: watch::Receiver, + reply: oneshot::Sender>, +} +struct CancelOnDrop { + call: watch::Sender, + environment: watch::Sender, + complete: bool, +} +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if !self.complete { + self.call.send_replace(true); + self.environment.send_replace(true); + } + } +} + +/// Exclusive owner-facing environment handle; Drop requests retained retirement. +#[non_exhaustive] +pub struct EnvironmentHandle { + id: EnvironmentId, + calls: mpsc::Sender, + signals: Signals, + settlement: watch::Receiver>>, +} +impl fmt::Debug for EnvironmentHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentHandle") + .field("id", &self.id) + .field("status", &*self.signals.status.borrow()) + .finish_non_exhaustive() + } +} +impl EnvironmentHandle { + /// Read the fresh immutable environment identity. + /// + /// ```no_run + /// # fn id(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let id = environment.id(); } + /// ``` + #[must_use] + pub fn id(&self) -> EnvironmentId { + self.id + } + /// Read lifecycle status without waiting for a business mailbox. + /// + /// ```no_run + /// # fn status(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let status = environment.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> EnvironmentStatus { + self.signals.status.borrow().clone() + } + /// Request retirement through its priority stop signal. + /// + /// ```no_run + /// # fn stop(environment: &rustack_lambda_core::docker::EnvironmentHandle) { environment.cancel(); } + /// ``` + pub fn cancel(&self) { + self.signals.stop.send_replace(true); + } + /// Await readiness without cancelling an accepted allocation when dropped. + /// + /// # Errors + /// Returns invalid/expired budget, startup failure or owner failure. The owner + /// remains responsible for shutdown/Drop and retained cleanup. + /// + /// ```no_run + /// # async fn ready(environment: &rustack_lambda_core::docker::EnvironmentHandle) { let ready = environment.wait_ready(tokio::time::Instant::now()+std::time::Duration::from_secs(15)).await; } + /// ``` + pub async fn wait_ready(&self, deadline: Instant) -> Result<(), EnvironmentError> { + check_deadline(deadline)?; + let mut status = self.signals.status.subscribe(); + let observed = timeout_at( + deadline, + status.wait_for(|status| { + !matches!( + status.phase, + EnvironmentPhase::Allocating | EnvironmentPhase::Initializing + ) + }), + ) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)? + .clone(); + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if matches!( + observed.phase, + EnvironmentPhase::Idle | EnvironmentPhase::Busy + ) { + Ok(()) + } else { + Err(observed.error.unwrap_or(EnvironmentError::Stopped)) + } + } + /// Submit one runtime invocation to a ready environment, without replay. + /// + /// # Errors + /// Returns bounded admission, deadline/cancellation, protocol or function-exit + /// errors. Dropping this future retires the environment; shutdown separately + /// awaits cleanup. The supplied absolute invocation deadline is authoritative. + /// + /// ```no_run + /// # async fn invoke(environment: &mut rustack_lambda_core::docker::EnvironmentHandle, invocation: rustack_lambda_runtime::session::Invocation) { let result = environment.invoke(invocation, true).await; } + /// ``` + pub async fn invoke( + &mut self, + invocation: Invocation, + capture_logs: bool, + ) -> Result { + let deadline = invocation.parts().1; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if self.status().phase != EnvironmentPhase::Idle { + return Err(EnvironmentError::Stopped); + } + let (stop, cancellation) = watch::channel(false); + let mut cancel = CancelOnDrop { + call: stop, + environment: self.signals.stop.clone(), + complete: false, + }; + let (reply, response) = oneshot::channel(); + self.calls + .try_send(Call { + invocation, + capture_logs, + cancellation, + reply, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => EnvironmentError::Stopped, + })?; + let result = timeout_at(deadline, response) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + cancel.complete = result.is_ok(); + result + } + /// Request stop and await retained container/IO settlement under one deadline. + /// + /// # Errors + /// Returns timeout while ownership remains retained, or cached supervision + /// failure. Business failure remains in status, independent of clean retirement. + /// + /// ```no_run + /// # async fn stop(environment: &mut rustack_lambda_core::docker::EnvironmentHandle) { let clean = environment.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.cancel(); + check_deadline(deadline)?; + let settled = timeout_at(deadline, self.settlement.wait_for(Option::is_some)) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)? + .clone() + .ok_or(EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + settled + } +} +impl Drop for EnvironmentHandle { + fn drop(&mut self) { + self.cancel(); + } +} + +/// Cloneable bounded allocation port; retain its manager through shutdown. +#[derive(Clone, TypedBuilder)] +#[builder( + builder_method(vis = ""), + build_method(vis = ""), + builder_type(vis = "") +)] +#[non_exhaustive] +pub struct EnvironmentPort { + sender: mpsc::Sender, + directory: DirectoryLease, + config: EnvironmentConfig, + control: watch::Sender, + budget: Arc, + health: Arc, +} +impl fmt::Debug for EnvironmentPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPort") + .field("capacity", &self.status()) + .finish_non_exhaustive() + } +} +impl EnvironmentPort { + /// Reserve count/memory and register a plan before any container operation. + /// + /// # Errors + /// Returns invalid scope/deadline, closed/unhealthy owner or bounded capacity. + /// Startup runs under the earlier of admission deadline and configured budget. + /// + /// ```no_run + /// # fn allocate(port: &rustack_lambda_core::docker::EnvironmentPort, plan: rustack_lambda_core::docker::EnvironmentPlan) { let environment = port.spawn(plan,tokio::time::Instant::now()+std::time::Duration::from_secs(15)); } + /// ``` + pub fn spawn( + &self, + plan: EnvironmentPlan, + deadline: Instant, + ) -> Result { + let deadline = deadline.min(Instant::now() + self.config.0.start); + check_deadline(deadline)?; + if !plan.same_scope(&self.directory) { + return Err(EnvironmentError::Configuration); + } + if *self.control.borrow() || self.health.failed.load(Ordering::Acquire) { + return Err(EnvironmentError::Stopped); + } + if self.health.quarantined.load(Ordering::Acquire) > 0 { + return Err(DockerError::Quarantined.into()); + } + let reservation = self.budget.reserve(plan.memory_bytes())?; + let plan = Arc::new(plan); + let (calls, mailbox) = mpsc::channel(4); + let signals = Signals::new(); + let (done, settlement) = watch::channel(None); + let handle = EnvironmentHandle { + id: plan.environment_id(), + calls, + signals: signals.clone(), + settlement, + }; + self.sender + .try_send(Envelope { + plan, + mailbox, + signals, + reservation, + done, + deadline: deadline.min(Instant::now() + self.config.0.start), + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => EnvironmentError::Stopped, + })?; + Ok(handle) + } + /// Read bounded resource accounting, including quarantine reservations. + /// + /// ```no_run + /// # fn status(port: &rustack_lambda_core::docker::EnvironmentPort) { let capacity = port.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> EnvironmentCapacity { + EnvironmentCapacity { + allocated: self.budget.count(), + memory_bytes: self.budget.memory.load(Ordering::Acquire), + quarantined: self.health.quarantined.load(Ordering::Acquire), + accepting: !*self.control.borrow() + && !self.health.failed.load(Ordering::Acquire) + && self.health.quarantined.load(Ordering::Acquire) == 0, + worker_failed: self.health.failed.load(Ordering::Acquire), + } + } +} + +/// Retained root owner; startup reconciliation/probes and executor pooling are separate. +#[non_exhaustive] +pub struct EnvironmentManager { + port: EnvironmentPort, + task: Option>>, + primary: Option>, + settled: Option>, + emergency: Arc, +} +impl fmt::Debug for EnvironmentManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentManager") + .field("port", &self.port) + .field("settled", &self.settled) + .finish_non_exhaustive() + } +} +impl EnvironmentManager { + /// Start ownership on the current runtime under a real directory capability. + /// + /// # Errors + /// Returns Worker when a Tokio runtime is unavailable. This alone does not + /// establish historical reconciliation or production Docker readiness. + /// + /// ```no_run + /// # fn start(docker: rustack_lambda_core::docker::DockerPort, lease: rustack_lambda_core::docker::DirectoryLease) { let manager = rustack_lambda_core::docker::EnvironmentManager::start(docker,lease,Default::default()); } + /// ``` + pub fn start( + docker: DockerPort, + directory: DirectoryLease, + config: EnvironmentConfig, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| EnvironmentError::Worker)?; + let (sender, inbox) = mpsc::channel(32); + let (control, _) = watch::channel(false); + let health = Arc::new(Health::new(control.clone())); + let budget = Arc::new(Budget::new(config)); + let emergency = Arc::new(CleanupTask::default()); + let port = EnvironmentPort { + sender, + directory, + config, + control: control.clone(), + budget, + health: health.clone(), + }; + let actor = Actor::builder() + .docker(docker) + .config(config) + .inbox(inbox) + .control(control) + .health(health) + .runtime(runtime.clone()) + .emergency(emergency.clone()) + .build(); + Ok(Self { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + }) + } + /// Clone the allocation port without transferring root shutdown ownership. + /// + /// ```no_run + /// # fn port(manager: &rustack_lambda_core::docker::EnvironmentManager) { let port = manager.port(); } + /// ``` + #[must_use] + pub fn port(&self) -> EnvironmentPort { + self.port.clone() + } + /// Stop admission and join normal/emergency cleanup without dropping records. + /// + /// # Errors + /// Returns timeout while cleanup stays owned, or cached worker/root failure. + /// Must finish before image owner or `DockerWire` shutdown. + /// + /// ```no_run + /// # async fn stop(manager: &mut rustack_lambda_core::docker::EnvironmentManager) { let result = manager.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(10)).await; } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.port.control.send_replace(true); + check_deadline(deadline)?; + if let Some(settled) = &self.settled { + return settled.clone(); + } + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker) + .and_then(|result| result), + None => Err(EnvironmentError::Worker), + }; + self.task.take(); + self.primary = Some(result); + } + timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + let result = self.primary.clone().ok_or(EnvironmentError::Worker)?; + self.settled = Some(result.clone()); + result + } +} +impl Drop for EnvironmentManager { + fn drop(&mut self) { + self.port.control.send_replace(true); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/environment/tests.rs b/crates/rustack-lambda-core/src/docker/environment/tests.rs new file mode 100644 index 00000000..ed8768fd --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/tests.rs @@ -0,0 +1,656 @@ +use serde_json::{Value, json}; +use tokio::time::timeout; + +use super::*; +use crate::docker::{ + plan::tests::{image_with, policy, revision}, + tests::Daemon, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn directory() -> DirectoryLease { + DirectoryLease::testing(uuid::Uuid::new_v4(), uuid::Uuid::new_v4()) +} +fn plan(directory: &DirectoryLease) -> EnvironmentPlan { + EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap() +} +fn inspected(plan: &EnvironmentPlan) -> Value { + let revision = plan.revision(); + let image = match &revision.artifact { + crate::artifact::DeploymentArtifact::Image(image) => { + image.resolved().metadata().image_id.as_str() + } + _ => panic!("image fixture"), + }; + json!({"Id":"a".repeat(64),"Name":format!("/{}",plan.relay_key().as_str()),"Config":{"Image":image,"Labels":{ + "rustack.lambda.ownerId":plan.directory.owner_id().to_string(),"rustack.lambda.runId":plan.directory.run_id().to_string(),"rustack.lambda.environmentId":plan.environment_id().to_string(),"rustack.lambda.functionResourceId":revision.resource_id.to_string(),"rustack.lambda.role":"relay"}},"State":{"Running":false,"OOMKilled":false,"ExitCode":0}}) +} +async fn quarantined(handle: &EnvironmentHandle) { + timeout(Duration::from_secs(3), async { + while handle.status().phase != EnvironmentPhase::Quarantined { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn finish_relay(daemon: &mut Daemon, owned: &Value) { + let observed = daemon.next().await; + assert!(observed.head.starts_with("GET ")); + observed + .reply(200, &serde_json::to_vec(owned).unwrap()) + .await; + let checked = daemon.next().await; + assert!(checked.head.contains(&"a".repeat(64))); + checked + .reply(200, &serde_json::to_vec(owned).unwrap()) + .await; + let removed = daemon.next().await; + assert!(removed.head.starts_with("DELETE ")); + removed.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; +} + +#[test] +fn test_should_validate_environment_budgets_and_reserve_count_and_memory_transactionally() { + for settings in [ + EnvironmentSettings::builder().environments(0).build(), + EnvironmentSettings::builder().environments(33).build(), + EnvironmentSettings::builder().memory_bytes(1).build(), + EnvironmentSettings::builder().io(Duration::ZERO).build(), + EnvironmentSettings::builder() + .start(Duration::from_secs(61)) + .build(), + EnvironmentSettings::builder() + .idle(Duration::from_millis(1500)) + .build(), + EnvironmentSettings::builder().log_ring_bytes(4095).build(), + ] { + assert!(EnvironmentConfig::try_from(settings).is_err()); + } + let config = EnvironmentConfig::try_from( + EnvironmentSettings::builder() + .environments(2) + .memory_bytes(256 * 1024 * 1024) + .build(), + ) + .unwrap(); + let budget = Arc::new(Budget::new(config)); + let first = budget.reserve(256 * 1024 * 1024).unwrap(); + assert!(budget.reserve(1).is_err()); + assert_eq!(budget.count(), 1); + drop(first); + assert_eq!(budget.count(), 0); + assert_eq!(budget.memory.load(Ordering::Acquire), 0); +} +#[tokio::test] +async fn test_should_reject_foreign_scope_without_admitting_or_mutating() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + assert!(matches!( + manager.port().spawn(plan(&directory()), deadline()), + Err(EnvironmentError::Configuration) + )); + assert_eq!(manager.port().status().allocated, 0); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_settle_definitive_create_rejection_without_containers() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + daemon.next().await.reply(400, b"{}").await; + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_retain_quota_after_500_and_initial_absence_until_late_create_is_removed() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + daemon.next().await.reply(500, b"{}").await; + daemon.next().await.reply(404, b"{}").await; + quarantined(&handle).await; + assert_eq!(manager.port().status().allocated, 1); + assert_eq!(manager.port().status().memory_bytes, 256 * 1024 * 1024); + assert!(matches!( + handle + .shutdown(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + finish_relay(&mut daemon, &owned).await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case("labels")] +#[case("image")] +#[case("name")] +#[case("id")] +#[tokio::test] +async fn test_should_quarantine_foreign_or_malformed_objects_without_deleting_them( + #[case] conflict: &str, +) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut foreign = owned.clone(); + match conflict { + "labels" => { + foreign["Config"]["Labels"]["rustack.lambda.ownerId"] = + json!(uuid::Uuid::new_v4().to_string()); + } + "image" => foreign["Config"]["Image"] = json!("sha256:foreign"), + "name" => foreign["Name"] = json!("/foreign"), + "id" => foreign["Id"] = json!("not-an-id"), + _ => panic!("case"), + } + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + daemon.next().await.reply(500, b"{}").await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&foreign).unwrap()) + .await; + quarantined(&handle).await; + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!(manager.port().status().allocated, 1); + finish_relay(&mut daemon, &owned).await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_finish_critical_create_after_caller_cancel_before_cleanup() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + let create = daemon.next().await; + handle.cancel(); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + assert_eq!(manager.port().status().allocated, 1); + create + .reply( + 201, + format!("{{\"Id\":\"{}\",\"Warnings\":[]}}", "a".repeat(64)).as_bytes(), + ) + .await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + daemon + .next() + .await + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + daemon.next().await.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_recover_retained_record_after_root_abort_with_create_in_flight() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = plan(&lease); + let owned = inspected(&planned); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + let create = daemon.next().await; + manager.task.as_ref().unwrap().abort(); + daemon.next().await.reply(404, b"{}").await; + assert_eq!(manager.port().status().allocated, 1); + create + .reply( + 201, + format!("{{\"Id\":\"{}\",\"Warnings\":[]}}", "a".repeat(64)).as_bytes(), + ) + .await; + finish_relay(&mut daemon, &owned).await; + assert!(matches!( + handle.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + manager.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + manager.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_settle_revoked_before_issue_without_any_daemon_mutation() { + use std::sync::atomic::AtomicBool; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let (release, _) = watch::channel(()); + let (stop, stopped) = watch::channel(false); + let relay = crate::artifact::PinnedImage::new( + image_with("1000", vec![]).resolved().clone(), + 4096.try_into().unwrap(), + release, + Arc::new(AtomicBool::new(true)), + stopped, + ); + let planned = + EnvironmentPlan::new(lease.clone(), relay, Arc::new(revision()), policy()).unwrap(); + stop.send_replace(true); + let mut manager = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let mut handle = manager.port().spawn(planned, deadline()).unwrap(); + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_release_unissued_allocation_when_socket_connection_fails() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + daemon.close().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + assert!(handle.wait_ready(deadline()).await.is_err()); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_should_admit_a_queued_call_after_health_io_takes_longer_than_its_cadence() { + use tokio::io::duplex; + + use crate::docker::{DockerOutputReader, FunctionLogs, plan::ContainerRole}; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let planned = Arc::new(plan(&lease)); + let environment: EnvironmentId = planned.environment_id(); + let mut owned = inspected(&planned); + owned["Name"] = json!(format!("/{}", planned.function_key().as_str())); + owned["Config"]["Labels"]["rustack.lambda.role"] = json!("function"); + owned["State"]["Running"] = json!(true); + let (client, peer) = ready_client(environment).await; + let (_quiet, reader) = duplex(8); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let io_task = Arc::new(CleanupTask::default()); + let signals = Signals::new(); + let health = Arc::new(Health::default()); + let config = EnvironmentConfig::default(); + let progress = Arc::new(cleanup::Progress::new()); + progress + .phase(ContainerRole::Function) + .send_replace(cleanup::Phase::Known("a".repeat(64).parse().unwrap())); + let cleanup = Arc::new(tokio::sync::Semaphore::new(2)); + let (global, _) = watch::channel(false); + let cleaner = cleanup::Cleaner::builder() + .docker(wire.port()) + .config(config) + .plan(planned.clone()) + .progress(progress.clone()) + .health(health.clone()) + .cleanup(cleanup.clone()) + .signals(signals.clone()) + .build(); + let worker = worker::Worker::builder() + .docker(wire.port()) + .config(config) + .plan(planned) + .progress(progress) + .cleanup(cleanup) + .health(health) + .signals(signals.clone()) + .global(global) + .io(io_task.clone()) + .runtime(Handle::current()) + .build(); + let mut io = io::IoGuard::new(Handle::current(), io_task.clone(), config, signals); + io.test_streams(client, logs); + let (calls, mut mailbox) = mpsc::channel(4); + let task = tokio::spawn(async move { + if let Some((reply, result)) = worker.serve_test(&mut io, &cleaner, &mut mailbox).await { + let _ = reply.send(result); + } + io.begin_cleanup(); + }); + let inspection = daemon.next().await; + let (stop, cancellation) = watch::channel(false); + let (reply, response) = oneshot::channel(); + calls + .send(Call { + invocation: foreign_invocation(), + capture_logs: false, + cancellation, + reply, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1100)).await; + inspection + .reply(200, &serde_json::to_vec(&owned).unwrap()) + .await; + assert!(matches!( + timeout(Duration::from_millis(500), response) + .await + .unwrap() + .unwrap(), + Err(EnvironmentError::Configuration) + )); + task.await.unwrap(); + io_task.wait().await.unwrap(); + peer.await.unwrap(); + drop(stop); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::normal(false, false)] +#[case::root_abort(true, false)] +#[case::wire_failure(true, true)] +#[tokio::test] +async fn test_should_keep_queued_handle_failure_distinct_from_clean_shutdown( + #[case] failed: bool, + #[case] wire_failure: bool, +) { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&lease), deadline()).unwrap(); + if wire_failure { + wire.port().raw.stop.send_replace(true); + } else if failed { + manager.task.as_ref().unwrap().abort(); + } else { + manager.port.control.send_replace(true); + } + let result = handle.shutdown(deadline()).await; + let joined = manager.shutdown(deadline()).await; + if failed { + assert!(matches!(result, Err(EnvironmentError::Worker))); + assert!(matches!(joined, Err(EnvironmentError::Worker))); + } else { + result.unwrap(); + joined.unwrap(); + } + assert_eq!(manager.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_allow_final_log_tail_at_eof_but_never_reuse_the_closed_environment() { + use rustack_lambda_runtime::{ + client::{ClientConfig, ControlClient}, + protocol::{Architecture, EnvironmentId}, + }; + + use crate::docker::{DockerOutputReader, FunctionLogs}; + let (reader, peer) = tokio::io::duplex(8); + drop(peer); + let client = ControlClient::start( + ClientConfig::new( + EnvironmentId::default(), + Architecture::Arm64, + Duration::from_secs(1), + ) + .unwrap(), + reader, + tokio::io::sink(), + ) + .unwrap(); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(b"".as_slice(), Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let cleanup = Arc::new(CleanupTask::default()); + let mut io = io::IoGuard::new( + Handle::current(), + cleanup.clone(), + EnvironmentConfig::default(), + Signals::new(), + ); + io.test_streams(client, logs); + timeout(Duration::from_secs(1), async { + loop { + assert!(io.tail().await.unwrap().is_empty()); + if !io.reusable() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(io.healthy().await.is_err()); + io.begin_cleanup(); + cleanup.wait().await.unwrap(); +} + +async fn ready_client( + environment: EnvironmentId, +) -> ( + rustack_lambda_runtime::client::ControlClient, + tokio::task::JoinHandle<()>, +) { + use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + protocol::{Acknowledgement, Architecture, Frame, Message, read_frame, write_frame}, + }; + use tokio::io::{duplex, split}; + let (stream, mut peer) = duplex(65536); + let (reader, writer) = split(stream); + let peer = tokio::spawn(async move { + for message in [ + Message::Hello { + architecture: Architecture::Arm64, + }, + Message::Ready, + Message::BootstrapReady, + ] { + write_frame(&mut peer, &Frame::new(environment, message), deadline()) + .await + .unwrap(); + } + let close = read_frame(&mut peer, deadline()).await.unwrap().unwrap(); + assert!(matches!(close.message(), Message::Close)); + write_frame( + &mut peer, + &Frame::new(environment, Message::Ack(Acknowledgement::Close)), + deadline(), + ) + .await + .unwrap(); + }); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + writer, + ) + .unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + (client, peer) +} + +fn foreign_invocation() -> rustack_lambda_runtime::session::Invocation { + use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, Payload}, + session::Invocation, + }; + Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:other" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn test_should_bound_long_caller_deadline_by_start_budget_before_admission_validation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut manager = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager + .port() + .spawn(plan(&lease), Instant::now() + Duration::from_mins(15)) + .unwrap(); + handle.cancel(); + handle.shutdown(deadline()).await.unwrap(); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::abandoned(false)] +#[case::consumed(true)] +#[tokio::test] +async fn test_should_retire_ready_reply_only_when_its_waiter_is_abandoned(#[case] consumed: bool) { + use std::{ + future::Future, + task::{Context, Waker}, + }; + + use rustack_lambda_runtime::protocol::Payload; + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (calls, mut mailbox) = mpsc::channel(4); + let (_done, settlement) = watch::channel(None); + let mut handle = EnvironmentHandle { + id: EnvironmentId::default(), + calls, + signals: signals.clone(), + settlement, + }; + { + let invoke = handle.invoke(foreign_invocation(), false); + tokio::pin!(invoke); + assert!( + invoke + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + let call = mailbox.recv().await.unwrap(); + call.reply + .send(Ok(EnvironmentResponse { + result: rustack_lambda_runtime::session::RuntimeResult::Success( + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ), + logs: Bytes::new(), + })) + .unwrap(); + if consumed { + assert!(matches!( + invoke + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + std::task::Poll::Ready(Ok(_)) + )); + } + } + assert_eq!( + *signals.stop.borrow(), + !consumed, + "only abandoned replies retire the environment" + ); +} diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs new file mode 100644 index 00000000..6706f2d8 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -0,0 +1,319 @@ +//! One ordered environment worker; all resources remain registered above its task. + +use std::{future::pending, sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::StatusCode; +use rustack_lambda_runtime::{ + client::ClientEvent, protocol::Architecture as RuntimeArchitecture, session::RuntimeResult, +}; +use tokio::{ + runtime::Handle, + sync::{Semaphore, mpsc, oneshot, watch}, + time::{Instant, interval, sleep_until}, +}; +use typed_builder::TypedBuilder; + +use super::{ + Call, EnvironmentConfig, EnvironmentError, EnvironmentPhase, EnvironmentResponse, + actor::{Health, Signals}, + cleanup::{Cleaner, Phase, Progress}, + io::IoGuard, +}; +use crate::{ + artifact::Architecture, + docker::{ + DockerError, DockerKey, DockerPort, EnvironmentPlan, connection::stopped as wait_for_stop, + mutation::MutationGuard, plan::ContainerRole, + }, + invocation::CleanupTask, +}; + +#[derive(TypedBuilder)] +pub(super) struct Worker { + docker: DockerPort, + config: EnvironmentConfig, + plan: Arc, + progress: Arc, + cleanup: Arc, + health: Arc, + signals: Signals, + global: watch::Sender, + io: Arc, + runtime: Handle, +} +pub(super) type FinalReply = ( + oneshot::Sender>, + Result, +); +impl Worker { + #[cfg(test)] + pub(super) async fn serve_test( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + ) -> Option { + self.serve(io, cleaner, mailbox).await + } + + pub(super) async fn run( + self, + mut mailbox: mpsc::Receiver, + deadline: Instant, + ) -> Result<(), EnvironmentError> { + let mut io = IoGuard::new( + self.runtime.clone(), + self.io.clone(), + self.config, + self.signals.clone(), + ); + let cleaner = self.cleaner(); + let pending = match self.initialize(&mut io, deadline).await { + Ok(()) => { + self.signals.phase(EnvironmentPhase::Idle); + self.serve(&mut io, &cleaner, &mut mailbox).await + } + Err(error) => { + self.signals.error(error); + None + } + }; + self.signals.phase(EnvironmentPhase::Retiring); + mailbox.close(); + while let Ok(call) = mailbox.try_recv() { + let _ = call.reply.send(Err(EnvironmentError::Stopped)); + } + cleaner.retire(&self.io, Some(&mut io)).await; + if let Some((reply, result)) = pending { + let _ = reply.send(result); + } + Ok(()) + } + fn cleaner(&self) -> Cleaner { + Cleaner::builder() + .docker(self.docker.clone()) + .config(self.config) + .plan(self.plan.clone()) + .progress(self.progress.clone()) + .health(self.health.clone()) + .cleanup(self.cleanup.clone()) + .signals(self.signals.clone()) + .build() + } + fn check(&self, deadline: Instant) -> Result<(), EnvironmentError> { + if *self.global.borrow() || *self.signals.stop.borrow() { + Err(EnvironmentError::Cancelled) + } else if Instant::now() >= deadline { + Err(EnvironmentError::Deadline) + } else { + Ok(()) + } + } + async fn create( + &self, + role: ContainerRole, + deadline: Instant, + ) -> Result { + self.check(deadline)?; + let mutation = MutationGuard::new(); + self.progress + .phase(role) + .send_replace(Phase::Uncertain(mutation.receipt())); + let created = self + .plan + .create_tracked(role, &self.docker, self.config.deadline(), mutation) + .await; + if matches!( + &created, + Err(DockerError::Busy + | DockerError::Status(StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND)) + ) { + self.progress.phase(role).send_replace(Phase::Clean); + } + let created = created?; + let inspected = self + .docker + .inspect_container(self.plan.key(role), self.config.deadline()) + .await?; + let id = self.plan.owned(role, &inspected)?; + if id.as_str() != created.id { + return Err(DockerError::Protocol.into()); + } + self.progress + .phase(role) + .send_replace(Phase::Known(id.clone())); + self.check(deadline)?; + Ok(id) + } + async fn initialize( + &self, + io: &mut IoGuard, + deadline: Instant, + ) -> Result<(), EnvironmentError> { + let relay = self.create(ContainerRole::Relay, deadline).await?; + self.signals.phase(EnvironmentPhase::Initializing); + let attached = self.docker.attach(&relay, self.config.deadline()).await?; + let architecture = match self.plan.revision().architecture { + Architecture::Arm64 => RuntimeArchitecture::Arm64, + Architecture::X86_64 => RuntimeArchitecture::X86_64, + }; + io.control(attached, self.plan.environment_id(), architecture)?; + self.check(deadline)?; + self.docker + .start_container(&relay, self.config.deadline()) + .await?; + self.check(deadline)?; + match self.next(io, deadline, None, None).await? { + ClientEvent::RelayReady => {} + _ => return Err(DockerError::Protocol.into()), + } + let function = self.create(ContainerRole::Function, deadline).await?; + io.logs( + self.docker + .attach_logs(&function, self.config.deadline()) + .await?, + )?; + self.check(deadline)?; + self.docker + .start_container(&function, self.config.deadline()) + .await?; + match self.next(io, deadline, Some(&self.cleaner()), None).await? { + ClientEvent::BootstrapReady => Ok(()), + ClientEvent::InitError(payload) if payload.as_ref().len() <= 32768 => { + Err(EnvironmentError::Initialization(payload)) + } + _ => Err(DockerError::Protocol.into()), + } + } + async fn next( + &self, + io: &mut IoGuard, + deadline: Instant, + health: Option<&Cleaner>, + mut cancellation: Option<&mut watch::Receiver>, + ) -> Result { + let mut global = self.global.subscribe(); + let mut stopped = self.signals.stop.subscribe(); + let mut tick = interval(Duration::from_secs(1)); + tick.tick().await; + loop { + self.check(deadline)?; + if cancellation + .as_ref() + .is_some_and(|cancel| *cancel.borrow() || cancel.has_changed().is_err()) + { + return Err(EnvironmentError::Cancelled); + } + tokio::select! {biased; + ()=wait_for_stop(&mut global)=>return Err(EnvironmentError::Cancelled), + ()=wait_for_stop(&mut stopped)=>return Err(EnvironmentError::Cancelled), + ()=sleep_until(deadline)=>return Err(EnvironmentError::Deadline), + ()=async{match &mut cancellation{Some(cancel)=>{let _=cancel.wait_for(|stop|*stop).await;},None=>pending().await,}}=>return Err(EnvironmentError::Cancelled), + result=io.client()?.next(deadline)=>return result.map_err(EnvironmentError::from), + _=tick.tick(),if health.is_some()=>{if let Some(health)=health{health.inspect_health().await?;io.healthy().await?;tick.reset();}}, + } + } + } + async fn serve( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + ) -> Option { + let mut global = self.global.subscribe(); + let mut stopped = self.signals.stop.subscribe(); + let mut tick = interval(Duration::from_secs(1)); + tick.tick().await; + let mut idle = Instant::now() + self.config.0.idle; + loop { + let action = tokio::select! {biased; + ()=wait_for_stop(&mut global)=>return None, + ()=wait_for_stop(&mut stopped)=>return None, + ()=sleep_until(idle)=>return None, + _=tick.tick()=>{let result=async{cleaner.inspect_health().await?;io.healthy().await}.await;tick.reset();if let Err(error)=result{self.signals.error(error);return None;}continue;}, + result=io.client().ok()?.next(idle)=>{self.signals.error(result.err().map_or(EnvironmentError::Docker(DockerError::Protocol),EnvironmentError::from));return None;}, + call=mailbox.recv()=>call, + }; + let mut call = action?; + self.signals.phase(EnvironmentPhase::Busy); + let result = self.call(io, cleaner, &mut call).await; + let reusable = matches!( + &result, + Ok(EnvironmentResponse { + result: RuntimeResult::Success(_), + .. + }) + ) && !call.reply.is_closed() + && io.reusable(); + if reusable { + self.signals.phase(EnvironmentPhase::Idle); + let _ = call.reply.send(result); + idle = Instant::now() + self.config.0.idle; + } else { + if let Err(error) = &result { + self.signals.error(error.clone()); + } + return Some((call.reply, result)); + } + } + } + async fn call( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + call: &mut Call, + ) -> Result { + let (attempt, deadline, arn, _, _) = call.invocation.parts(); + let name = arn + .as_ref() + .split_once(":function:") + .and_then(|(_, name)| name.split(':').next()); + if name != Some(self.plan.revision().function_name.as_str()) { + return Err(EnvironmentError::Configuration); + } + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } + io.client()? + .invoke( + call.invocation.clone(), + deadline.min(self.config.deadline()), + ) + .await?; + let result = self + .next(io, deadline, Some(cleaner), Some(&mut call.cancellation)) + .await; + match result { + Ok(ClientEvent::InvocationResult { + attempt: returned, + result, + }) if returned == attempt => { + let tail = io.tail().await?; + let logs = if call.capture_logs { + tail + } else { + Bytes::new() + }; + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } + Ok(EnvironmentResponse { result, logs }) + } + Ok(_) => Err(DockerError::Protocol.into()), + Err(error) => { + if io + .client()? + .cancel(attempt, self.config.deadline()) + .await + .is_ok() + { + let _ = io.client()?.next(self.config.deadline()).await; + } + Err(error) + } + } + } +} diff --git a/crates/rustack-lambda-core/src/docker/logs.rs b/crates/rustack-lambda-core/src/docker/logs.rs index a2914a60..b666b9d3 100644 --- a/crates/rustack-lambda-core/src/docker/logs.rs +++ b/crates/rustack-lambda-core/src/docker/logs.rs @@ -91,7 +91,7 @@ impl FunctionLogs { Self::start_reader(reader, ring_bytes) } - fn start_reader( + pub(super) fn start_reader( reader: DockerOutputReader, ring_bytes: usize, ) -> Result { diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 614516e2..1c3be644 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -4,10 +4,12 @@ //! retain preallocated identities and reservations through reconciliation. mod connection; +mod environment; mod framing; mod image; mod lease; mod logs; +mod mutation; mod ownership; mod pin; mod plan; @@ -25,6 +27,11 @@ use std::{ pub use bollard_stubs::models; use bytes::Bytes; use connection::exchange; +pub use environment::{ + EnvironmentCapacity, EnvironmentConfig, EnvironmentError, EnvironmentHandle, + EnvironmentManager, EnvironmentPhase, EnvironmentPort, EnvironmentResponse, + EnvironmentSettings, EnvironmentStatus, +}; pub use framing::{DockerOutputReader, MAX_DOCKER_RECORD, OutputChannel, OutputChunk}; use http::{Method, StatusCode}; use hyper::upgrade::Upgraded; @@ -90,6 +97,7 @@ impl Operation { } struct RawRequest { + mutation: Option, operation: Operation, method: Method, target: String, @@ -374,7 +382,7 @@ impl DockerPort { body: &ContainerCreateBody, deadline: Instant, ) -> Result { - self.create_container_inner(name, body, None, deadline) + self.create_container_inner(name, body, None, deadline, None) .await } @@ -398,7 +406,7 @@ impl DockerPort { architecture: Architecture, deadline: Instant, ) -> Result { - self.create_container_inner(name, body, Some(architecture), deadline) + self.create_container_inner(name, body, Some(architecture), deadline, None) .await } @@ -408,6 +416,7 @@ impl DockerPort { body: &ContainerCreateBody, architecture: Option, deadline: Instant, + mutation: Option, ) -> Result { let query = { let mut query = form_urlencoded::Serializer::new(String::new()); @@ -418,15 +427,19 @@ impl DockerPort { query.finish() }; decode( - self.call( - Operation::Create, - Method::POST, - format!("/containers/create?{query}"), - encode(body)?, - StatusCode::CREATED, - deadline, - ) - .await?, + self.raw + .request( + RawRequest { + mutation, + operation: Operation::Create, + method: Method::POST, + target: format!("/v{}/containers/create?{query}", self.version), + body: encode(body)?, + expected: StatusCode::CREATED, + }, + deadline, + ) + .await?, deadline, ) } @@ -581,6 +594,7 @@ impl DockerPort { self.raw .request( RawRequest { + mutation: None, operation, method, target, @@ -624,6 +638,7 @@ impl DockerWire { let response = raw .request( RawRequest { + mutation: None, operation: Operation::Version, method: Method::GET, target: "/version".into(), diff --git a/crates/rustack-lambda-core/src/docker/mutation.rs b/crates/rustack-lambda-core/src/docker/mutation.rs new file mode 100644 index 00000000..bece8280 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/mutation.rs @@ -0,0 +1,31 @@ +//! A retained receipt distinguishes definitely unissued from uncertain mutation IO. + +use std::sync::{ + Arc, + atomic::{AtomicU8, Ordering}, +}; + +#[derive(Debug, Clone)] +pub(super) struct MutationReceipt(Arc); +pub(super) struct MutationGuard(MutationReceipt); +impl MutationGuard { + pub(super) fn new() -> Self { + Self(MutationReceipt(Arc::new(AtomicU8::new(0)))) + } + pub(super) fn receipt(&self) -> MutationReceipt { + self.0.clone() + } + pub(super) fn issued(&mut self) { + self.0.0.store(1, Ordering::Release); + } +} +impl Drop for MutationGuard { + fn drop(&mut self) { + self.0.0.fetch_or(2, Ordering::Release); + } +} +impl MutationReceipt { + pub(super) fn definitely_unissued(&self) -> bool { + self.0.load(Ordering::Acquire) == 2 + } +} diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs index 6bb7b305..5feccc6a 100644 --- a/crates/rustack-lambda-core/src/docker/plan.rs +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -16,9 +16,10 @@ use uuid::Uuid; use super::{ DirectoryLease, DockerError, DockerKey, DockerPort, models::{ - ContainerCreateBody, ContainerCreateResponse, HealthConfig, HostConfig, - HostConfigLogConfig, RestartPolicy, RestartPolicyNameEnum, + ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, HealthConfig, + HostConfig, HostConfigLogConfig, RestartPolicy, RestartPolicyNameEnum, }, + mutation::MutationGuard, }; use crate::{ artifact::{DeploymentArtifact, PinnedImage}, @@ -29,6 +30,12 @@ use crate::{ const MIB: u64 = 1024 * 1024; const RELAY_MEMORY: u64 = 64 * MIB; +#[derive(Debug, Clone, Copy)] +pub(super) enum ContainerRole { + Relay, + Function, +} + /// Operator network selection; a bridge still requires daemon startup validation. #[derive(Clone)] #[non_exhaustive] @@ -154,7 +161,7 @@ struct Bodies { /// Fixed create requests retaining revision, image availability and namespace leases. #[non_exhaustive] pub struct EnvironmentPlan { - _directory: DirectoryLease, + pub(super) directory: DirectoryLease, relay: PinnedImage, revision: Arc, identity: Identity, @@ -170,6 +177,46 @@ impl fmt::Debug for EnvironmentPlan { } } impl EnvironmentPlan { + pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { + self.directory.owner_id() == directory.owner_id() + && self.directory.run_id() == directory.run_id() + } + pub(super) fn revision(&self) -> &ExecutionRevision { + &self.revision + } + pub(super) fn key(&self, role: ContainerRole) -> &DockerKey { + match role { + ContainerRole::Relay => self.relay_key(), + ContainerRole::Function => self.function_key(), + } + } + pub(super) fn owned( + &self, + role: ContainerRole, + inspected: &ContainerInspectResponse, + ) -> Result { + let body = match role { + ContainerRole::Relay => &self.bodies.relay, + ContainerRole::Function => &self.bodies.function, + }; + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let expected = body.labels.as_ref().ok_or(DockerError::Configuration)?; + let actual = config.labels.as_ref().ok_or(DockerError::Quarantined)?; + let id = inspected.id.as_deref().ok_or(DockerError::Protocol)?; + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || inspected.name.as_deref() != Some(format!("/{}", self.key(role).as_str()).as_str()) + || config.image != body.image + || !expected + .iter() + .all(|(key, value)| actual.get(key) == Some(value)) + { + return Err(DockerError::Quarantined); + } + id.parse() + } /// Allocate identities and validate both immutable create bodies before IO. /// /// # Errors @@ -271,7 +318,7 @@ impl EnvironmentPlan { .checked_add(RELAY_MEMORY) .ok_or(DockerError::Configuration)?; Ok(Self { - _directory: directory, + directory, relay, revision, identity, @@ -382,6 +429,27 @@ impl EnvironmentPlan { ) .await } + pub(super) async fn create_tracked( + &self, + role: ContainerRole, + port: &DockerPort, + deadline: Instant, + mutation: MutationGuard, + ) -> Result { + self.live()?; + let body = match role { + ContainerRole::Relay => &self.bodies.relay, + ContainerRole::Function => &self.bodies.function, + }; + port.create_container_inner( + self.key(role), + body, + Some(self.revision.architecture), + deadline, + Some(mutation), + ) + .await + } fn live(&self) -> Result<(), DockerError> { match &self.revision.artifact { DeploymentArtifact::Image(function) if self.relay.is_live() && function.is_live() => { @@ -604,4 +672,4 @@ fn function_environment( #[cfg(test)] #[path = "plan_tests.rs"] -mod tests; +pub(super) mod tests; diff --git a/crates/rustack-lambda-core/src/docker/plan_tests.rs b/crates/rustack-lambda-core/src/docker/plan_tests.rs index f05062c2..eadbddc1 100644 --- a/crates/rustack-lambda-core/src/docker/plan_tests.rs +++ b/crates/rustack-lambda-core/src/docker/plan_tests.rs @@ -16,7 +16,7 @@ use crate::{ fn directory() -> DirectoryLease { DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()) } -fn policy() -> ContainerPolicy { +pub(in crate::docker) fn policy() -> ContainerPolicy { ContainerPolicy::new( "none".parse().unwrap(), 1000, @@ -26,7 +26,7 @@ fn policy() -> ContainerPolicy { ) .unwrap() } -fn image_with(user: &str, environment: Vec) -> PinnedImage { +pub(in crate::docker) fn image_with(user: &str, environment: Vec) -> PinnedImage { let mut metadata = ResolvedImage::fixture(Architecture::Arm64) .metadata() .clone(); @@ -49,7 +49,7 @@ fn image_with(user: &str, environment: Vec) -> PinnedImage { .unwrap(); PinnedImage::fixture(metadata.try_into().unwrap()) } -fn revision() -> ExecutionRevision { +pub(in crate::docker) fn revision() -> ExecutionRevision { ExecutionRevision::builder() .resource_id(FunctionResourceId::new()) .revision_id(Uuid::new_v4().to_string()) diff --git a/crates/rustack-lambda-core/src/docker/tests.rs b/crates/rustack-lambda-core/src/docker/tests.rs index 863dcd4f..7bd6bce3 100644 --- a/crates/rustack-lambda-core/src/docker/tests.rs +++ b/crates/rustack-lambda-core/src/docker/tests.rs @@ -324,6 +324,7 @@ async fn test_should_bound_request_serialization_and_redact_request_debug() { ..Default::default() }; let request = RawRequest { + mutation: None, operation: Operation::Create, method: Method::POST, target: "/secret-target".into(), @@ -421,6 +422,7 @@ async fn test_should_keep_cleanup_capacity_while_business_calls_and_mailbox_are_ .business .try_send(Envelope { request: RawRequest { + mutation: None, operation: Operation::Info, method: Method::GET, target: "/v1.53/info".into(), diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs new file mode 100644 index 00000000..ae76e2ad --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -0,0 +1,277 @@ +//! Real container lifecycle qualification of the retained environment owner. +#![cfg(unix)] +#![forbid(unsafe_code)] +use std::{collections::HashMap, env, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use bytes::Bytes; +use rustack_lambda_core::{ + artifact::{Architecture, DeploymentArtifact, PinnedImage}, + docker::{ + ContainerPolicy, DirectoryLease, DockerWire, EnvironmentConfig, EnvironmentError, + EnvironmentHandle, EnvironmentManager, EnvironmentPlan, EnvironmentSettings, FunctionLogs, + ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OwnerFilter, RelayBudgets, + }, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, +}; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + protocol::{AttemptId, Payload}, + session::{Invocation, RuntimeResult}, +}; +use serde_json::{Value, json}; +use tempfile::TempDir; +use tokio::time::{Instant, timeout}; +use uuid::Uuid; +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +struct Fixture { + owner: EnvironmentManager, + images: ImageLeaseManager, + wire: DockerWire, + lease: DirectoryLease, + relay: Option, + function: Option, + directory: Option, +} +impl Fixture { + async fn new() -> Result { + let socket = + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket required")?; + let wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let directory = tempfile::tempdir()?; + let lease = DirectoryLease::acquire(directory.path().into(), deadline()).await?; + let images = ImageLeaseManager::start( + wire.port(), + ImagePinScope::new(lease.clone()), + ImageLeaseConfig::default(), + )?; + let arch = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let config = EnvironmentConfig::try_from( + EnvironmentSettings::builder() + .environments(1) + .memory_bytes(256 * 1024 * 1024) + .io(Duration::from_secs(60)) + .start(Duration::from_secs(60)) + .build(), + )?; + let owner = EnvironmentManager::start(wire.port(), lease.clone(), config)?; + let mut fixture = Self { + owner, + images, + wire, + lease, + relay: None, + function: None, + directory: Some(directory), + }; + if let Err(error) = fixture.prepare(arch).await { + let cleanup = fixture.cleanup().await; + return Err(error.context(format!( + "setup cleanup: {}", + if cleanup.is_ok() { + "confirmed" + } else { + "unresolved" + } + ))); + } + Ok(fixture) + } + async fn prepare(&mut self, arch: Architecture) -> Result<()> { + let relay_reference = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let relay = self + .images + .port() + .prepare(&relay_reference, arch, None, deadline()) + .await?; + self.relay = Some(relay); + let suffix = if arch == Architecture::Arm64 { + "arm64" + } else { + "amd64" + }; + let function = self + .images + .port() + .prepare( + &format!("rustack-p0-handler:{suffix}").parse()?, + arch, + None, + deadline(), + ) + .await?; + self.function = Some(function); + Ok(()) + } + + fn plan(&self) -> Result { + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("environment-fixture".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture( + self.function + .as_ref() + .context("function image")? + .resolved() + .metadata() + .architecture, + ) + .artifact(DeploymentArtifact::Image( + self.function.as_ref().context("function image")?.clone(), + )) + .image_config(None) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ])) + .timeout(Duration::from_secs(120)) + .memory_mb(192) + .ephemeral_storage_mb(512) + .build(); + Ok(EnvironmentPlan::new( + self.lease.clone(), + self.relay.as_ref().context("relay image")?.clone(), + Arc::new(revision), + ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?, + )?) + } + async fn cleanup(&mut self) -> Result<()> { + let result = async { + self.owner.shutdown(deadline()).await?; + ensure!( + self.owner.port().status().allocated == 0, + "allocation reservation remains" + ); + let containers = self + .wire + .port() + .list_containers( + OwnerFilter::new(self.lease.owner_id(), Some(self.lease.run_id()), None)?, + deadline(), + ) + .await?; + ensure!(containers.is_empty(), "owned containers remain"); + self.relay.take(); + self.function.take(); + self.images.shutdown(deadline()).await?; + self.wire.shutdown().await?; + Ok(()) + } + .await; + if result.is_err() { + if let Some(directory) = self.directory.take() { + let _ = directory.keep(); + } + } + result + } +} +fn invocation(event: &Value) -> Result { + Ok(Invocation::new( + AttemptId::default(), + Instant::now() + Duration::from_secs(120), + "arn:aws:lambda:us-east-1:000000000000:function:environment-fixture:prod".parse()?, + local_trace_id()?, + Payload::try_from(Bytes::from(serde_json::to_vec(event)?))?, + )?) +} +async fn call(handle: &mut EnvironmentHandle, event: &Value, failed: bool) -> Result { + let response = handle.invoke(invocation(event)?, true).await?; + match response.result { + RuntimeResult::Success(payload) if !failed => Ok(serde_json::from_slice(payload.as_ref())?), + RuntimeResult::FunctionError(payload) if failed => { + Ok(serde_json::from_slice(payload.as_ref())?) + } + _ => anyhow::bail!("unexpected handler result"), + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_own_real_initialization_warm_calls_function_error_and_cleanup() -> Result<()> { + let mut fixture = Fixture::new().await?; + let result=async{ + let mut handle=fixture.owner.port().spawn(fixture.plan()?,deadline())?;handle.wait_ready(deadline()).await?; + ensure!(matches!(fixture.owner.port().spawn(fixture.plan()?,deadline()),Err(EnvironmentError::Busy)),"global environment bound not enforced"); + let first=call(&mut handle,&json!({"value":35,"padding":"x".repeat(6*1024*1024-4096),"response_bytes":6*1024*1024-4096}),false).await?; + ensure!(first["computed"]==42&&first["counter"]==1,"computed result differs");ensure!(first["large"].as_str().is_some_and(|value|value.len()==6*1024*1024-4096),"large result differs"); + let second=call(&mut handle,&json!({"value":0}),false).await?;ensure!(second["computed"]==7&&second["counter"]==2,"warm result differs"); + let failure=call(&mut handle,&json!({"fail":true}),true).await?;ensure!(failure["errorType"]=="ValueError","FunctionError differs");handle.shutdown(deadline()).await?;Ok(()) + }.await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_retire_running_code_after_abandoned_call_and_join_all_owned_io() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let result = async { + let plan = fixture.plan()?; + let key = plan.function_key().clone(); + let mut handle = fixture.owner.port().spawn(plan, deadline())?; + handle.wait_ready(deadline()).await?; + let mut observed = FunctionLogs::start( + fixture.wire.port().attach_logs(&key, deadline()).await?, + Duration::from_secs(60), + 262_144, + )?; + let call = tokio::spawn(async move { + handle + .invoke(invocation(&json!({"value":1,"sleep":true})).unwrap(), false) + .await + }); + let observation = timeout(Duration::from_secs(30), async { + loop { + let snapshot = observed.snapshot(deadline()).await?; + if snapshot + .tail + .windows(14) + .any(|part| part == b"P0_BUSY_START=") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("handler did not start"); + call.abort(); + let joined = call.await; + let retired = fixture.owner.shutdown(deadline()).await; + let logs = observed.shutdown().await; + observation??; + ensure!( + matches!(joined,Err(ref error) if error.is_cancelled()), + "abandoned call was not cancelled" + ); + retired?; + logs?; + Ok(()) + } + .await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} diff --git a/crates/rustack-lambda-runtime/src/client.rs b/crates/rustack-lambda-runtime/src/client.rs index 6e2327b2..cd9162a1 100644 --- a/crates/rustack-lambda-runtime/src/client.rs +++ b/crates/rustack-lambda-runtime/src/client.rs @@ -17,7 +17,7 @@ use crate::{ }, relay::RelayError, session::{Invocation, RuntimeResult}, - transport::{InputEnd, Settings, Transport, after}, + transport::{InputEnd, Settings, Transport, after, preserve_worker_failure}, }; /// Validated host identity, expected architecture and control IO budget. @@ -344,7 +344,7 @@ impl ControlClient { } .await; let io = self.transport.shutdown().await; - let outcome = outcome.and(io); + let outcome = preserve_worker_failure(outcome, io); self.settlement = Some(outcome.clone()); outcome } diff --git a/crates/rustack-lambda-runtime/src/client/tests.rs b/crates/rustack-lambda-runtime/src/client/tests.rs index 37b89f90..f2d4d980 100644 --- a/crates/rustack-lambda-runtime/src/client/tests.rs +++ b/crates/rustack-lambda-runtime/src/client/tests.rs @@ -590,3 +590,39 @@ fn test_should_validate_client_io_budget() { assert!(ClientConfig::new(EnvironmentId::default(), Architecture::Arm64, budget).is_err()); } } + +#[tokio::test] +async fn test_should_preserve_transport_worker_failure_after_a_prior_scope_error() { + let environment = EnvironmentId::default(); + let (reader, mut peer) = duplex(4096); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + tokio::io::sink(), + ) + .unwrap(); + peer.write_all( + &encode(&Frame::new( + EnvironmentId::default(), + Message::Hello { + architecture: Architecture::Arm64, + }, + )) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + client.next(deadline()).await, + Err(RelayError::Scope) + )); + client.transport.abort_owner(); + assert!(matches!( + client.shutdown(deadline()).await, + Err(RelayError::Worker) + )); + assert!(matches!( + client.shutdown(deadline()).await, + Err(RelayError::Worker) + )); +} diff --git a/crates/rustack-lambda-runtime/src/transport.rs b/crates/rustack-lambda-runtime/src/transport.rs index 21c87d47..8cc7a354 100644 --- a/crates/rustack-lambda-runtime/src/transport.rs +++ b/crates/rustack-lambda-runtime/src/transport.rs @@ -18,6 +18,17 @@ use crate::{ const MAILBOX_CAPACITY: usize = 2; +pub(crate) fn preserve_worker_failure( + primary: Result<(), RelayError>, + joined: Result<(), RelayError>, +) -> Result<(), RelayError> { + if matches!(&primary, Err(RelayError::Worker)) || matches!(&joined, Err(RelayError::Worker)) { + Err(RelayError::Worker) + } else { + primary.and(joined) + } +} + #[derive(Debug, Clone, Copy)] pub(crate) enum InputEnd { Close, @@ -73,6 +84,13 @@ pub(crate) struct Transport { } impl Transport { + #[cfg(test)] + pub(crate) fn abort_owner(&self) { + if let Some(task) = &self.owner.task { + task.abort(); + } + } + pub(crate) fn start(reader: R, writer: W, settings: Settings) -> Result where R: AsyncRead + Unpin + Send + 'static, @@ -213,7 +231,7 @@ impl Owner { Err(error) if error.is_cancelled() => Ok(()), Ok(Ok(WorkerExit::OutputClosed)) | Err(_) => Err(RelayError::Worker), }; - outcome = outcome.and(result); + outcome = preserve_worker_failure(outcome, result); } self.status.send_replace(Some(outcome.clone())); outcome diff --git a/specs/index.md b/specs/index.md index 0d7593ee..fcf10414 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs and immutable allocation plans. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until environment/pool lifecycle, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until pool/executor integration, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 574bf4d2..8fb73193 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -508,6 +508,40 @@ Absent/root/numeric-zero image USER is replaced with configured nonzero UID:GID; Tests pin immutable IDs, platforms, distinct names/labels, image overrides, env ordering/authoritative metadata/redaction, user policy, all resource/swap/tmpfs/health/log/restart fields, invalid revision/policy/region rejection before mutation, and actual directory/image lease lifetime. Real wire qualification consumes the plan for both containers and verifies the handler's cgroup/user/rootfs/tmpfs output, actual argv/workdir overrides, warm/error results and owned cleanup. These are allocation policy checks; environment/pool lifecycle, historical reconciliation, named bridge validation, probes and SDK executor acceptance remain required. +### 4.3 Retained environment owner and allocation reservations + +`EnvironmentManager` is the low-level environment owner above DockerWire, separate from the executor's future warm/LRU pool. It accepts only EnvironmentPlans for its retained directory owner/run. Validated EnvironmentSettings bound1..32 allocated environments,256MiB..1TiB aggregate memory including relay overhead,1..60s independent IO/start budgets,1..86,400s idle lease and4KiB..1MiB channel rings. Defaults are32/8GiB,5s IO,15s start,600s idle and256KiB rings. Reserve count and checked aggregate memory before enqueueing; queued/allocating/busy/idle/retiring/quarantined records all retain their reservation. A bounded32-message admission queue and per-environment bounded invocation mailbox cannot create unbounded accepted work. Pool/provider admission remains a separate required consumer. + +One root actor owns keyed records above a JoinSet of environment workers. Each record retains Arc plan, reservation, separate relay/function mutation phases, stop/status/settlement channels and a retained IO cleanup task. Record Uncertain before issuing create, retaining a wire mutation receipt. Its RAII guard moves with the accepted request, marks possible issuance immediately before HTTP send and records completion even when validation, connect, queue or worker teardown fails. Only a completed never-issued receipt proves absence without lookup; a still-queued receipt does not. An issued create is awaited under its independent IO budget even after caller/start expiry. Complete400/404 validation failures and unissued mailbox Busy are known no-effect;500, lost/truncated replies and invalid IDs remain uncertain. Never replay create. Observe the preallocated name and verify exact owner/run/environment/resource/role labels, captured Config.Image and canonical64-byte lowercase container ID before adopting it. A foreign/malformed object is quarantined, never killed or removed. An initial404 cannot settle an uncertain create; retain and retry until its single possible commit is observed and cleaned. + +Successful initialization is relay create/verify→attach→start→Hello/Ready→function create/verify→output attach→start→BootstrapReady. Always mutate verified container IDs, not a name that could be retargeted. Preallocated names remain the uncertainty recovery keys. Distinguish initialization payload, relay/protocol error, function early exit/OOM, cancellation and deadline errors. Init payloads are capped at32KiB. Worker states are Allocating/Initializing/Idle/Busy/Retiring/Quarantined/Removed; phase and primary bounded error are observable without payload logging. `wait_ready(deadline)` is a cancel-safe read of that state. An owner handle's Drop or explicit cancel signals retirement, independently of entered runtime. Caller loss never aborts accepted create ownership. + +After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline; exact Lambda handler-timeout-at-delivery and per-invocation log attribution remain executor integration requirements and must be completed before production enablement. + +Normal retirement invalidates/cancels the active attempt, then inspects/kills/waits/removes the verified function, joins log/control/bridge IO, then removes the verified relay. Keep count/memory/image/directory references until both container absence and IO settlement are established. Retry cleanup at1/2/4 seconds capped30s; failed cleanup is observable Quarantined and prevents safe readiness/snapshot success. A verified known ID's404 is terminal absence; an uncertain name's404 is not. Separate reserved cleanup concurrency prevents business saturation from starving retirement. No generic container/image deletion is permitted. + +IO handles live in a worker guard which transfers them to the record's retained cleanup task on abnormal worker Drop using its captured runtime handle. Root recovery aborts and joins failed workers, then joins their transferred IO and cleans recorded containers before releasing reservations. A root actor Drop transfers records and joins to retained emergency recovery, preserving off-runtime ownership. Shutdown stops admission, retires workers and awaits root/emergency joins through the same absolute caller deadline, preserving ownership across timed-out/cancelled waiters and caching failure. Worker/root panic latches failed admission; clean later recovery cannot rewrite that failure as successful shutdown. A failed IO supervision receipt remains owned/quarantined rather than being treated as clean. + +Known operational protocol/IO failures must remain distinct from uncertain supervision. Control transport shutdown joins its workers even after a protocol error; prioritize Worker when combining handshake and IO-join failures so a preceding protocol error cannot mask uncertain cleanup. The environment may retire a known-bad session once its IO and containers are proven stopped, preserving the business error. A failed/aborted cleanup supervisor cannot grant that proof. Tests require ordinary/warm/error execution, count/memory pressure, aborted admission/call/shutdown, critical delayed create including500→404→late commit, foreign labels/IDs, cleanup quarantine/recovery, idle expiry, worker/root panic and actual two-container/IO cleanup. Production selection remains disabled until the executor/pool, historical reconciliation, pulls, network/probes and complete M0 gates land. + +```text +Provider / future Pool Environment root Environment worker + │ plan + reserved budget ────▶│ record(plan, phases, permit) │ + │ │ ABOVE retained JoinSet ──────────▶│ create/attach/start + │◀── handle/status ──────────│ │ client + logs + bridge + │ invoke / cancel ──────────────────────────────────────────────▶│ one active attempt + │ │ │ + │ drop / shutdown ───────────▶│ priority stop ───────────────────▶│ retire function + │ │ │ join IO → retire relay + │ │◀──── known clean settlement ─────│ + │ │ remove record / release budget │ + │ │ │ panic / abort + │ │◀── guard transfers IO cleanup ────┤ + │ │ join failed worker + IO receipt │ + │ │ reconcile phases / quarantine │ + │ │ retain records until proven clean │ +``` + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 74652d56..45d0ef11 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -123,6 +123,8 @@ Expose capability fields for workflow type/query profile/state types/integration The workflow drain interval permits already accepted executions to finish. Then cancel remaining executions with ABORTED and error=Rustack.Shutdown, preserving terminal history. This is deliberate local shutdown behavior, not cloud-style pause/resume. Revoke callback tokens, cancel owned children and await Lambda cleanup before Lambda itself quiesces. Quarantine or unfinished cleanup prevents publishing a successful consistency snapshot. Quiescence must be idempotent and share one global absolute deadline, not restart a full timeout per worker. +Within Lambda, stop deployment/invocation producers before retiring the pool/environment consumers. Join EnvironmentManager before revoking image leases, and join image cleanup before DockerWire shutdown. Environment count and aggregate function+relay memory remain reserved through allocating, idle, busy and quarantined states; a timed-out shutdown waiter leaves that ownership intact. A removed environment's business failure is distinct from failed supervision: known protocol/handler errors may retire cleanly, while Worker failure remains cached even after physical recovery. The lower-level owner alone is not a startup readiness receipt; historical uncertain effects and daemon/network/resource probes remain separate required gates. + Snapshot coverage: | Resource | Persist | Restore behavior | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 2cba4e4b..53c94ca4 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -100,6 +100,10 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi ## 4. Security acceptance +`make test-lambda-environments` verifies the retained root/worker owner independently of Docker: transactional count/memory admission, scope rejection, definitive create rejection, canceled critical create,500→404→late commit, foreign label/image/name/ID quarantine, root abort recovery, revoked/connect-failed never-issued settlement, slow-health command fairness, failed versus normal queued-handle settlement, final log EOF without reuse and bounded startup under a long caller deadline. The transport regression combines an actual scope error with owner abort and requires Worker precedence/caching. Never count these scripted cases as image code execution. + +`make test-lambda-environments-docker` builds the required images and explicitly runs two real owner cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, exact handler-timeout-at-delivery, per-invocation log attribution, startup bridge/resource probes, historical reconciliation or warm-pool limits. + `make test-lambda-environment-plans` verifies §4.2/D30 before allocation: fixed image IDs/platforms, distinct names and ownership roles, relay/function resource/swap/tmpfs/health/log/restart policy, user selection/grammar, bounded revision/region/env/argv validation, single-key inherited→captured→runtime environment precedence with deletion markers, retained real-directory and both-image lease lifetime, and revoked availability before IO. A spawned create regression proves the wire future is Send. The actual two-case wire suite consumes EnvironmentPlan for both ordinary RIC execution and empty ImageConfig overrides: require handler-reported CPU/memory/no-swap/PIDs,512MiB tmpfs/mode, cap-drop/no-new-privileges/read-only rootfs, and configured metadata despite intentionally conflicting image defaults. Inspect real CMD/ENTRYPOINT/WORKDIR and require expected exits142/0 for inherited-empty and reset-entrypoint variants. Join control/log streams and remove both owned containers/aliases on all normal test outcomes. This is allocation-policy qualification, not lifecycle/pool, startup bridge/probe, historical reconciliation or SDK executor acceptance. `make test-lambda-logs` verifies the independent function output owner: binary stdout/stderr ordering, separate bounded overwrite rings and saturating discard counts, combined4KiB tail, EOF/failure retention, socket closure on framing failure, snapshot admission bounds, partial-frame reads retained through completed/abandoned snapshot requests, fixed partial/silence deadlines, output-flood responsiveness, cancelled shutdown join, off-runtime Drop and cached actor panic. A scripted daemon pins output-only/no-replay attach query and driver handoff. The real wire suite additionally attaches before function start, enables explicit handler stdout/stderr markers, requires both markers without affecting computed results, and joins logs independently after function removal. These tests qualify capture, not complete per-invocation log attribution or the still-pending executor. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index f0f6e7e2..9dbeada3 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs and immutable environment allocation plans, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker environment/pool executor, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker warm/LRU pool and executor integration, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -146,6 +146,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 directory namespace | Implemented/reviewed; runtime consumers remain | DirectoryLease binds permanent nonreplaced file locking, durable checksummed owner/fresh run and trusted descriptor-relative filesystem operations to all image scopes/pins/jobs/recovery. W60–W62 fixes independently rechecked; 38 ownership cases plus abandoned image-owner cleanup pass. Workspace2,122 tests, core443 tests/103 doctests, boundary874 unit/doc passes, all standard/strict/audit/deny gates pass. Both real two-case wire suites pass on arm64 VZ and amd64/QEMU; post-test owned containers/internal aliases are empty. This does not prove previous-run reconciliation, Linux-host lock qualification or Docker executor readiness. | | 1.3 function logs | Implemented/reviewed; environment integration remains | Output-only attach and FunctionLogs retain partial framing across snapshot traffic, bounded independent rings/combined4KiB binary tail, fair service, early socket closure and cached joined shutdown (§3.7/D29).18 focused tests pass; W63/W64 fixes independently rechecked. Core461 tests/107 doctests, workspace2,140 tests, boundary896 unit/doc passes and all standard/strict gates pass. Both real two-case wire suites receive deliberate handler stdout/stderr markers independently from computed results and join logs during cleanup. No new dependency/version. Per-invocation log attribution, environment integration and complete executor acceptance remain pending. | | 1.3 allocation policy | Implemented/reviewed; lifecycle owner remains | EnvironmentPlan fixes both create bodies to captured identities/platforms/revision, retained directory/image leases, validated policy and authoritative unique-key environment (§4.2/D30).54 new cases pass; W65–W67 fixed/rechecked. Core515 tests/116 doctests, workspace2,194 tests, boundary959 unit/doc passes and standard/strict/audit/deny gates pass. Both real two-case wire suites now consume plans for normal/empty ImageConfig execution, with actual resource/tmpfs/privilege/env assertions; all five shared runtime Docker regressions also pass per daemon. No production pool/executor or historical reconciliation readiness is claimed. | +| 1.3 environment lifecycle | Implemented/reviewed; pool/executor integration remains | EnvironmentManager retains count/memory/plan/container/IO ownership above supervised workers, including uncertain creates, cancelled callers, panic recovery and quarantine (§4.3/D31). W68–W77 fixes and final independent recheck pass. Twenty scripted lifecycle cases plus one runtime failure-precedence regression pass. Core535 unit tests/127 doctests, workspace2,215 tests/403 gated cases, boundary800 unit tests/191 doctests and all standard/strict/audit/deny gates pass. Actual owner2, wire2 and shared runtime5 cases pass on each arm64 VZ and amd64/QEMU daemon; final owner2 cases are rechecked after the ready-reply cancellation fix. Both daemons have no owned containers/internal aliases afterward. Exact delivery-based handler timeout, invocation log attribution, warm/LRU checkout, historical reconciliation/pulls/network/probes and final executor wiring remain required before M0. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) is also implemented/reviewed. Next implement historical-effect reconciliation and Docker environment/pool actors and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete directory ownership/previous-run reconciliation, allowed pulls and remaining network/probe consumers plus function-log/environment integration, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Next implement warm/LRU checkout and the real executor, including exact handler timeout from delivery and invocation log attribution, consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 5c369036..92ee7bd3 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -324,6 +324,29 @@ Both explicit two-case real Docker wire suites pass on Engine29.5.2 arm64 VZ and The plan deliberately cannot authorize unreserved or untracked side effects: the future environment/pool owner must register it above retained IO and reconcile every uncertain create. Named bridges still require daemon validation; explicit none provides disconnected pure-handler tests only. Production Docker remains unavailable pending lifecycle/pool, reconciliation/pulls/network/probes and final wiring, and P2–P5 remain required. +## P1 task1.3 retained environment owner review + +Independent read-only review covers §4.3/D31: root records/reservations above JoinSet workers, verified-ID mutation, critical create ownership, wire issuance receipts, guarded IO transfer, phase-aware cleanup/recovery, cancellation and cached shutdown failure. The final recheck has no remaining in-slice finding. An initially reported capture_logs=false EOF issue was withdrawn after direct re-read confirmed that every result samples log status before selecting returned tail bytes. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W68 | P2; `environment/worker.rs`, health scheduling | A biased periodic tick can remain ready after slow IO and starve accepted calls. Reschedule health after each completed check | Actual serve-loop test queues a call during a health RPC lasting longer than its period and requires bounded command service afterward | +| W69 | P2; `environment/worker.rs`, unissued create | Revocation/connect/queue failure before send must not become permanent uncertain creation. Move one RAII mutation guard with the accepted wire request and permit never-issued proof only after its final disposal | Revoked-before-create and missing-socket cases settle with no mutation/quota leak;500→404 still retains uncertainty until the late object is observed | +| W70 | P2; `environment/actor.rs`, recovered/queued settlement | Physical cleanup cannot rewrite a failed supervisor into successful handle shutdown. Preserve Worker for recovered, queued and existing records after root failure | Root abort during create and normal/root-abort/wire-stop queued tests; repeated failed shutdown remains failed while cleaned quota releases | +| W71 | P2; `tests/docker_environment.rs`, namespace lifetime | Preparation before fixture ownership or early-return cleanup errors could unlink a namespace while image cleanup remains live. Construct the cleanup owner before preparation and retain namespace files on every unresolved error | Final source rechecked; partial image leases are retained immediately and setup failure invokes cleanup | +| W72 | P3; `tests/docker_environment.rs`, failed observation | Busy-observation failure must not detach the invocation task or observer IO. Always abort/join the call and attempt owner/log shutdown before propagating observation failure | Final source rechecked; successful real busy-cancel qualification uses the same path | +| W73 | P3; cancellation evidence | Any JoinError can include panic. Require is_cancelled for the abandoned invocation task | Explicit cancellation assertion in both real owner lanes | +| W74 | P2; self-review, runtime transport/client shutdown | A preceding protocol error could hide a later transport supervisor failure through Result::and. Worker failure now takes precedence through worker drain and client settlement | Actual scope-error followed by transport-owner abort returns/caches Worker | +| W75 | P2; self-review, log EOF at result | EOF after a valid result should not discard the result, but the environment must not return to warm reuse. Read tail/status for every result and gate reuse independently from LogType | EOF tail/health/reuse regression; raw observed results remain separate from lifecycle settlement | +| W76 | P2; self-review, startup deadline | Valid long caller deadlines were checked against a wire limit before clamping startup. Clamp to the configured bounded startup budget first | Fifteen-minute caller deadline admits/cancels cleanly without daemon mutation | +| W77 | P2; self-review, ready reply abandonment | Once the worker sends a result, losing the call-only cancellation receiver can hide abandonment. Retain an environment stop guard until the result is consumed and deadline-validated | Two public invoke-future poll cases distinguish ready reply abandoned versus successfully consumed | + +Final-source standard workspace build/test/fmt/Clippy/rustdoc pass with2,215 tests, zero failures and403 separately resource-gated cases. Lambda core has535 unit tests and127 doctests; runtime has98 unit tests and48 doctests. The focused target passes20 environment cases and the transport failure-precedence regression. The six-library strict boundary target passes800 unit tests/191 doctests (991 total), with nine separately gated Docker cases, and all strict production checks. Fresh audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources all pass without new exceptions or dependency/version changes. + +Actual owner2, wire2 and shared runtime5 tests explicitly pass with zero ignored on both retained Engine29.5.2 daemons: Linux arm64 VZ and Linux amd64/QEMU. The final owner2 cases are rerun on both after the ready-reply cancellation correction. Owner qualification proves actual official-RIC initialization, computed near6MiB/warm results, function failure, quota rejection, independently observed running-handler cancellation and joined retirement. Read-only post-run queries find no owner/test-labeled containers or internal Lambda aliases. QEMU is not native-x86 platform acceptance; Desktop/rootless and full SDK/executor qualification remain mandatory. + +The owner remains a lower-level runtime consumer. It enforces supplied invocation deadlines and retains cleanup, while exact handler timeout starting at delivery, per-invocation log attribution, warm/LRU checkout limits, startup reconciliation/pulls/network/probes and final provider/executor wiring remain mandatory M0 integration work. No production backend capability is enabled by this change; P2–P5 remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index ffec2de0..fd83d071 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -272,3 +272,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why:** one plan supplies resource admission totals and immutable allocation inputs for the future environment/pool owner. Docker create futures must be Send for supervised Tokio workers, and raw request serializers must leave scope before awaits. A plan neither reserves capacity nor proves startup/reconciliation readiness. - **Pinned by:** [runtime §4.2](./rustack-workflows-20-lambda-runtime-design.md#42-immutable-environment-allocation-plan), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date:** 2026-10-26. No dependency/version change is required. Network none is deliberate pure-handler opt-in; bridge and resource probes remain mandatory for production enablement. + +## D31 — Keep environment ownership above workers and track possible wire issuance + +- **Context:** cancellation, task panic or a failed HTTP reply can outlive the allocation request. Releasing a count/memory/image/directory reservation before actual cleanup permits orphaned or unaccounted execution. +- **Alternatives considered:** worker-local quota guards disappear on panic; immediate404 cannot settle an in-flight/uncertain create; classifying every Stopped/Deadline as unissued can ignore a sent request; treating all protocol errors as uncertain supervision leaks otherwise-settled quota. +- **Decision:** a root actor retains plans, mutation phases, reservations and IO cleanup receipts above its JoinSet. A single RAII wire mutation guard moves with the accepted request, marks possible issuance before send and proves never-issued only after final disposal. Worker guards transfer control/log/bridge ownership to retained cleanup on Drop; root recovery joins workers/IO and reconciles verified IDs. Preserve Worker failure through recovery, independently of physical quota release. +- **Why:** every accepted effect retains an owner, while validation/connect/queue failures can settle without permanent false uncertainty. Health checks reschedule after completion to avoid backlog starvation; log EOF permits an already-observed valid result but forbids reuse. Known protocol failures remain business failures once IO joins; uncertain supervision cannot be masked by an earlier protocol error. +- **Pinned by:** [runtime §4.3](./rustack-workflows-20-lambda-runtime-design.md#43-retained-environment-owner-and-allocation-reservations), [verification](./rustack-workflows-70-verification-plan.md), [implementation](./rustack-workflows-90-impl-plan.md). +- **Date:** 2026-10-26. This low-level owner does not replace provider admission, warm/LRU pooling, startup reconciliation/probes or exact Lambda timeout/log integration. Production Docker stays disabled until those M0 consumers pass. From de91fc06e1ed4ab707a35c22f1d86a1fa30b0929 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 02:16:25 -0700 Subject: [PATCH 024/106] phase 1: retain bounded warm pool dispatch and result ownership Implement runtime section 4.4 and D32: one bounded FIFO queue, canonical validated allocation identity, consumption receipts and deadlines, warm retention limits, one-at-a-time LRU eviction, and retained worker/root recovery. Resolve and independently recheck W78-W81. All standard gates pass with 2,233 workspace tests and 1,014 strict-boundary unit/doc tests. Audit and deny pass. Four actual owner/pool RIC cases pass per arm64 VZ and amd64/QEMU daemon, including warm reuse, revision/capacity isolation, FunctionError replacement and observed busy cancellation. No owned resources remain. M0 still requires exact delivery-based timeout/log attribution, real executor/provider composition, reconciliation/pulls/network/probes; P2-P5 remain required. --- Makefile | 9 + .../src/docker/environment/mod.rs | 43 +- .../src/docker/environment/test_peer.rs | 73 ++ crates/rustack-lambda-core/src/docker/mod.rs | 2 + crates/rustack-lambda-core/src/docker/plan.rs | 67 +- .../src/docker/pool/actor.rs | 454 ++++++++++++ .../src/docker/pool/mod.rs | 348 +++++++++ .../src/docker/pool/tests.rs | 677 ++++++++++++++++++ .../tests/docker_environment.rs | 162 ++++- specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 29 + .../rustack-workflows-60-operations-design.md | 2 +- .../rustack-workflows-70-verification-plan.md | 4 +- specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 17 + specs/rustack-workflows-99-decisions.md | 9 + 16 files changed, 1883 insertions(+), 22 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/environment/test_peer.rs create mode 100644 crates/rustack-lambda-core/src/docker/pool/actor.rs create mode 100644 crates/rustack-lambda-core/src/docker/pool/mod.rs create mode 100644 crates/rustack-lambda-core/src/docker/pool/tests.rs diff --git a/Makefile b/Makefile index 2ee9fc00..e871f93b 100644 --- a/Makefile +++ b/Makefile @@ -889,3 +889,12 @@ test-lambda-environments: @cargo test -p rustack-lambda-runtime --lib test_should_preserve_transport_worker_failure_after_a_prior_scope_error .PHONY: test-lambda-environments + +# Warm checkout ownership, bounded FIFO, LRU and result-consumption regressions. +test-lambda-pool: + @cargo test -p rustack-lambda-core --lib docker::pool::tests + +# Includes the real warm-counter/revision-eviction case and owner regressions. +test-lambda-pool-docker: test-lambda-environments-docker + +.PHONY: test-lambda-pool test-lambda-pool-docker diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index f6c48dee..0fd888ee 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -227,6 +227,13 @@ impl fmt::Debug for EnvironmentHandle { } } impl EnvironmentHandle { + pub(super) fn retirement(&self) -> Retirement { + Retirement { + stop: self.signals.stop.clone(), + status: self.signals.status.subscribe(), + settlement: self.settlement.clone(), + } + } /// Read the fresh immutable environment identity. /// /// ```no_run @@ -407,6 +414,13 @@ impl EnvironmentPort { &self, plan: EnvironmentPlan, deadline: Instant, + ) -> Result { + self.spawn_shared(Arc::new(plan), deadline) + } + pub(super) fn spawn_shared( + &self, + plan: Arc, + deadline: Instant, ) -> Result { let deadline = deadline.min(Instant::now() + self.config.0.start); check_deadline(deadline)?; @@ -420,7 +434,6 @@ impl EnvironmentPort { return Err(DockerError::Quarantined.into()); } let reservation = self.budget.reserve(plan.memory_bytes())?; - let plan = Arc::new(plan); let (calls, mailbox) = mpsc::channel(4); let signals = Signals::new(); let (done, settlement) = watch::channel(None); @@ -464,6 +477,31 @@ impl EnvironmentPort { } } +/// A pool keeps this receipt above workers which own exclusive handles. +#[derive(Debug, Clone)] +pub(super) struct Retirement { + stop: watch::Sender, + status: watch::Receiver, + settlement: watch::Receiver>>, +} +impl Retirement { + pub(super) fn cancel(&self) { + self.stop.send_replace(true); + } + pub(super) async fn settle(mut self) -> Result<(), EnvironmentError> { + self.cancel(); + self.settlement + .wait_for(Option::is_some) + .await + .map_err(|_| EnvironmentError::Worker)? + .clone() + .ok_or(EnvironmentError::Worker)? + } + pub(super) fn removed(&self) -> bool { + self.status.borrow().phase == EnvironmentPhase::Removed + } +} + /// Retained root owner; startup reconciliation/probes and executor pooling are separate. #[non_exhaustive] pub struct EnvironmentManager { @@ -583,3 +621,6 @@ impl Drop for EnvironmentManager { #[cfg(test)] mod tests; + +#[cfg(test)] +pub(in crate::docker) mod test_peer; diff --git a/crates/rustack-lambda-core/src/docker/environment/test_peer.rs b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs new file mode 100644 index 00000000..e294870b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs @@ -0,0 +1,73 @@ +//! Test-only peer for exercising public handle ownership without external processes. +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use rustack_lambda_runtime::session::RuntimeResult; + +use super::*; + +pub(in crate::docker) struct Peer { + calls: mpsc::Receiver, + signals: Signals, + done: watch::Sender>>, +} +pub(in crate::docker) fn pair() -> (EnvironmentHandle, Peer) { + let (calls, mailbox) = mpsc::channel(4); + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (done, settlement) = watch::channel(None); + ( + EnvironmentHandle { + id: EnvironmentId::default(), + calls, + signals: signals.clone(), + settlement, + }, + Peer { + calls: mailbox, + signals, + done, + }, + ) +} +impl Peer { + pub(in crate::docker) async fn serve(self, count: Arc) { + self.serve_inner(count, None, None).await; + } + pub(in crate::docker) async fn serve_gated( + self, + count: Arc, + release: oneshot::Receiver<()>, + stopping: oneshot::Sender<()>, + ) { + self.serve_inner(count, Some(release), Some(stopping)).await; + } + async fn serve_inner( + mut self, + count: Arc, + release: Option>, + stopping: Option>, + ) { + let mut stop = self.signals.stop.subscribe(); + loop { + tokio::select! { biased; + _ = stop.wait_for(|stop| *stop) => break, + call = self.calls.recv() => { + let Some(call) = call else { break; }; + count.fetch_add(1, Ordering::AcqRel); + let _ = call.reply.send(Ok(EnvironmentResponse { result: RuntimeResult::Success(Payload::try_from(Bytes::from_static(b"{}" )).unwrap()), logs: Bytes::new() })); + } + } + } + if let Some(stopping) = stopping { + let _ = stopping.send(()); + } + if let Some(release) = release { + let _ = release.await; + } + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Ok(()))); + } +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 1c3be644..bb8a4ea6 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -13,6 +13,7 @@ mod mutation; mod ownership; mod pin; mod plan; +mod pool; mod types; use std::{ @@ -47,6 +48,7 @@ pub use ownership::{DirectoryLease, DirectoryLeaseError}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; pub use pin::{ImagePin, ImagePinScope}; pub use plan::{ContainerPolicy, EnvironmentPlan, NetworkSelection, RelayBudgets}; +pub use pool::{EnvironmentPool, PoolConfig, PoolPort, PoolSettings, PoolStatus}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs index 5feccc6a..734fec84 100644 --- a/crates/rustack-lambda-core/src/docker/plan.rs +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -10,6 +10,7 @@ use std::{ use rustack_lambda_model::types::ImageConfig; use rustack_lambda_runtime::protocol::EnvironmentId; +use serde::Serialize; use tokio::time::Instant; use uuid::Uuid; @@ -37,7 +38,7 @@ pub(super) enum ContainerRole { } /// Operator network selection; a bridge still requires daemon startup validation. -#[derive(Clone)] +#[derive(Clone, Serialize)] #[non_exhaustive] pub struct NetworkSelection(String); impl fmt::Debug for NetworkSelection { @@ -60,7 +61,7 @@ impl FromStr for NetworkSelection { } /// Integral relay session budgets independent of handler and deployment budgets. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Serialize)] #[non_exhaustive] pub struct RelayBudgets { io: Duration, @@ -103,7 +104,7 @@ impl RelayBudgets { } /// Validated operator limits, independent of user function configuration. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] #[non_exhaustive] pub struct ContainerPolicy { network: NetworkSelection, @@ -166,6 +167,7 @@ pub struct EnvironmentPlan { revision: Arc, identity: Identity, bodies: Bodies, + warm_key: String, } impl fmt::Debug for EnvironmentPlan { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -177,6 +179,9 @@ impl fmt::Debug for EnvironmentPlan { } } impl EnvironmentPlan { + pub(super) fn warm_key(&self) -> &str { + &self.warm_key + } pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { self.directory.owner_id() == directory.owner_id() && self.directory.run_id() == directory.run_id() @@ -283,13 +288,6 @@ impl EnvironmentPlan { relay_body.attach_stdin = Some(true); relay_body.open_stdin = Some(true); relay_body.stdin_once = Some(true); - relay_body.host_config = Some(limits( - policy.network.0, - RELAY_MEMORY, - 100_000_000, - 64, - MIB, - )?); let selected_user = image_user( &function.resolved().metadata().execution.metadata().user, &user, @@ -302,6 +300,15 @@ impl EnvironmentPlan { "function", )?; function_body.env = Some(function_environment(&revision, environment)?); + let warm_key = warm_identity(&directory, &relay, &revision, &policy)?; + relay_body.host_config = Some(limits( + policy.network.0, + RELAY_MEMORY, + 100_000_000, + 64, + MIB, + )?); + let memory = u64::from(revision.memory_mb) .checked_mul(MIB) .ok_or(DockerError::Configuration)?; @@ -327,6 +334,7 @@ impl EnvironmentPlan { function: function_body, memory, }, + warm_key, }) } @@ -450,7 +458,7 @@ impl EnvironmentPlan { ) .await } - fn live(&self) -> Result<(), DockerError> { + pub(super) fn live(&self) -> Result<(), DockerError> { match &self.revision.artifact { DeploymentArtifact::Image(function) if self.relay.is_live() && function.is_live() => { Ok(()) @@ -461,6 +469,43 @@ impl EnvironmentPlan { } } +fn warm_identity( + directory: &DirectoryLease, + relay: &PinnedImage, + revision: &ExecutionRevision, + policy: &ContainerPolicy, +) -> Result { + let function = image(revision)?; + let environment: BTreeMap<_, _> = revision.environment.iter().collect(); + let encoded = serde_json::to_vec(&( + ( + directory.owner_id().to_string(), + directory.run_id().to_string(), + ), + ( + revision.resource_id.to_string(), + &revision.revision_id, + &revision.function_name, + &revision.qualifier, + ), + ( + revision.architecture, + function.resolved().metadata(), + relay.resolved().metadata(), + ), + ( + &revision.image_config, + environment, + revision.timeout, + revision.memory_mb, + revision.ephemeral_storage_mb, + ), + &policy, + )) + .map_err(|_| DockerError::Configuration)?; + Ok(crate::storage::compute_sha256(&encoded)) +} + fn image(revision: &ExecutionRevision) -> Result<&PinnedImage, DockerError> { match &revision.artifact { DeploymentArtifact::Image(image) diff --git a/crates/rustack-lambda-core/src/docker/pool/actor.rs b/crates/rustack-lambda-core/src/docker/pool/actor.rs new file mode 100644 index 00000000..da569fc5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/actor.rs @@ -0,0 +1,454 @@ +//! Pool registry, FIFO dispatch and retained emergency retirement. + +use std::{ + collections::{HashMap, VecDeque}, + mem, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use rustack_lambda_runtime::session::RuntimeResult; +use tokio::{ + runtime::Handle, + sync::{mpsc, oneshot, watch}, + task::{Id, JoinSet}, + time::{Instant, interval, sleep, sleep_until}, +}; +use typed_builder::TypedBuilder; + +use super::{Call, Offer, PoolConfig, PoolStatus}; +use crate::{ + docker::{ + EnvironmentError, EnvironmentHandle, EnvironmentPhase, EnvironmentPort, + connection::stopped, environment::Retirement, + }, + executor::ExecutorError, + invocation::CleanupTask, +}; + +pub(super) struct Warm { + key: String, + environment: EnvironmentHandle, +} +pub(super) struct Completion { + warm: Option, + failed: bool, +} +#[derive(TypedBuilder)] +pub(super) struct Actor { + environments: EnvironmentPort, + config: PoolConfig, + inbox: mpsc::Receiver, + stop: watch::Sender, + failed: Arc, + status: watch::Sender, + emergency: Arc, + runtime: Handle, + #[builder(default)] + pending: VecDeque, + #[builder(default)] + warm: VecDeque, + #[builder(default)] + jobs: JoinSet, + #[builder(default)] + receipts: HashMap, + #[builder(default)] + settled: bool, + #[builder(default)] + eviction: Option, +} +impl Actor { + pub(super) async fn run(mut self) -> Result<(), EnvironmentError> { + let mut stop = self.stop.subscribe(); + let mut tick = interval(Duration::from_millis(100)); + let mut stopping = false; + loop { + if *stop.borrow() || self.inbox.is_closed() { + stopping = true; + } + if stopping { + self.inbox.close(); + while let Ok(call) = self.inbox.try_recv() { + call.reject(self.closed_error()); + } + while let Some(call) = self.pending.pop_front() { + call.reject(self.closed_error()); + } + while self.jobs.len() < 32 { + let Some(warm) = self.warm.pop_front() else { + break; + }; + self.retire(warm.environment); + } + } else { + self.expire(); + self.dispatch(); + } + self.status.send_replace(PoolStatus { + active: self.jobs.len(), + warm: self.warm.len(), + ..PoolStatus::default() + }); + if stopping && self.jobs.is_empty() && self.warm.is_empty() { + self.settled = true; + return if self.failed.load(Ordering::Acquire) { + Err(EnvironmentError::Worker) + } else { + Ok(()) + }; + } + tokio::select! { biased; + () = stopped(&mut stop), if !stopping => { stopping = true; }, + completion = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => { if let Some(completion) = completion { self.completed(completion); } }, + call = self.inbox.recv(), if !stopping => { match call { Some(call) => self.pending.push_back(call), None => stopping = true } }, + _ = tick.tick() => {}, + } + } + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + self.stop.send_replace(true); + } + fn closed_error(&self) -> EnvironmentError { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + } + fn expire(&mut self) { + let mut index = 0; + while index < self.pending.len() { + let error = self.pending.get(index).and_then(Call::error); + if let Some(error) = error { + if let Some(call) = self.pending.remove(index) { + call.reject(error); + } + } else { + index += 1; + } + } + let mut index = 0; + while index < self.warm.len() && self.jobs.len() < 32 { + if self + .warm + .get(index) + .is_some_and(|warm| warm.environment.status().phase != EnvironmentPhase::Idle) + { + if let Some(warm) = self.warm.remove(index) { + self.retire(warm.environment); + } + } else { + index += 1; + } + } + } + fn dispatch(&mut self) { + if self.eviction.is_some() { + return; + } + while self.jobs.len() < 32 { + let Some(mut call) = self.pending.pop_front() else { + return; + }; + if let Some(error) = call.error() { + call.reject(error); + continue; + } + if let Err(error) = call.plan.live() { + call.reject(error.into()); + continue; + } + let key = call.plan.warm_key().to_owned(); + let warm = self + .warm + .iter() + .position(|warm| warm.key == key) + .and_then(|index| self.warm.remove(index)); + let environment = match warm { + Some(warm) => Ok(warm.environment), + None => self + .environments + .spawn_shared(call.plan.clone(), call.deadline()), + }; + match environment { + Ok(environment) => { + call.queue.take(); + let receipt = environment.retirement(); + let stop = self.stop.subscribe(); + let job = self.jobs.spawn(invoke(environment, key, call, stop)); + self.receipts.insert(job.id(), receipt); + } + Err(EnvironmentError::Busy) => { + self.pending.push_front(call); + if let Some(warm) = self.warm.pop_front() { + self.eviction = Some(self.retire(warm.environment)); + } + return; + } + Err(error) => { + call.reject(error); + } + } + } + } + fn retire(&mut self, environment: EnvironmentHandle) -> Id { + let receipt = environment.retirement(); + receipt.cancel(); + drop(environment); + self.retire_receipt(receipt) + } + fn retire_receipt(&mut self, receipt: Retirement) -> Id { + receipt.cancel(); + let job = self.jobs.spawn(retire(receipt.clone())); + self.receipts.insert(job.id(), receipt); + job.id() + } + fn completed(&mut self, result: Result<(Id, Completion), tokio::task::JoinError>) { + let id = match &result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let receipt = self.receipts.remove(&id); + if self.eviction == Some(id) { + self.eviction = None; + } + if let Ok((_, completion)) = result { + if completion.failed { + self.fail(); + } + if let Some(warm) = completion.warm { + let count = self.warm.iter().filter(|idle| idle.key == warm.key).count(); + if !*self.stop.borrow() + && warm.environment.status().phase == EnvironmentPhase::Idle + && self.warm.len() < usize::from(self.config.0.warm) + && count < usize::from(self.config.0.warm_per_revision) + { + self.warm.push_back(warm); + } else { + self.retire(warm.environment); + } + } + } else { + self.fail(); + if let Some(receipt) = receipt { + self.retire_receipt(receipt); + } + } + } +} +impl Drop for Actor { + fn drop(&mut self) { + if !self.settled { + self.fail(); + } + while let Ok(call) = self.inbox.try_recv() { + call.reject(self.closed_error()); + } + for call in self.pending.drain(..) { + call.reject(EnvironmentError::Worker); + } + if self.receipts.is_empty() && self.warm.is_empty() { + return; + } + let mut jobs = mem::take(&mut self.jobs); + let mut receipts: Vec<_> = mem::take(&mut self.receipts).into_values().collect(); + for warm in self.warm.drain(..) { + receipts.push(warm.environment.retirement()); + drop(warm); + } + for receipt in &receipts { + receipt.cancel(); + } + jobs.abort_all(); + self.emergency.start_on(&self.runtime, move || async move { + while jobs.join_next().await.is_some() {} + recover(receipts).await + }); + } +} +async fn recover(receipts: Vec) -> Result<(), ExecutorError> { + let mut cleanup = JoinSet::new(); + let mut index = HashMap::new(); + let mut failed = false; + for receipt in receipts { + let job = cleanup.spawn(retire(receipt.clone())); + index.insert(job.id(), receipt); + } + while let Some(result) = cleanup.join_next_with_id().await { + let id = match &result { + Ok((id, _)) => *id, + Err(error) => error.id(), + }; + let Some(receipt) = index.remove(&id) else { + continue; + }; + if let Ok((_, completion)) = result { + failed |= completion.failed; + if receipt.removed() { + continue; + } + } else { + failed = true; + } + let job = cleanup.spawn(retire(receipt.clone())); + index.insert(job.id(), receipt); + } + if failed { + Err(ExecutorError::CleanupFailed( + "Pool recovery supervision failed".into(), + )) + } else { + Ok(()) + } +} +async fn retire(receipt: Retirement) -> Completion { + let mut failed = false; + loop { + let result = receipt.clone().settle().await; + failed |= result.is_err(); + if receipt.removed() { + return Completion { warm: None, failed }; + } + sleep(Duration::from_secs(1)).await; + } +} +async fn invoke( + mut environment: EnvironmentHandle, + key: String, + call: Call, + mut stop: watch::Receiver, +) -> Completion { + let deadline = call.deadline(); + let mut cancellation = call.cancellation.clone(); + let operation = async { + if let Some(error) = call.error() { + return Err(error); + } + environment + .wait_ready(deadline.min(Instant::now() + Duration::from_secs(60))) + .await?; + environment + .invoke(call.invocation.clone(), call.capture_logs) + .await + }; + let result = tokio::select! { biased; + () = stopped(&mut stop) => Err(EnvironmentError::Stopped), + () = stopped(&mut cancellation) => Err(EnvironmentError::Cancelled), + () = sleep_until(deadline) => Err(EnvironmentError::Deadline), + result = operation => result, + }; + if matches!(&result, Ok(response) if matches!(response.result, RuntimeResult::Success(_))) + && call.error().is_none() + && !*stop.borrow() + { + let (receipt, consumed) = oneshot::channel(); + let sent = call + .reply + .send(Offer { + result, + receipt: Some(receipt), + }) + .is_ok(); + if sent { + let accepted = tokio::select! { biased; + () = stopped(&mut stop) => false, + () = sleep_until(deadline) => false, + result = consumed => result.is_ok() && Instant::now() < deadline, + () = stopped(&mut cancellation) => false, + }; + if accepted { + return Completion { + warm: Some(Warm { key, environment }), + failed: false, + }; + } + } + let receipt = environment.retirement(); + drop(environment); + retire(receipt).await + } else { + let receipt = environment.retirement(); + drop(environment); + let completion = retire(receipt).await; + let _ = call.reply.send(Offer { + result, + receipt: None, + }); + completion + } +} + +#[cfg(test)] +pub(super) fn testing_start( + environments: EnvironmentPort, + config: PoolConfig, + warm: Vec<(String, EnvironmentHandle)>, +) -> super::EnvironmentPool { + testing_start_inner(environments, config, warm, false) +} +#[cfg(test)] +pub(super) fn testing_failed_worker_start( + environments: EnvironmentPort, + warm: Vec<(String, EnvironmentHandle)>, +) -> super::EnvironmentPool { + testing_start_inner(environments, PoolConfig::default(), warm, true) +} +#[cfg(test)] +fn testing_start_inner( + environments: EnvironmentPort, + config: PoolConfig, + warm: Vec<(String, EnvironmentHandle)>, + failed_worker: bool, +) -> super::EnvironmentPool { + let runtime = Handle::current(); + let maximum = usize::from(config.0.queue); + let (sender, inbox) = mpsc::channel(maximum); + let (stop, _) = watch::channel(false); + let failed = Arc::new(AtomicBool::new(false)); + let (status, observed) = watch::channel(PoolStatus::default()); + let emergency = Arc::new(CleanupTask::default()); + let mut actor = Actor::builder() + .environments(environments) + .config(config) + .inbox(inbox) + .stop(stop.clone()) + .failed(failed.clone()) + .status(status) + .emergency(emergency.clone()) + .runtime(runtime.clone()) + .warm( + warm.into_iter() + .map(|(key, environment)| Warm { key, environment }) + .collect(), + ) + .build(); + if failed_worker { + let warm = actor.warm.pop_front().unwrap(); + let receipt = warm.environment.retirement(); + let job = actor.jobs.spawn(async move { + let _owned = warm; + std::future::pending::().await + }); + actor.receipts.insert(job.id(), receipt); + job.abort(); + } + let port = super::PoolPort { + sender, + queue: Arc::new(tokio::sync::Semaphore::new(maximum)), + maximum, + stop, + failed, + status: observed, + }; + super::EnvironmentPool { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + } +} diff --git a/crates/rustack-lambda-core/src/docker/pool/mod.rs b/crates/rustack-lambda-core/src/docker/pool/mod.rs new file mode 100644 index 00000000..0246027f --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/mod.rs @@ -0,0 +1,348 @@ +//! Bounded warm dispatch; external resource ownership stays with `EnvironmentManager`. + +mod actor; + +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use rustack_lambda_runtime::session::Invocation; +use tokio::{ + runtime::Handle, + sync::{Semaphore, mpsc, oneshot, watch}, + task::JoinHandle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + EnvironmentError, EnvironmentPlan, EnvironmentPort, EnvironmentResponse, check_deadline, +}; +use crate::invocation::CleanupTask; + +/// Operator limits for waiting calls and retained idle environments. +#[derive(Debug, Clone, Copy, TypedBuilder)] +#[non_exhaustive] +pub struct PoolSettings { + /// Globally retained idle environments,0..32. + #[builder(default = 32)] + pub warm: u8, + /// Idle environments per immutable identity,0..32. + #[builder(default = 1)] + pub warm_per_revision: u8, + /// Accepted waiting calls across mailbox and actor queue,1..256. + #[builder(default = 256)] + pub queue: u16, +} +impl Default for PoolSettings { + fn default() -> Self { + Self::builder().build() + } +} +/// Validated warm retention and admission limits. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct PoolConfig(PoolSettings); +impl TryFrom for PoolConfig { + type Error = EnvironmentError; + fn try_from(settings: PoolSettings) -> Result { + if settings.warm > 32 + || settings.warm_per_revision > 32 + || !(1..=256).contains(&settings.queue) + { + return Err(EnvironmentError::Configuration); + } + Ok(Self(settings)) + } +} +/// Read-only pool projection; environment accounting includes quarantine separately. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct PoolStatus { + /// Accepted calls still waiting for dispatch. + pub queued: usize, + /// Invocation/retirement jobs; at most32. + pub active: usize, + /// Idle handles retained for reuse. + pub warm: usize, + /// Whether admission remains open and healthy. + pub accepting: bool, + /// A worker/root supervision failure closed admission. + pub worker_failed: bool, +} +struct Offer { + result: Result, + receipt: Option>, +} +struct Call { + plan: Arc, + invocation: Invocation, + capture_logs: bool, + cancellation: watch::Receiver, + reply: oneshot::Sender, + queue: Option, +} +impl Call { + fn deadline(&self) -> Instant { + self.invocation.parts().1 + } + fn error(&self) -> Option { + if Instant::now() >= self.deadline() { + Some(EnvironmentError::Deadline) + } else if self.reply.is_closed() + || *self.cancellation.borrow() + || self.cancellation.has_changed().is_err() + { + Some(EnvironmentError::Cancelled) + } else { + None + } + } + fn reject(self, error: EnvironmentError) { + let _ = self.reply.send(Offer { + result: Err(error), + receipt: None, + }); + } +} +struct CancelOnDrop { + stop: watch::Sender, + consumed: bool, +} +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if !self.consumed { + self.stop.send_replace(true); + } + } +} +/// Cloneable call port; the retained pool owner must outlive its shutdown. +#[derive(Clone)] +#[non_exhaustive] +pub struct PoolPort { + sender: mpsc::Sender, + queue: Arc, + maximum: usize, + stop: watch::Sender, + failed: Arc, + status: watch::Receiver, +} +impl fmt::Debug for PoolPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PoolPort") + .field("status", &self.status()) + .finish() + } +} +impl PoolPort { + /// Execute once through a matching warm environment or bounded cold allocation. + /// + /// # Errors + /// Returns queue overload, deadline, cancellation, initialization or runtime + /// failure. Dropping the future signals retained cleanup without replay. + /// + /// ```no_run + /// # async fn invoke(port: rustack_lambda_core::docker::PoolPort, plan: rustack_lambda_core::docker::EnvironmentPlan, call: rustack_lambda_runtime::session::Invocation) { + /// let result = port.invoke(plan, call, false).await; + /// # } + /// ``` + pub async fn invoke( + &self, + plan: EnvironmentPlan, + invocation: Invocation, + capture_logs: bool, + ) -> Result { + let deadline = invocation.parts().1; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if !self.status().accepting { + return Err(self.closed_error()); + } + plan.live()?; + let queue = self + .queue + .clone() + .try_acquire_owned() + .map_err(|_| EnvironmentError::Busy)?; + let (stop, cancellation) = watch::channel(false); + let mut cancel = CancelOnDrop { + stop, + consumed: false, + }; + let (reply, response) = oneshot::channel(); + self.sender + .try_send(Call { + plan: Arc::new(plan), + invocation, + capture_logs, + cancellation, + reply, + queue: Some(queue), + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, + mpsc::error::TrySendError::Closed(_) => self.closed_error(), + })?; + let offer = timeout_at(deadline, response) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| self.closed_error())?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + if let Some(receipt) = offer.receipt { + let _ = receipt.send(()); + } + cancel.consumed = true; + offer.result + } + fn closed_error(&self) -> EnvironmentError { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + } + /// Read pool counts without waiting for the business mailbox. + /// + /// ```no_run + /// # fn status(port: &rustack_lambda_core::docker::PoolPort) { let counts = port.status(); } + /// ``` + #[must_use] + pub fn status(&self) -> PoolStatus { + let mut status = *self.status.borrow(); + status.queued = self.maximum.saturating_sub(self.queue.available_permits()); + status.worker_failed = self.failed.load(Ordering::Acquire); + status.accepting = !*self.stop.borrow() && !status.worker_failed; + status + } +} +/// Retained pool root, shut down before the environment and image owners. +#[non_exhaustive] +pub struct EnvironmentPool { + port: PoolPort, + task: Option>>, + primary: Option>, + settled: Option>, + emergency: Arc, +} +impl fmt::Debug for EnvironmentPool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EnvironmentPool") + .field("port", &self.port) + .field("settled", &self.settled) + .finish_non_exhaustive() + } +} +impl EnvironmentPool { + /// Start bounded checkout over an already-owned environment service. + /// + /// # Errors + /// Returns Worker when no Tokio runtime is available. This does not establish + /// Docker startup readiness or enable the production executor. + /// + /// ```no_run + /// # fn start(environments: rustack_lambda_core::docker::EnvironmentPort) { + /// let pool = rustack_lambda_core::docker::EnvironmentPool::start(environments, Default::default()); + /// # } + /// ``` + pub fn start( + environments: EnvironmentPort, + config: PoolConfig, + ) -> Result { + let runtime = Handle::try_current().map_err(|_| EnvironmentError::Worker)?; + let maximum = usize::from(config.0.queue); + let (sender, inbox) = mpsc::channel(maximum); + let (stop, _) = watch::channel(false); + let failed = Arc::new(AtomicBool::new(false)); + let (status, observed) = watch::channel(PoolStatus::default()); + let emergency = Arc::new(CleanupTask::default()); + let actor = actor::Actor::builder() + .environments(environments) + .config(config) + .inbox(inbox) + .stop(stop.clone()) + .failed(failed.clone()) + .status(status) + .emergency(emergency.clone()) + .runtime(runtime.clone()) + .build(); + let port = PoolPort { + sender, + queue: Arc::new(Semaphore::new(maximum)), + maximum, + stop, + failed, + status: observed, + }; + Ok(Self { + port, + task: Some(runtime.spawn(actor.run())), + primary: None, + settled: None, + emergency, + }) + } + /// Clone the call port without transferring root shutdown ownership. + /// + /// ```no_run + /// # fn port(pool: &rustack_lambda_core::docker::EnvironmentPool) { let port = pool.port(); } + /// ``` + #[must_use] + pub fn port(&self) -> PoolPort { + self.port.clone() + } + /// Close admission, retire active/idle environments and join retained cleanup. + /// + /// # Errors + /// Returns timeout while cleanup stays owned, or a cached supervision failure. + /// A cancelled waiter does not lose the join or emergency cleanup receipt. + /// + /// ```no_run + /// # async fn stop(pool: &mut rustack_lambda_core::docker::EnvironmentPool) { + /// let clean = pool.shutdown(tokio::time::Instant::now()+std::time::Duration::from_secs(60)).await; + /// # } + /// ``` + pub async fn shutdown(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + self.port.stop.send_replace(true); + check_deadline(deadline)?; + if let Some(settled) = &self.settled { + return settled.clone(); + } + if self.primary.is_none() { + let result = match self.task.as_mut() { + Some(task) => timeout_at(deadline, task) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker) + .and_then(|result| result), + None => Err(EnvironmentError::Worker), + }; + self.task.take(); + self.primary = Some(result); + } + timeout_at(deadline, self.emergency.wait()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline); + } + let result = self.primary.clone().ok_or(EnvironmentError::Worker)?; + self.settled = Some(result.clone()); + result + } +} +impl Drop for EnvironmentPool { + fn drop(&mut self) { + self.port.stop.send_replace(true); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/pool/tests.rs b/crates/rustack-lambda-core/src/docker/pool/tests.rs new file mode 100644 index 00000000..17f70dcb --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/tests.rs @@ -0,0 +1,677 @@ +use std::time::Duration; + +use super::*; +use crate::docker::{ + DirectoryLease, EnvironmentConfig, EnvironmentManager, RelayBudgets, + plan::tests::{image_with, policy, revision}, + tests::Daemon, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn directory() -> DirectoryLease { + DirectoryLease::testing(uuid::Uuid::new_v4(), uuid::Uuid::new_v4()) +} +fn plan(directory: &DirectoryLease) -> EnvironmentPlan { + EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap() +} +fn invocation(until: Instant) -> Invocation { + Invocation::new( + rustack_lambda_runtime::protocol::AttemptId::default(), + until, + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture" + .parse() + .unwrap(), + rustack_lambda_runtime::metadata::local_trace_id().unwrap(), + rustack_lambda_runtime::protocol::Payload::try_from(bytes::Bytes::from_static(b"{}")) + .unwrap(), + ) + .unwrap() +} +#[test] +fn test_should_reject_invalid_pool_limits_and_allow_disabled_warm_retention() { + for settings in [ + PoolSettings::builder().warm(33).build(), + PoolSettings::builder().warm_per_revision(33).build(), + PoolSettings::builder().queue(0).build(), + PoolSettings::builder().queue(257).build(), + ] { + assert!(PoolConfig::try_from(settings).is_err()); + } + assert!( + PoolConfig::try_from(PoolSettings::builder().warm(0).warm_per_revision(0).build()).is_ok() + ); +} +#[tokio::test] +async fn test_should_reject_queued_call_after_pool_stop_without_allocation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let port = pool.port(); + pool.shutdown(deadline()).await.unwrap(); + assert!(matches!( + port.invoke(plan(&lease), invocation(deadline()), false) + .await, + Err(EnvironmentError::Stopped) + )); + assert_eq!(environments.port().status().allocated, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_cache_root_failure_and_release_queued_permits_without_mutation() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + pool.task.as_ref().unwrap().abort(); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(pool.port().status().worker_failed); + assert_eq!(environments.port().status().allocated, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_key_allocation_values_without_fresh_identities_or_hashmap_order() { + let lease = directory(); + let original = revision(); + let make = |revision, policy| { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + Arc::new(revision), + policy, + ) + .unwrap() + }; + let first = make(original.clone(), policy()); + let mut reordered = original.clone(); + let mut entries: Vec<_> = original.environment.iter().collect(); + entries.sort(); + reordered.environment = entries + .into_iter() + .rev() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let second = make(reordered, policy()); + assert_ne!(first.environment_id(), second.environment_id()); + assert_eq!(first.warm_key(), second.warm_key()); + let mut changed = original.clone(); + changed + .environment + .insert("TEST_VALUE".into(), "another-secret".into()); + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let mut changed = original.clone(); + changed.memory_mb = 256; + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let mut changed = original.clone(); + changed.revision_id = uuid::Uuid::new_v4().to_string(); + assert_ne!(first.warm_key(), make(changed, policy()).warm_key()); + let another_policy = crate::docker::ContainerPolicy::new( + "none".parse().unwrap(), + 1001, + 1000, + 1_000_000_000, + RelayBudgets::default(), + ) + .unwrap(); + assert_ne!( + first.warm_key(), + make(original.clone(), another_policy).warm_key() + ); + assert_ne!( + first.warm_key(), + EnvironmentPlan::new( + directory(), + image_with("1000", vec![]), + Arc::new(original), + policy() + ) + .unwrap() + .warm_key() + ); + assert!(!format!("{first:?}").contains("secret-value")); +} +async fn idle(port: &PoolPort) { + tokio::time::timeout(Duration::from_secs(3), async { + while port.status().active != 0 || port.status().warm != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +#[tokio::test] +async fn test_should_reuse_only_consumed_warm_results_and_join_idle_shutdown() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let count = Arc::new(AtomicUsize::new(0)); + let served = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(make().warm_key().into(), handle)], + ); + let port = pool.port(); + for expected in 1..=3 { + port.invoke(make(), invocation(deadline()), false) + .await + .unwrap(); + idle(&port).await; + assert_eq!(count.load(Ordering::Acquire), expected); + } + pool.shutdown(deadline()).await.unwrap(); + served.await.unwrap(); + assert_eq!(port.status().warm, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case::abandoned(true)] +#[case::live_but_unpolled(false)] +#[tokio::test] +async fn test_should_retire_unconsumed_ready_reply_even_without_public_future_polling( + #[case] abandoned: bool, +) { + use std::{future::Future, sync::atomic::AtomicUsize, task::Poll}; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let key = plan.warm_key().to_owned(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let served = tokio::spawn(peer.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(key, handle)], + ); + let port = pool.port(); + let mut call = Box::pin(port.invoke( + plan, + invocation(Instant::now() + Duration::from_millis(250)), + false, + )); + assert!(matches!( + call.as_mut() + .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())), + Poll::Pending + )); + tokio::time::sleep(Duration::from_millis(20)).await; + if abandoned { + drop(call); + } else { + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + receipt.removed(), + "unpolled waiter retained its environment past deadline" + ); + assert!(matches!( + call.as_mut() + .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())), + Poll::Ready(Err(EnvironmentError::Deadline)) + )); + } + tokio::time::timeout(Duration::from_secs(2), served) + .await + .unwrap() + .unwrap(); + assert!(receipt.removed()); + pool.shutdown(deadline()).await.unwrap(); + assert_eq!(port.status().warm, 0); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_bound_one_queue_across_mailbox_and_waiting_dispatch_then_cancel_without_replay() + { + use std::{ + future::Future, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start( + environments.port(), + PoolSettings::builder().queue(1).build().try_into().unwrap(), + ) + .unwrap(); + let port = pool.port(); + let mut waiting = Box::pin(port.invoke(plan(&lease), invocation(deadline()), false)); + assert!(matches!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::sleep(Duration::from_millis(120)).await; + assert_eq!(port.status().queued, 1); + assert_eq!(port.status().active, 0); + assert!(matches!( + port.invoke(plan(&lease), invocation(deadline()), false) + .await, + Err(EnvironmentError::Busy) + )); + drop(waiting); + tokio::time::timeout(Duration::from_secs(2), async { + while port.status().queued != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(daemon.requests.try_recv().is_err()); + pool.shutdown(deadline()).await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::idle(false)] +#[case::offered_result(true)] +#[tokio::test] +async fn test_should_recover_idle_or_dispatched_environment_after_root_abort( + #[case] offered: bool, +) { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let key = plan.warm_key().into(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let served = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(key, handle)], + ); + let port = pool.port(); + let mut call = Box::pin(port.invoke(plan, invocation(deadline()), false)); + if offered { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::timeout(Duration::from_secs(2), async { + while count.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + pool.task.as_ref().unwrap().abort(); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(receipt.removed()); + served.await.unwrap(); + drop(call); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_one_lru_eviction_barrier_while_cleanup_is_delayed() { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let (oldest, first) = crate::docker::environment::test_peer::pair(); + let first_receipt = oldest.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let first_task = tokio::spawn(first.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let (next, second) = crate::docker::environment::test_peer::pair(); + let second_receipt = next.retirement(); + let second_task = tokio::spawn(second.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![("oldest".into(), oldest), ("next".into(), next)], + ); + let port = pool.port(); + let mut waiting = Box::pin(port.invoke(plan(&lease), invocation(deadline()), false)); + assert!(matches!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + tokio::time::timeout(Duration::from_secs(2), stopped) + .await + .unwrap() + .unwrap(); + tokio::time::sleep(Duration::from_millis(350)).await; + assert!(!first_receipt.removed()); + assert!(!second_receipt.removed()); + assert_eq!(port.status().warm, 1); + assert_eq!(port.status().active, 1); + drop(waiting); + release.send(()).unwrap(); + pool.shutdown(deadline()).await.unwrap(); + first_task.await.unwrap(); + second_task.await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_keep_retirement_owned_when_shutdown_waiter_is_cancelled() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let served = tokio::spawn(peer.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![("idle".into(), handle)], + ); + assert!( + tokio::time::timeout(Duration::from_millis(30), pool.shutdown(deadline())) + .await + .is_err() + ); + stopped.await.unwrap(); + assert!(!receipt.removed()); + release.send(()).unwrap(); + pool.shutdown(deadline()).await.unwrap(); + pool.shutdown(deadline()).await.unwrap(); + served.await.unwrap(); + assert!(receipt.removed()); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[test] +fn test_should_reject_oversized_captured_inputs_before_warm_fingerprinting() { + let lease = directory(); + let mut invalid = revision(); + invalid + .environment + .insert("OVERSIZED".into(), "s".repeat(8192)); + assert!( + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + Arc::new(invalid), + policy() + ) + .is_err() + ); + let mut invalid = revision(); + invalid.image_config = Some(rustack_lambda_model::types::ImageConfig { + command: Some(vec!["x".repeat(32768)]), + ..Default::default() + }); + assert!( + EnvironmentPlan::new( + lease, + image_with("1000", vec![]), + Arc::new(invalid), + policy() + ) + .is_err() + ); +} + +#[tokio::test] +async fn test_should_preserve_retirement_above_an_aborted_dispatch_worker() { + use std::sync::atomic::AtomicUsize; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease, EnvironmentConfig::default()).unwrap(); + let (handle, peer) = crate::docker::environment::test_peer::pair(); + let receipt = handle.retirement(); + let served = tokio::spawn(peer.serve(Arc::new(AtomicUsize::new(0)))); + let mut pool = + actor::testing_failed_worker_start(environments.port(), vec![("owned".into(), handle)]); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(pool.port().status().worker_failed); + assert!(receipt.removed()); + served.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_wait_for_capacity_in_fifo_order_and_never_replay_rejected_startup() { + use std::{ + future::Future, + task::{Context, Poll, Waker}, + }; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let config = crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(); + let mut environments = EnvironmentManager::start(wire.port(), lease.clone(), config).unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let port = pool.port(); + let first = plan(&lease); + let first_key = first.relay_key().as_str().to_owned(); + let second = plan(&lease); + let second_key = second.relay_key().as_str().to_owned(); + let mut first = Box::pin(port.invoke(first, invocation(deadline()), false)); + let mut second = Box::pin(port.invoke(second, invocation(deadline()), false)); + for call in [&mut first, &mut second] { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + } + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + for key in [first_key, second_key] { + let request = daemon.next().await; + assert!( + percent_encoding::percent_decode_str(&request.head) + .decode_utf8() + .unwrap() + .contains(&key) + ); + request.reply(400, b"{}").await; + } + assert!(first.await.is_err()); + assert!(second.await.is_err()); + pool.shutdown(deadline()).await.unwrap(); + assert!(daemon.requests.try_recv().is_err()); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[rstest::rstest] +#[case::no_global_retention(0, 1, 0)] +#[case::no_revision_retention(32, 0, 0)] +#[case::one_per_revision(32, 1, 1)] +#[tokio::test] +async fn test_should_retire_successful_excess_concurrent_returns( + #[case] warm: u8, + #[case] per_revision: u8, + #[case] retained: usize, +) { + use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Poll, Waker}, + }; + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let key = make().warm_key().to_owned(); + let count = Arc::new(AtomicUsize::new(0)); + let mut seeded = Vec::new(); + let mut peers = Vec::new(); + for _ in 0..2 { + let (handle, peer) = crate::docker::environment::test_peer::pair(); + seeded.push((key.clone(), handle)); + peers.push(tokio::spawn(peer.serve(count.clone()))); + } + let config = PoolSettings::builder() + .warm(warm) + .warm_per_revision(per_revision) + .build() + .try_into() + .unwrap(); + let mut pool = actor::testing_start(environments.port(), config, seeded); + let port = pool.port(); + let mut first = Box::pin(port.invoke(make(), invocation(deadline()), false)); + let mut second = Box::pin(port.invoke(make(), invocation(deadline()), false)); + for call in [&mut first, &mut second] { + assert!(matches!( + call.as_mut().poll(&mut Context::from_waker(Waker::noop())), + Poll::Pending + )); + } + tokio::time::timeout(Duration::from_secs(2), async { + while count.load(Ordering::Acquire) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + first.await.unwrap(); + second.await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), async { + while port.status().active != 0 || port.status().warm != retained { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + pool.shutdown(deadline()).await.unwrap(); + for peer in peers { + peer.await.unwrap(); + } + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs index ae76e2ad..68bf972e 100644 --- a/crates/rustack-lambda-core/tests/docker_environment.rs +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -9,8 +9,9 @@ use rustack_lambda_core::{ artifact::{Architecture, DeploymentArtifact, PinnedImage}, docker::{ ContainerPolicy, DirectoryLease, DockerWire, EnvironmentConfig, EnvironmentError, - EnvironmentHandle, EnvironmentManager, EnvironmentPlan, EnvironmentSettings, FunctionLogs, - ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OwnerFilter, RelayBudgets, + EnvironmentHandle, EnvironmentManager, EnvironmentPlan, EnvironmentPool, + EnvironmentSettings, FunctionLogs, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, + OwnerFilter, PoolConfig, RelayBudgets, }, executor::{ExecutionRevision, ExecutorBackend}, identity::FunctionResourceId, @@ -111,7 +112,7 @@ impl Fixture { Ok(()) } - fn plan(&self) -> Result { + fn revision(&self) -> Result> { let revision = ExecutionRevision::builder() .resource_id(FunctionResourceId::new()) .revision_id(Uuid::new_v4().to_string()) @@ -140,10 +141,16 @@ impl Fixture { .memory_mb(192) .ephemeral_storage_mb(512) .build(); + Ok(Arc::new(revision)) + } + fn plan(&self) -> Result { + self.plan_revision(self.revision()?) + } + fn plan_revision(&self, revision: Arc) -> Result { Ok(EnvironmentPlan::new( self.lease.clone(), self.relay.as_ref().context("relay image")?.clone(), - Arc::new(revision), + revision, ContainerPolicy::new( "none".parse()?, 1000, @@ -275,3 +282,150 @@ async fn test_should_retire_running_code_after_abandoned_call_and_join_all_owned let cleanup = fixture.cleanup().await; result.and(cleanup) } + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_pool_real_warm_calls_and_evict_capacity_across_revision_changes() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let original = fixture.revision()?; + let port = pool.port(); + for expected in 1..=2 { + let response = port + .invoke( + fixture.plan_revision(original.clone())?, + invocation(&json!({"value":35}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("unexpected function failure"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["computed"] == 42 && value["counter"] == expected, + "pool did not preserve warm counter" + ); + } + let mut changed = (*original).clone(); + changed.revision_id = Uuid::new_v4().to_string(); + changed + .environment + .insert("TEST_VALUE".into(), "changed".into()); + let changed = Arc::new(changed); + for revision in [changed, original.clone()] { + let response = port + .invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":0,"policy":true}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("unexpected function failure"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["computed"] == 7 && value["counter"] == 1, + "revision or evicted environment was reused" + ); + } + let failed = port + .invoke( + fixture.plan_revision(original.clone())?, + invocation(&json!({"fail":true}))?, + false, + ) + .await?; + let RuntimeResult::FunctionError(payload) = failed.result else { + anyhow::bail!("pool hid FunctionError"); + }; + let error: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + error["errorType"] == "ValueError", + "pool error payload differs" + ); + let response = port + .invoke( + fixture.plan_revision(original)?, + invocation(&json!({"value":0}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("replacement invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!( + value["counter"] == 1, + "failed environment was returned to warm pool" + ); + Ok(()) + } + .await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_join_pool_retirement_after_abandoning_an_observed_running_handler() +-> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let revision = fixture.revision()?; + let plan = fixture.plan_revision(revision.clone())?; + let key = plan.function_key().clone(); + let port = pool.port(); + let initial = port + .invoke(plan, invocation(&json!({"value":0}))?, false) + .await?; + ensure!( + matches!(initial.result, RuntimeResult::Success(_)), + "warm setup failed" + ); + let plan = fixture.plan_revision(revision)?; + let invocation = invocation(&json!({"value":1,"sleep":true}))?; + let mut observed = FunctionLogs::start( + fixture.wire.port().attach_logs(&key, deadline()).await?, + Duration::from_secs(60), + 262_144, + )?; + let call = tokio::spawn(async move { port.invoke(plan, invocation, false).await }); + let observation = timeout(Duration::from_secs(30), async { + loop { + let snapshot = observed.snapshot(deadline()).await?; + if snapshot + .tail + .windows(14) + .any(|part| part == b"P0_BUSY_START=") + { + break Ok::<(), anyhow::Error>(()); + } + tokio::task::yield_now().await; + } + }) + .await + .context("pooled handler did not start"); + call.abort(); + let joined = call.await; + let retired = pool.shutdown(deadline()).await; + let logs = observed.shutdown().await; + observation??; + ensure!( + matches!(joined,Err(ref error) if error.is_cancelled()), + "pooled invocation task was not cancelled" + ); + retired?; + logs?; + Ok(()) + } + .await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} diff --git a/specs/index.md b/specs/index.md index fcf10414..4c352288 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership and bounded FIFO/warm/LRU pool dispatch. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until pool/executor integration, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until executor integration, exact invocation timing/log attribution, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 8fb73193..31fcfffd 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -542,6 +542,35 @@ Provider / future Pool Environment root Environment worker │ │ retain records until proven clean │ ``` +### 4.4 Warm checkout and retained pool dispatch + +`EnvironmentPool` owns a bounded FIFO of accepted calls, a JoinSet of at most32 invocation/retirement jobs and an LRU of exclusive idle EnvironmentHandles. It consumes validated EnvironmentPlans and runtime Invocations through a cloneable PoolPort; executor/provider wiring remains a required consumer. PoolConfig bounds globally retained warm environments0..32, warm retention per identity0..32 (default1), and accepted waiting calls1..256 (default256). The EnvironmentManager remains the authoritative allocation/memory/quarantine owner. A separate queue permit spans channel admission and the actor's pending queue, and releases only on dispatch/rejection. Calls cannot build an unbounded second queue behind the mailbox. + +Warm identity is a SHA-256 of canonical typed allocation inputs: directory owner/run, resource/revision/name/resolved version/backend, architecture, function and relay resolved image metadata, complete ImageConfig, sorted captured environment, timeout/memory/ephemeral storage and operator container policy/relay budgets. Hash raw validated values, never redacted Debug output. Equivalent HashMap insertion orders match. Fresh environment UUIDs/names/log streams are excluded; configuration, image, scope or policy changes do not share environments. Construct the identity before admission and recheck live image leases on dispatch, including the warm path. + +The actor services stop and completions before new admissions and periodically expires cancelled/deadline-exhausted calls, reaps dead idle handles and revisits capacity. Dispatch FIFO head to a matching healthy Idle handle or perform one cold allocation. Known no-effect Busy admission may wait and retry; never retry a successfully admitted environment's startup or invocation. If another identity occupies idle capacity, retire the least-recently-used idle handle, await its settlement, then retry allocation. Do not evict busy environments or release their authoritative reservation early. Idle environments retain the manager's independent idle deadline. Per-identity/global warm limits apply on successful return; excess environments retire. A retired handle's identity is not transferable to another function. + +Every dispatched call has a separate cancellation lane. Caller timeout/Drop, including a ready-but-unconsumed result, retires its environment. After a successful runtime result the worker waits for the public receiver's synchronous consumption receipt before returning an Idle handle to the LRU. Keep the absolute deadline and cancellation authoritative through offering and receipt waiting even when the public future remains alive but unpolled; worker-side expiry retires that environment. Receipt readiness precedes sender-closed cancellation because successful consumption synchronously acknowledges before dropping its cancellation sender. A valid FunctionError remains a business result but its environment retires. An unsuccessful invocation is not replayed. A waiting public future does not own cleanup: accepted dispatch remains with the actor until resources are reusable or retired. Supplied runtime absolute deadlines govern queue/init/invoke; delivery-based handler timeout and invocation log attribution still require the executor/runtime integration specified in §3. + +Before spawning any environment task, retain a cloneable retirement receipt (priority stop, status and cached settlement) above the JoinSet. Worker panic aborts admission and retires that receipt even though its exclusive handle was dropped. Root Drop transfers the JoinSet, all receipts and idle handles into captured-runtime emergency cleanup; abort/join workers before awaiting environment retirement. Normal shutdown closes admission, rejects queued calls, cancels dispatched calls and joins every active/idle retirement. Cancelled/timed-out shutdown waiters retain that operation; cached worker failure cannot become success after physical cleanup. Underlying quarantined resources remain accounted by EnvironmentManager and prevent clean pool shutdown. Infrastructure diagnostics never include payload, environment variables or unhashed allocation inputs. + +```text +Caller Pool actor / retained registry Environment owner + │ queue permit + plan + call │ │ + │────────────────────────────────▶│ FIFO + immutable warm identity │ + │ │ matching Idle ──────┐ │ + │ │ cold spawn ────────┼──────────▶│ reserve/create + │ │ retain receipt │ │ + │ │ supervised call ◀──┘ │ + │◀──────── offered result ────────│ │ + │ consumed receipt ──────────────▶│ success → bounded warm LRU │ + │ Drop / expiry ─────────────────▶│ stop → join retirement ───────▶│ retained cleanup + │ │ worker/root panic │ + │ │ abort/join → receipt cleanup ─▶│ quarantine/settle +``` + +Tests must pin identity isolation/order normalization, queue+active bounds, FIFO capacity waiting, LRU and per-identity retention, dead idle handles, cancellation before/after dispatch and at ready-result receipt, no replay after startup failure, root/worker abort recovery and cancelled/repeated shutdown. Actual image-RIC qualification must prove warm counter reuse, independent-function/revision cold counters, bounded capacity eviction and no owned allocations after joined shutdown. This pool does not establish startup reconciliation, registry/network/resource probes or production backend readiness. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 45d0ef11..6498a0b3 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -123,7 +123,7 @@ Expose capability fields for workflow type/query profile/state types/integration The workflow drain interval permits already accepted executions to finish. Then cancel remaining executions with ABORTED and error=Rustack.Shutdown, preserving terminal history. This is deliberate local shutdown behavior, not cloud-style pause/resume. Revoke callback tokens, cancel owned children and await Lambda cleanup before Lambda itself quiesces. Quarantine or unfinished cleanup prevents publishing a successful consistency snapshot. Quiescence must be idempotent and share one global absolute deadline, not restart a full timeout per worker. -Within Lambda, stop deployment/invocation producers before retiring the pool/environment consumers. Join EnvironmentManager before revoking image leases, and join image cleanup before DockerWire shutdown. Environment count and aggregate function+relay memory remain reserved through allocating, idle, busy and quarantined states; a timed-out shutdown waiter leaves that ownership intact. A removed environment's business failure is distinct from failed supervision: known protocol/handler errors may retire cleanly, while Worker failure remains cached even after physical recovery. The lower-level owner alone is not a startup readiness receipt; historical uncertain effects and daemon/network/resource probes remain separate required gates. +Within Lambda, stop deployment/invocation producers before retiring the pool/environment consumers. Join EnvironmentPool before EnvironmentManager, then join EnvironmentManager before revoking image leases, and join image cleanup before DockerWire shutdown. Environment count and aggregate function+relay memory remain reserved through allocating, idle, busy and quarantined states; a timed-out shutdown waiter leaves that ownership intact. A removed environment's business failure is distinct from failed supervision: known protocol/handler errors may retire cleanly, while Worker failure remains cached even after physical recovery. The lower-level owner alone is not a startup readiness receipt; historical uncertain effects and daemon/network/resource probes remain separate required gates. Snapshot coverage: diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 53c94ca4..82c634fe 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -102,7 +102,9 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi `make test-lambda-environments` verifies the retained root/worker owner independently of Docker: transactional count/memory admission, scope rejection, definitive create rejection, canceled critical create,500→404→late commit, foreign label/image/name/ID quarantine, root abort recovery, revoked/connect-failed never-issued settlement, slow-health command fairness, failed versus normal queued-handle settlement, final log EOF without reuse and bounded startup under a long caller deadline. The transport regression combines an actual scope error with owner abort and requires Worker precedence/caching. Never count these scripted cases as image code execution. -`make test-lambda-environments-docker` builds the required images and explicitly runs two real owner cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, exact handler-timeout-at-delivery, per-invocation log attribution, startup bridge/resource probes, historical reconciliation or warm-pool limits. +`make test-lambda-pool` verifies one accepted FIFO queue across mailbox/actor waiting, canonical typed allocation identity, deadline/cancellation while waiting or offering a result, warm reuse only after consumption, zero warm retention and excess concurrent per-identity returns, one LRU eviction barrier under delayed cleanup, FIFO no-replay startup rejection, cached worker/root failure, receipt recovery and cancelled/repeated shutdown. The unpolled-live response regression requires environment removal before polling the public future again. Source inspection establishes bounds validation before fingerprint materialization; oversized-capture cases independently pin rejection. `make test-lambda-pool-docker` runs the shared four-case owner/pool real-image suite. None of these tests substitutes for full executor/provider SDK acceptance. + +`make test-lambda-environments-docker` builds the required images and explicitly runs four real owner/pool cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, exact handler-timeout-at-delivery, per-invocation log attribution, startup bridge/resource probes or historical reconciliation. The pool cases use the same production owner and actual RIC to prove warm counter1/2, captured revision isolation, one-slot LRU eviction, FunctionError retirement/replacement and independently observed running-handler abandonment with joined cleanup. `make test-lambda-environment-plans` verifies §4.2/D30 before allocation: fixed image IDs/platforms, distinct names and ownership roles, relay/function resource/swap/tmpfs/health/log/restart policy, user selection/grammar, bounded revision/region/env/argv validation, single-key inherited→captured→runtime environment precedence with deletion markers, retained real-directory and both-image lease lifetime, and revoked availability before IO. A spawned create regression proves the wire future is Send. The actual two-case wire suite consumes EnvironmentPlan for both ordinary RIC execution and empty ImageConfig overrides: require handler-reported CPU/memory/no-swap/PIDs,512MiB tmpfs/mode, cap-drop/no-new-privileges/read-only rootfs, and configured metadata despite intentionally conflicting image defaults. Inspect real CMD/ENTRYPOINT/WORKDIR and require expected exits142/0 for inherited-empty and reset-entrypoint variants. Join control/log streams and remove both owned containers/aliases on all normal test outcomes. This is allocation-policy qualification, not lifecycle/pool, startup bridge/probe, historical reconciliation or SDK executor acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 9dbeada3..a6ed33a2 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker warm/LRU pool and executor integration, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership and bounded warm/LRU pool dispatch, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker executor integration and exact invocation timing/log attribution, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -147,6 +147,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 function logs | Implemented/reviewed; environment integration remains | Output-only attach and FunctionLogs retain partial framing across snapshot traffic, bounded independent rings/combined4KiB binary tail, fair service, early socket closure and cached joined shutdown (§3.7/D29).18 focused tests pass; W63/W64 fixes independently rechecked. Core461 tests/107 doctests, workspace2,140 tests, boundary896 unit/doc passes and all standard/strict gates pass. Both real two-case wire suites receive deliberate handler stdout/stderr markers independently from computed results and join logs during cleanup. No new dependency/version. Per-invocation log attribution, environment integration and complete executor acceptance remain pending. | | 1.3 allocation policy | Implemented/reviewed; lifecycle owner remains | EnvironmentPlan fixes both create bodies to captured identities/platforms/revision, retained directory/image leases, validated policy and authoritative unique-key environment (§4.2/D30).54 new cases pass; W65–W67 fixed/rechecked. Core515 tests/116 doctests, workspace2,194 tests, boundary959 unit/doc passes and standard/strict/audit/deny gates pass. Both real two-case wire suites now consume plans for normal/empty ImageConfig execution, with actual resource/tmpfs/privilege/env assertions; all five shared runtime Docker regressions also pass per daemon. No production pool/executor or historical reconciliation readiness is claimed. | | 1.3 environment lifecycle | Implemented/reviewed; pool/executor integration remains | EnvironmentManager retains count/memory/plan/container/IO ownership above supervised workers, including uncertain creates, cancelled callers, panic recovery and quarantine (§4.3/D31). W68–W77 fixes and final independent recheck pass. Twenty scripted lifecycle cases plus one runtime failure-precedence regression pass. Core535 unit tests/127 doctests, workspace2,215 tests/403 gated cases, boundary800 unit tests/191 doctests and all standard/strict/audit/deny gates pass. Actual owner2, wire2 and shared runtime5 cases pass on each arm64 VZ and amd64/QEMU daemon; final owner2 cases are rechecked after the ready-reply cancellation fix. Both daemons have no owned containers/internal aliases afterward. Exact delivery-based handler timeout, invocation log attribution, warm/LRU checkout, historical reconciliation/pulls/network/probes and final executor wiring remain required before M0. | +| 1.3 warm pool | Implemented/reviewed; executor composition remains | §4.4/D32 implement one bounded accepted FIFO queue, canonical validated allocation identity, receipt-gated warm return, independent worker-side receipt deadline, one-at-a-time capacity LRU eviction, zero/excess retention and retained panic/shutdown retirement. W78–W81 fixes and final independent recheck pass. Eighteen scripted pool cases pass. Core553 unit tests/132 doctests; workspace2,233 tests/405 separately gated; strict boundary818 unit tests/196 doctests (1,014 total). Standard build/test/fmt/Clippy/rustdoc, strict production checks and fresh audit/deny pass. Both arm64 VZ and amd64/QEMU explicitly pass four owner/pool real-RIC cases, including warm1/2, revision/capacity eviction, FunctionError replacement counter1 and observed running-handler abandonment; no owned containers/internal aliases remain afterward. Exact delivery-based handler timeout/log attribution and production Executor/provider/startup composition remain required. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Next implement warm/LRU checkout and the real executor, including exact handler timeout from delivery and invocation log attribution, consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Next implement exact handler timeout from delivery and invocation log attribution, then the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 92ee7bd3..65bf6ad4 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -347,6 +347,23 @@ Actual owner2, wire2 and shared runtime5 tests explicitly pass with zero ignored The owner remains a lower-level runtime consumer. It enforces supplied invocation deadlines and retains cleanup, while exact handler timeout starting at delivery, per-invocation log attribution, warm/LRU checkout limits, startup reconciliation/pulls/network/probes and final provider/executor wiring remain mandatory M0 integration work. No production backend capability is enabled by this change; P2–P5 remain required. +## P1 task1.3 warm pool dispatch review + +Independent read-only review covers §4.4/D32: bounded admission/FIFO dispatch, canonical warm identity, exclusive handles, receipt-gated return, LRU eviction and retained root/worker retirement. The final source recheck has no remaining findings. This remains a runtime consumer, not production Docker executor acceptance. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W78 | P2; spec preflight, runtime §4.4 | A live but unpolled public receiver cannot enforce its own timeout. The worker retains the absolute deadline through result offering and consumption receipt waiting | Ready reply stays unpolled past deadline; assert environment Removed before polling that public future again | +| W79 | P2; `docker/plan.rs`, warm identity | Hashing before complete boundary validation allocated for unbounded captured environment/ImageConfig inputs. Build bounded image bodies and validate function environment before fingerprint serialization | Independent source-order recheck plus separate oversized-input rejection regressions | +| W80 | P2; `docker/pool/actor.rs`, capacity eviction | Repeated Busy before slow cleanup completed could evict the entire LRU for one waiter. Retain one eviction task barrier and await its settlement before retrying dispatch | Two idle peers with a deliberately held first retirement; repeated ticks keep the second idle handle alive | +| W81 | P3; `tests/docker_environment.rs`, pooled busy-cancel observer | Fallible plan/invocation construction after attaching the observer could leave it only dropped on error. Move both constructions before observer startup | Final read-only recheck confirms every post-attach path reaches explicit call abort/join and pool/log shutdown before propagating observation failure | + +Final-source workspace build/test/fmt/Clippy/rustdoc all pass:2,233 tests, zero failed,405 separately resource-gated. Lambda core has553 unit tests and132 doctests. The18 pool cases cover runtime zero/global/per-identity retention, queued cancellation/capacity/FIFO, ready/unpolled results, exact warm counts, slow LRU cleanup and root/worker/shutdown ownership. The six-library boundary target passes818 unit tests/196 doctests (1,014 total),11 separately gated Docker cases and strict production Clippy. Fresh audit scans474 dependencies without vulnerabilities; all deny categories pass. No dependency or advisory exception was added. + +Four explicit owner/pool real-RIC cases pass with zero ignored on each retained Engine29.5.2 daemon (arm64 VZ and amd64/QEMU). Actual evidence includes near6MiB owner results, warm counters1/2, revision isolation and one-slot capacity eviction, real FunctionError followed by replacement counter1, and independent busy-marker observation before aborting a pooled invocation. Both cancellation cases require a cancelled join and attempt every call/pool/log cleanup before propagating observation errors. Final read-only daemon/image queries find no owner-labeled containers or internal image aliases. Native-x86/Desktop/rootless and full SDK/executor qualification remain required; QEMU is not native-x86 acceptance. + +Exact handler timeout from Runtime API delivery, per-invocation log attribution, historical reconciliation, pulls/network/resource probes and real Executor/provider composition remain mandatory M0 work. P2–P5 remain required by the complete objective. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index fd83d071..57a7201b 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -281,3 +281,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why:** every accepted effect retains an owner, while validation/connect/queue failures can settle without permanent false uncertainty. Health checks reschedule after completion to avoid backlog starvation; log EOF permits an already-observed valid result but forbids reuse. Known protocol failures remain business failures once IO joins; uncertain supervision cannot be masked by an earlier protocol error. - **Pinned by:** [runtime §4.3](./rustack-workflows-20-lambda-runtime-design.md#43-retained-environment-owner-and-allocation-reservations), [verification](./rustack-workflows-70-verification-plan.md), [implementation](./rustack-workflows-90-impl-plan.md). - **Date:** 2026-10-26. This low-level owner does not replace provider admission, warm/LRU pooling, startup reconciliation/probes or exact Lambda timeout/log integration. Production Docker stays disabled until those M0 consumers pass. + +## D32 — Pool dispatch retains receipt-based ownership through warm return + +- **Context**: §4.4 warm execution must not reuse an environment after a caller abandons a ready result or a dispatch worker fails. +- **Alternatives considered**: transferring handles to HTTP futures loses cleanup ownership; returning to the LRU immediately after send loses result-consumption cancellation; keying by redacted Debug hides configuration differences. +- **Decision**: retain pool tasks and environment retirement receipts above the JoinSet, require a consumption receipt before warm return and hash canonical typed allocation inputs. +- **Why**: queue, execution, warm retention and cleanup have explicit bounded owners; underlying EnvironmentManager reservations remain authoritative through quarantine. +- **Pinned by**: [runtime §4.4](./rustack-workflows-20-lambda-runtime-design.md#44-warm-checkout-and-retained-pool-dispatch), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. From 8f284c7d4c4dc8763409ec16b83b075d0f548501 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 02:38:02 -0700 Subject: [PATCH 025/106] phase 1: arm handler timeouts at consumed runtime delivery Implement runtime section 3.8 and D33: control protocol v2 carries validated handler budgets and correlated timeout results. Consumed poll receipts arm effective deadlines shared by HTTP metadata and session enforcement; Docker owners attach immutable captured budgets and join timeout retirement. Native preserves independent initialization and caller deadline categories. Independently reviewed W82-W84 fixes include the actual receipt/timer selection race. All standard gates pass with 2,255 workspace tests, 1,038 strict-boundary unit/doc tests and 131 timeout cases including 12 real Native fixtures. Audit and deny pass. Twelve explicit owner/pool/wire/runtime cases pass per arm64 VZ and amd64/QEMU daemon, with captured-timeout override rejection, capacity settlement and cold replacement; no owned resources remain. Invocation log attribution, real executor/provider composition, reconciliation/pulls/network/probes and P2-P5 remain required; M0 is not closed. --- Makefile | 10 + .../src/docker/environment/mod.rs | 3 + .../src/docker/environment/worker.rs | 12 +- .../src/executor/instance.rs | 43 ++-- .../tests/docker_environment.rs | 50 ++++- crates/rustack-lambda-runtime/src/client.rs | 27 ++- .../src/client/tests.rs | 84 ++++++++ crates/rustack-lambda-runtime/src/http.rs | 2 +- .../rustack-lambda-runtime/src/http/tests.rs | 45 +++++ .../src/protocol/mod.rs | 14 +- .../src/protocol/tests.rs | 85 +++++++- .../src/protocol/types.rs | 55 ++++- crates/rustack-lambda-runtime/src/relay.rs | 18 +- .../rustack-lambda-runtime/src/relay/tests.rs | 32 ++- crates/rustack-lambda-runtime/src/session.rs | 117 +++++++++-- .../src/session/tests.rs | 189 +++++++++++++++++- crates/rustack-lambda-runtime/tests/docker.rs | 1 + specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 42 +++- .../rustack-workflows-60-operations-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 6 +- specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 17 ++ specs/rustack-workflows-99-decisions.md | 9 + .../echo-bootstrap/tests/hardening.rs | 55 ++++- 25 files changed, 861 insertions(+), 66 deletions(-) diff --git a/Makefile b/Makefile index e871f93b..b9fa0e8c 100644 --- a/Makefile +++ b/Makefile @@ -898,3 +898,13 @@ test-lambda-pool: test-lambda-pool-docker: test-lambda-environments-docker .PHONY: test-lambda-pool test-lambda-pool-docker + +# Delivery clocks, protocol-v2 correlation and actual Native process retirement. +test-lambda-timeouts: + @cargo test -p rustack-lambda-runtime --lib + @cargo test -p rustack-lambda-echo-bootstrap --test hardening + +# Captured handler-timeout/retirement/replacement through the real image pool. +test-lambda-timeouts-docker: test-lambda-environments-docker + +.PHONY: test-lambda-timeouts test-lambda-timeouts-docker diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index 0fd888ee..32f2207f 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -105,6 +105,9 @@ pub enum EnvironmentError { /// Startup, call or waiting deadline expired. #[error("environment deadline exceeded")] Deadline, + /// The captured function timeout expired after Runtime API delivery. + #[error("function timed out after {0:?}")] + Timeout(Duration), /// This owner no longer accepts work. #[error("environment is stopped")] Stopped, diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs index 6706f2d8..45822a4e 100644 --- a/crates/rustack-lambda-core/src/docker/environment/worker.rs +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -5,7 +5,9 @@ use std::{future::pending, sync::Arc, time::Duration}; use bytes::Bytes; use http::StatusCode; use rustack_lambda_runtime::{ - client::ClientEvent, protocol::Architecture as RuntimeArchitecture, session::RuntimeResult, + client::ClientEvent, + protocol::{Architecture as RuntimeArchitecture, HandlerTimeout}, + session::RuntimeResult, }; use tokio::{ runtime::Handle, @@ -278,7 +280,10 @@ impl Worker { } io.client()? .invoke( - call.invocation.clone(), + call.invocation.clone().with_handler_timeout( + HandlerTimeout::try_from(self.plan.revision().timeout) + .map_err(|_| EnvironmentError::Configuration)?, + ), deadline.min(self.config.deadline()), ) .await?; @@ -286,6 +291,9 @@ impl Worker { .next(io, deadline, Some(cleaner), Some(&mut call.cancellation)) .await; match result { + Ok(ClientEvent::InvocationTimedOut(returned)) if returned == attempt => { + Err(EnvironmentError::Timeout(self.plan.revision().timeout)) + } Ok(ClientEvent::InvocationResult { attempt: returned, result, diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 6a5e9d28..3493c664 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -18,7 +18,7 @@ use bytes::Bytes; use dashmap::DashMap; use rustack_lambda_runtime::{ metadata::local_trace_id, - protocol::{AttemptId, Payload}, + protocol::{AttemptId, HandlerTimeout, Payload}, }; use tokio::{ sync::{OwnedSemaphorePermit, Semaphore, oneshot}, @@ -243,6 +243,8 @@ impl InstancePool { .map_err(|_| ExecutorError::InvalidCode("Invalid invocation clock metadata".into()))?; let payload = Payload::try_from(req.payload.clone()) .map_err(|_| ExecutorError::InvalidCode("Invocation payload too large".into()))?; + let handler_timeout = HandlerTimeout::try_from(req.revision.timeout) + .map_err(|_| ExecutorError::InvalidCode("Invalid handler timeout".into()))?; let key = Self::key(&req); let (instance, retired) = self.try_acquire(&key); retire_all(retired).await?; @@ -260,18 +262,18 @@ impl InstancePool { Some(instance) => instance, None => self.spawn_new(&req, &context).await?, }; - let Some(handler_deadline) = TokioInstant::now().checked_add(req.revision.timeout) else { - instance.retire().await?; - return Err(ExecutorError::InvalidCode("Invalid handler timeout".into())); - }; - let deadline = handler_deadline.min(context.deadline()); - let Ok(invocation) = Invocation::new(attempt, deadline, arn, trace, payload) else { + let Ok(invocation) = Invocation::new(attempt, context.deadline(), arn, trace, payload) + else { instance.retire().await?; context.check()?; return Err(ExecutorError::Timeout(req.revision.timeout)); }; let result = instance - .run(invocation, &context, req.revision.timeout) + .run( + invocation.with_handler_timeout(handler_timeout), + &context, + req.revision.timeout, + ) .await; let result = context.check().and(result); let reusable = matches!(&result, Ok(RuntimeResult::Success(_))) @@ -371,7 +373,7 @@ impl InstancePool { biased; reason = context.cancelled() => return Err(reason.error()), result = timeout_at(deadline, runtime_api::start_at(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), self.idle_timeout)) => { - result.map_err(|_| ExecutorError::Timeout(self.init_timeout))?.map_err(|_| ExecutorError::InitFailed("Runtime API startup failed".into()))? + result.map_err(|_| initialization_error(SessionError::Deadline, self.init_timeout, context))?.map_err(|_| ExecutorError::InitFailed("Runtime API startup failed".into()))? } }; let addr = api.addr(); @@ -379,7 +381,7 @@ impl InstancePool { let backend = tokio::select! { biased; reason = context.cancelled() => Err(reason.error()), - result = timeout_at(deadline, self.backend.spawn(req, addr)) => result.map_err(|_| ExecutorError::Timeout(self.init_timeout)).and_then(|result| result), + result = timeout_at(deadline, self.backend.spawn(req, addr)) => result.map_err(|_| initialization_error(SessionError::Deadline, self.init_timeout, context)).and_then(|result| result), }; let backend = match backend { Ok(backend) => backend, @@ -406,14 +408,14 @@ impl InstancePool { Ok(_) => Err(ExecutorError::InitFailed("Bootstrap reported initialization failure".into())), Err(_) => Err(ExecutorError::RuntimeExited("Runtime initialization channel closed".into())), }, - ready = instance.api.session().bootstrap_ready(deadline) => ready.map_err(|error| invocation_error(error, self.init_timeout)), + ready = instance.api.session().bootstrap_ready(deadline) => ready.map_err(|error| initialization_error(error, self.init_timeout, context)), }, None => instance .api .session() .bootstrap_ready(deadline) .await - .map_err(|error| invocation_error(error, self.init_timeout)), + .map_err(|error| initialization_error(error, self.init_timeout, context)), } }; tokio::select! { @@ -472,9 +474,24 @@ impl InstancePool { } } -fn invocation_error(error: SessionError, timeout: Duration) -> ExecutorError { +fn initialization_error( + error: SessionError, + timeout: Duration, + context: &InvocationContext, +) -> ExecutorError { + if let Err(error) = context.check() { + return error; + } match error { SessionError::Deadline => ExecutorError::Timeout(timeout), + _ => invocation_error(error, timeout), + } +} + +fn invocation_error(error: SessionError, timeout: Duration) -> ExecutorError { + match error { + SessionError::HandlerTimeout => ExecutorError::Timeout(timeout), + SessionError::Deadline => ExecutorError::DeadlineExceeded, SessionError::Cancelled => ExecutorError::Cancelled, SessionError::SupervisorFailed => { ExecutorError::CleanupFailed("Runtime session supervisor failed".into()) diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs index 68bf972e..092b943a 100644 --- a/crates/rustack-lambda-core/tests/docker_environment.rs +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -18,7 +18,7 @@ use rustack_lambda_core::{ }; use rustack_lambda_runtime::{ metadata::local_trace_id, - protocol::{AttemptId, Payload}, + protocol::{AttemptId, HandlerTimeout, Payload}, session::{Invocation, RuntimeResult}, }; use serde_json::{Value, json}; @@ -429,3 +429,51 @@ async fn test_should_join_pool_retirement_after_abandoning_an_observed_running_h let cleanup = fixture.cleanup().await; result.and(retired.map_err(Into::into)).and(cleanup) } + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_enforce_captured_handler_timeout_and_replace_the_retired_pooled_environment() +-> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let mut revision = (*fixture.revision()?).clone(); + revision.timeout = Duration::from_secs(3); + let revision = Arc::new(revision); + let port = pool.port(); + let response = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":0}))?, + false, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("timeout fixture warm setup failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["counter"] == 1, "initial counter differs"); + let result = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":1,"sleep":true}))?.with_handler_timeout(HandlerTimeout::try_from(Duration::from_mins(15))?), + false, + ).await; + ensure!( + matches!(result, Err(EnvironmentError::Timeout(duration)) if duration == Duration::from_secs(3)), + "handler timeout was not preserved", + ); + ensure!(fixture.owner.port().status().allocated == 0, "timeout returned before resource settlement"); + let response = port.invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":35}))?, + false, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("timeout replacement invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["computed"] == 42 && value["counter"] == 1, "timed out environment was reused"); + Ok(()) + }.await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} diff --git a/crates/rustack-lambda-runtime/src/client.rs b/crates/rustack-lambda-runtime/src/client.rs index cd9162a1..ac84fe81 100644 --- a/crates/rustack-lambda-runtime/src/client.rs +++ b/crates/rustack-lambda-runtime/src/client.rs @@ -13,7 +13,8 @@ use tokio::{ use crate::{ metadata::deadline_unix_ms, protocol::{ - Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, Message, Outcome, Payload, + Acknowledgement, Architecture, AttemptId, EnvironmentId, Frame, HandlerTimeout, Message, + Outcome, Payload, }, relay::RelayError, session::{Invocation, RuntimeResult}, @@ -72,6 +73,8 @@ pub enum ClientEvent { /// Raw success or function error, without an implicit retry. result: RuntimeResult, }, + /// A matching delivered handler clock expired; this session is retiring. + InvocationTimedOut(AttemptId), /// Initialization failed; this environment cannot be reused. InitError(Payload), /// The relay invalidated the attempt; external code may still be running. @@ -102,6 +105,7 @@ struct State { closing: bool, writing: bool, delivered: bool, + handler_timeout: Option, } /// One host control session with retained IO and fail-closed send semantics. @@ -153,6 +157,7 @@ impl ControlClient { closing: false, writing: false, delivered: false, + handler_timeout: None, }, last_attempt: None, settlement: None, @@ -245,11 +250,13 @@ impl ControlClient { _ => return Err(self.fail(RelayError::Deadline)), }; self.last_attempt = Some(attempt); + self.state.handler_timeout = invocation.handler_timeout(); self.state.phase = Phase::Busy(attempt); self.send( Message::Invoke { attempt, deadline_unix_ms, + handler_timeout_ms: invocation.handler_timeout(), function_arn: function_arn.clone(), trace_id: trace_id.clone(), payload: payload.clone(), @@ -433,6 +440,24 @@ impl ControlClient { } _ => Err(RelayError::UnexpectedMessage), }, + Message::InvocationTimedOut { attempt } if self.state.handler_timeout.is_some() => { + match &mut self.state.phase { + Phase::Busy(active) if *active == attempt => { + self.state.delivered = true; + self.state.phase = Phase::Retiring; + Ok(Some(ClientEvent::InvocationTimedOut(attempt))) + } + Phase::Cancelling { + attempt: active, + outcome_seen, + } if *active == attempt && !*outcome_seen => { + self.state.delivered = true; + *outcome_seen = true; + Ok(None) + } + _ => Err(RelayError::UnexpectedMessage), + } + } Message::InitError { payload } if !self.state.delivered => { match &mut self.state.phase { Phase::Initializing | Phase::Idle | Phase::Busy(_) => { diff --git a/crates/rustack-lambda-runtime/src/client/tests.rs b/crates/rustack-lambda-runtime/src/client/tests.rs index f2d4d980..8656a1e8 100644 --- a/crates/rustack-lambda-runtime/src/client/tests.rs +++ b/crates/rustack-lambda-runtime/src/client/tests.rs @@ -626,3 +626,87 @@ async fn test_should_preserve_transport_worker_failure_after_a_prior_scope_error Err(RelayError::Worker) )); } + +#[rstest::rstest] +#[case::idle("idle")] +#[case::initializing("initializing")] +#[case::without_budget("without_budget")] +#[case::foreign_attempt("foreign")] +#[case::duplicate("duplicate")] +#[case::init_after_timeout("init_after_timeout")] +#[tokio::test] +async fn test_should_reject_impossible_handler_timeout_events(#[case] condition: &str) { + let mut peer = Peer::new(4096); + peer.ready(condition != "initializing").await; + let call = invocation(); + let attempt = call.parts().0; + if !matches!(condition, "idle" | "initializing") { + let call = if condition == "without_budget" { + call + } else { + call.with_handler_timeout( + crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap(), + ) + }; + peer.client.invoke(call, deadline()).await.unwrap(); + let sent = peer.read().await; + assert!( + matches!( + sent, + Message::Invoke { + handler_timeout_ms: Some(_), + .. + } + ) || condition == "without_budget" + ); + } + let reported = if condition == "foreign" { + AttemptId::default() + } else { + attempt + }; + peer.send(Message::InvocationTimedOut { attempt: reported }) + .await; + if matches!(condition, "duplicate" | "init_after_timeout") { + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::InvocationTimedOut(attempt) + ); + assert!(!peer.client.is_idle()); + let message = if condition == "duplicate" { + Message::InvocationTimedOut { attempt } + } else { + Message::InitError { + payload: Payload::try_from(Bytes::new()).unwrap(), + } + }; + peer.send(message).await; + } + assert!(matches!( + peer.client.next(deadline()).await, + Err(RelayError::UnexpectedMessage) + )); + assert!(peer.client.shutdown(deadline()).await.is_err()); +} +#[tokio::test] +async fn test_should_discard_only_one_matching_handler_timeout_during_cancel_before_ack() { + let mut peer = Peer::new(4096); + peer.ready(true).await; + let call = invocation().with_handler_timeout( + crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap(), + ); + let attempt = call.parts().0; + peer.client.invoke(call, deadline()).await.unwrap(); + peer.read().await; + peer.client.cancel(attempt, deadline()).await.unwrap(); + assert_eq!(peer.read().await, Message::Cancel { attempt }); + peer.send(Message::InvocationTimedOut { attempt }).await; + peer.send(Message::Ack(Acknowledgement::Cancel(attempt))) + .await; + assert_eq!( + peer.client.next(deadline()).await.unwrap(), + ClientEvent::Cancelled(attempt) + ); + assert!(!peer.client.is_idle()); + peer.close().await; +} diff --git a/crates/rustack-lambda-runtime/src/http.rs b/crates/rustack-lambda-runtime/src/http.rs index d26f38de..1b0abb91 100644 --- a/crates/rustack-lambda-runtime/src/http.rs +++ b/crates/rustack-lambda-runtime/src/http.rs @@ -510,7 +510,7 @@ fn status(error: SessionError) -> StatusCode { match error { SessionError::Busy => StatusCode::CONFLICT, SessionError::UnknownAttempt | SessionError::InvalidMetadata => StatusCode::BAD_REQUEST, - SessionError::Deadline => StatusCode::REQUEST_TIMEOUT, + SessionError::Deadline | SessionError::HandlerTimeout => StatusCode::REQUEST_TIMEOUT, _ => StatusCode::SERVICE_UNAVAILABLE, } } diff --git a/crates/rustack-lambda-runtime/src/http/tests.rs b/crates/rustack-lambda-runtime/src/http/tests.rs index 65da0227..0fd3e6d7 100644 --- a/crates/rustack-lambda-runtime/src/http/tests.rs +++ b/crates/rustack-lambda-runtime/src/http/tests.rs @@ -506,3 +506,48 @@ async fn test_should_reject_all_trailers_before_initialization_or_invocation_mut } api.shutdown().await.unwrap(); } + +#[tokio::test] +async fn test_should_publish_effective_delivery_deadline_and_join_api_after_handler_timeout() { + use crate::{metadata::deadline_unix_ms, protocol::HandlerTimeout}; + let mut api = start().await.unwrap(); + let handle = api.session().clone(); + let mut call = invocation(); + call = Invocation::new( + call.parts().0, + Instant::now() + Duration::from_secs(30), + call.parts().2.clone(), + call.parts().3.clone(), + call.parts().4.clone(), + ) + .unwrap() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let outer = deadline_unix_ms(call.parts().1).unwrap().get(); + let ticket = handle + .submit(call, Instant::now() + Duration::from_secs(3)) + .await + .unwrap(); + let reply = request( + api.addr(), + "GET", + "/2018-06-01/runtime/invocation/next", + "", + b"", + ) + .await; + assert_eq!(reply.status, 200); + let effective = reply + .header("Lambda-Runtime-Deadline-Ms") + .unwrap() + .parse::() + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + assert!(u128::from(effective) <= now + 1000 && u128::from(effective) > now); + assert!(effective + 20_000 < outer); + assert_eq!(ticket.wait().await, Err(SessionError::HandlerTimeout)); + api.shutdown().await.unwrap(); + assert!(TcpStream::connect(api.addr()).await.is_err()); +} diff --git a/crates/rustack-lambda-runtime/src/protocol/mod.rs b/crates/rustack-lambda-runtime/src/protocol/mod.rs index 7846abdc..45f772e3 100644 --- a/crates/rustack-lambda-runtime/src/protocol/mod.rs +++ b/crates/rustack-lambda-runtime/src/protocol/mod.rs @@ -19,8 +19,8 @@ use tokio::{ time::{Instant, timeout_at}, }; pub use types::{ - Acknowledgement, Architecture, AttemptId, EnvironmentId, FunctionArn, Message, Outcome, - Payload, TraceId, + Acknowledgement, Architecture, AttemptId, EnvironmentId, FunctionArn, HandlerTimeout, Message, + Outcome, Payload, TraceId, }; /// Maximum event or result payload, in bytes. @@ -29,6 +29,8 @@ pub const MAX_PAYLOAD_BYTES: usize = 6 * 1024 * 1024; pub const MAX_METADATA_BYTES: usize = 64 * 1024; /// Maximum frame body, excluding its four-byte prefix. pub const MAX_FRAME_BYTES: usize = MAX_PAYLOAD_BYTES + MAX_METADATA_BYTES; +/// Host/relay contract version; older relays cannot enforce delivery-based timeouts. +pub const PROTOCOL_VERSION: u32 = 2; /// Protocol failures contain classes and IO sources, never invalid peer values. #[derive(Debug, Error)] @@ -43,7 +45,7 @@ pub enum ProtocolError { /// The valid CBOR does not match the typed message schema. #[error("invalid control message schema")] InvalidSchema, - /// A version other than one was supplied. + /// An incompatible host/relay version was supplied. #[error("unsupported control protocol version")] UnsupportedVersion, /// A validated primitive has invalid syntax or range. @@ -85,7 +87,7 @@ impl TryFrom for Frame { type Error = ProtocolError; fn try_from(raw: RawFrame) -> Result { - if raw.version != 1 { + if raw.version != PROTOCOL_VERSION { return Err(ProtocolError::UnsupportedVersion); } Ok(Self::new(raw.environment, raw.message)) @@ -93,7 +95,7 @@ impl TryFrom for Frame { } impl Frame { - /// Wrap a typed message in protocol version one. + /// Wrap a typed message in the current protocol version. /// /// ``` /// use rustack_lambda_runtime::protocol::{EnvironmentId, Frame, Message}; @@ -103,7 +105,7 @@ impl Frame { #[must_use] pub fn new(environment: EnvironmentId, message: Message) -> Self { Self { - version: 1, + version: PROTOCOL_VERSION, environment, message, } diff --git a/crates/rustack-lambda-runtime/src/protocol/tests.rs b/crates/rustack-lambda-runtime/src/protocol/tests.rs index 01d26c56..1081b287 100644 --- a/crates/rustack-lambda-runtime/src/protocol/tests.rs +++ b/crates/rustack-lambda-runtime/src/protocol/tests.rs @@ -27,6 +27,7 @@ fn messages() -> Vec { Message::Invoke { attempt, deadline_unix_ms: NonZeroU64::new(1_800_000_000_000).unwrap(), + handler_timeout_ms: None, function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" .parse() .unwrap(), @@ -50,6 +51,7 @@ fn messages() -> Vec { Message::Close, Message::Ack(Acknowledgement::Close), Message::Ack(Acknowledgement::Cancel(attempt)), + Message::InvocationTimedOut { attempt }, ] } @@ -193,7 +195,7 @@ fn test_should_reject_unknown_duplicate_and_mismatched_variant_fields() { Err(ProtocolError::InvalidSchema) )); let mut version = frame; - *field(&mut version, "version") = Value::Integer(2.into()); + *field(&mut version, "version") = Value::Integer(1.into()); assert!(matches!( decode(&body(&version)), Err(ProtocolError::UnsupportedVersion) @@ -421,3 +423,84 @@ async fn test_should_enforce_absolute_deadlines_for_partial_reads_and_stalled_wr "expired reads must not consume a ready frame" ); } + +#[rstest] +#[case(0)] +#[case(999)] +#[case(1001)] +#[case(900_001)] +#[case(u32::MAX)] +fn test_should_reject_out_of_profile_handler_timeouts_before_session_admission( + #[case] milliseconds: u32, +) { + assert!(HandlerTimeout::try_from(milliseconds).is_err()); + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(HandlerTimeout::try_from(1000).unwrap()); + } + let mut packet = value(invoke); + *field( + field(field(&mut packet, "message"), "data"), + "handlerTimeoutMs", + ) = Value::Integer(milliseconds.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidSchema) + )); +} +#[test] +fn test_should_round_trip_handler_budget_timeout_event_and_reject_older_relays() { + for seconds in [1, 900] { + let timeout = HandlerTimeout::try_from(Duration::from_secs(seconds)).unwrap(); + assert_eq!(timeout.duration(), Duration::from_secs(seconds)); + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(timeout); + } + let packet = value(invoke.clone()); + assert_eq!(decode(&body(&packet)).unwrap().into_message(), invoke); + } + for timeout in [ + Duration::ZERO, + Duration::from_millis(1001), + Duration::from_secs(901), + Duration::MAX, + ] { + assert!(HandlerTimeout::try_from(timeout).is_err()); + } + let message = Message::InvocationTimedOut { + attempt: AttemptId::default(), + }; + let packet = value(message.clone()); + assert_eq!(decode(&body(&packet)).unwrap().into_message(), message); + for version in [0, 1, 3, u32::MAX] { + let mut packet = value(Message::Ready); + *field(&mut packet, "version") = Value::Integer(version.into()); + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::UnsupportedVersion) + )); + } + let mut invoke = messages().into_iter().nth(4).unwrap(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut invoke + { + *handler_timeout_ms = Some(HandlerTimeout::try_from(1000).unwrap()); + } + let mut packet = value(invoke); + *field( + field(field(&mut packet, "message"), "data"), + "handlerTimeoutMs", + ) = Value::Null; + assert!(matches!( + decode(&body(&packet)), + Err(ProtocolError::InvalidStructure) + )); +} diff --git a/crates/rustack-lambda-runtime/src/protocol/types.rs b/crates/rustack-lambda-runtime/src/protocol/types.rs index ebd63cad..af75f76a 100644 --- a/crates/rustack-lambda-runtime/src/protocol/types.rs +++ b/crates/rustack-lambda-runtime/src/protocol/types.rs @@ -1,6 +1,6 @@ //! Validated metadata and closed control-message variants. -use std::{fmt, num::NonZeroU64, str::FromStr}; +use std::{fmt, num::NonZeroU64, str::FromStr, time::Duration}; use bytes::Bytes; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -56,6 +56,51 @@ identity!( EnvironmentId, "Canonical nonnil environment UUID; parse with `FromStr`, generate with `Default`." ); + +/// Validated whole-second handler timeout, encoded as milliseconds on the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "u32", into = "u32")] +pub struct HandlerTimeout(u32); + +impl TryFrom for HandlerTimeout { + type Error = ProtocolError; + fn try_from(milliseconds: u32) -> Result { + if !(1000..=900_000).contains(&milliseconds) || !milliseconds.is_multiple_of(1000) { + return Err(ProtocolError::InvalidMetadata); + } + Ok(Self(milliseconds)) + } +} +impl TryFrom for HandlerTimeout { + type Error = ProtocolError; + fn try_from(timeout: Duration) -> Result { + if timeout.subsec_nanos() != 0 { + return Err(ProtocolError::InvalidMetadata); + } + Self::try_from( + u32::try_from(timeout.as_millis()).map_err(|_| ProtocolError::InvalidMetadata)?, + ) + } +} +impl From for u32 { + fn from(timeout: HandlerTimeout) -> Self { + timeout.0 + } +} +impl HandlerTimeout { + /// Return the validated relative budget without starting a clock. + /// + /// ``` + /// use std::time::Duration; + /// let timeout = rustack_lambda_runtime::protocol::HandlerTimeout::try_from(Duration::from_secs(3))?; + /// assert_eq!(timeout.duration(), Duration::from_secs(3)); + /// # Ok::<(), rustack_lambda_runtime::protocol::ProtocolError>(()) + /// ``` + #[must_use] + pub fn duration(self) -> Duration { + Duration::from_millis(u64::from(self.0)) + } +} identity!( AttemptId, "Canonical nonnil invocation attempt UUID; parse with `FromStr`, generate with `Default`." @@ -315,6 +360,9 @@ pub enum Message { attempt: AttemptId, /// Absolute wall deadline, checked and converted by the receiver. deadline_unix_ms: NonZeroU64, + /// Optional captured handler budget, armed only at consumed delivery. + #[serde(default, skip_serializing_if = "Option::is_none")] + handler_timeout_ms: Option, /// Requested ARN, preserving an invoked alias. function_arn: FunctionArn, /// Runtime trace header. @@ -331,6 +379,11 @@ pub enum Message { /// Raw result or error bytes. payload: Payload, }, + /// The delivered handler clock expired before the caller's absolute deadline. + InvocationTimedOut { + /// Timed-out active attempt; this environment cannot be reused. + attempt: AttemptId, + }, /// Fatal initialization failure before event delivery. InitError { /// Raw initialization error document. diff --git a/crates/rustack-lambda-runtime/src/relay.rs b/crates/rustack-lambda-runtime/src/relay.rs index 6761e42f..a43d2c3d 100644 --- a/crates/rustack-lambda-runtime/src/relay.rs +++ b/crates/rustack-lambda-runtime/src/relay.rs @@ -332,7 +332,17 @@ impl RelayActor { self.phase = Phase::Idle; self.emit(Message::BootstrapReady).await?; } - result = self.phase.result() => { self.completed(result?).await?; } + result = self.phase.result() => { + match result { + Ok(result) => self.completed(result).await?, + Err(SessionError::HandlerTimeout) => { + let attempt=self.last_attempt.ok_or(RelayError::UnexpectedMessage)?; + self.emit(Message::InvocationTimedOut { attempt }).await?; + self.retire().await?; + } + Err(error) => return Err(error.into()), + } + } () = session.closed(), if !matches!(self.phase, Phase::Retiring(_)) => return Err(RelayError::Session(SessionError::Unavailable)), } } @@ -343,14 +353,18 @@ impl RelayActor { Message::Invoke { attempt, deadline_unix_ms, + handler_timeout_ms, function_arn, trace_id, payload, } if matches!(self.phase, Phase::Idle) && self.last_attempt != Some(attempt) => { let deadline = deadline_from_unix_ms(deadline_unix_ms).map_err(|_| RelayError::Deadline)?; - let invocation = + let mut invocation = Invocation::new(attempt, deadline, function_arn, trace_id, payload)?; + if let Some(timeout) = handler_timeout_ms { + invocation = invocation.with_handler_timeout(timeout); + } let ticket = self .api .session() diff --git a/crates/rustack-lambda-runtime/src/relay/tests.rs b/crates/rustack-lambda-runtime/src/relay/tests.rs index 9f1fae5d..9ebaa570 100644 --- a/crates/rustack-lambda-runtime/src/relay/tests.rs +++ b/crates/rustack-lambda-runtime/src/relay/tests.rs @@ -114,6 +114,7 @@ fn invoke() -> (AttemptId, Message) { Message::Invoke { attempt, deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(10)).unwrap(), + handler_timeout_ms: None, function_arn: "arn:aws:lambda:us-east-1:000000000000:function:echo:prod" .parse::() .unwrap(), @@ -246,7 +247,7 @@ async fn test_should_distinguish_function_and_initialization_errors_and_allow_cl #[tokio::test] async fn test_should_reject_foreign_scope_wrong_direction_and_invoke_before_bootstrap() { - for variant in 0..3 { + for variant in 0..4 { let mut harness = Harness::new().await; match variant { 0 => write_frame( @@ -257,6 +258,13 @@ async fn test_should_reject_foreign_scope_wrong_direction_and_invoke_before_boot .await .unwrap(), 1 => harness.send(Message::Ready).await, + 2 => { + harness + .send(Message::InvocationTimedOut { + attempt: AttemptId::default(), + }) + .await; + } _ => harness.send(invoke().1).await, } assert!( @@ -429,3 +437,25 @@ fn test_should_bound_relay_config() { .is_err() ); } + +#[tokio::test] +async fn test_should_forward_delivered_handler_timeout_and_preserve_close_ack_retirement() { + let mut harness = Harness::new().await; + let poll = harness.bootstrap().await; + let (attempt, mut message) = invoke(); + if let Message::Invoke { + handler_timeout_ms, .. + } = &mut message + { + *handler_timeout_ms = + Some(crate::protocol::HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + } + harness.send(message).await; + let response = poll.await.unwrap(); + assert_eq!(response.status, 200); + assert_eq!( + harness.next().await, + Message::InvocationTimedOut { attempt } + ); + harness.close().await; +} diff --git a/crates/rustack-lambda-runtime/src/session.rs b/crates/rustack-lambda-runtime/src/session.rs index 1d253e70..b97336a6 100644 --- a/crates/rustack-lambda-runtime/src/session.rs +++ b/crates/rustack-lambda-runtime/src/session.rs @@ -19,7 +19,7 @@ use tokio::{ }; use typed_builder::TypedBuilder; -use crate::protocol::{AttemptId, FunctionArn, Outcome, Payload, TraceId}; +use crate::protocol::{AttemptId, FunctionArn, HandlerTimeout, Outcome, Payload, TraceId}; const MAILBOX_CAPACITY: usize = 8; const MAX_IDLE_LEASE: Duration = Duration::from_hours(24); @@ -40,6 +40,9 @@ pub enum SessionError { /// Command, idle lease or attempt deadline expired. #[error("runtime session deadline exceeded")] Deadline, + /// The delivered handler's separate function timeout expired first. + #[error("function handler timeout exceeded")] + HandlerTimeout, /// Caller or environment owner cancelled the session. #[error("runtime session cancelled")] Cancelled, @@ -57,6 +60,7 @@ pub enum SessionError { pub struct Invocation { attempt: AttemptId, deadline: Instant, + handler_timeout: Option, function_arn: FunctionArn, trace_id: TraceId, payload: Payload, @@ -90,12 +94,48 @@ impl Invocation { Ok(Self { attempt, deadline, + handler_timeout: None, function_arn, trace_id, payload, }) } + /// Attach a validated handler budget; it starts only at consumed delivery. + /// + /// ```no_run + /// # fn timed(call: rustack_lambda_runtime::session::Invocation, timeout: rustack_lambda_runtime::protocol::HandlerTimeout) { + /// let timed = call.with_handler_timeout(timeout); + /// # } + /// ``` + #[must_use] + pub fn with_handler_timeout(mut self, timeout: HandlerTimeout) -> Self { + self.handler_timeout = Some(timeout); + self + } + + /// Read the relative handler budget without arming it. + /// + /// ```no_run + /// # fn inspect(call: &rustack_lambda_runtime::session::Invocation) { let timeout = call.handler_timeout(); } + /// ``` + #[must_use] + pub fn handler_timeout(&self) -> Option { + self.handler_timeout + } + + fn timing(&self, started: Instant) -> Result<(Instant, SessionError), SessionError> { + if let Some(timeout) = self.handler_timeout { + let handler = started + .checked_add(timeout.duration()) + .ok_or(SessionError::InvalidMetadata)?; + if handler < self.deadline { + return Ok((handler, SessionError::HandlerTimeout)); + } + } + Ok((self.deadline, SessionError::Deadline)) + } + /// Borrow all event metadata and payload without copying bytes. /// /// ``` @@ -432,11 +472,15 @@ impl SessionHandle { check_deadline(deadline)?; let (reply, response) = oneshot::channel(); self.send(Command::Next { reply }, deadline)?; - let delivery = receive(response, deadline).await??; + let mut delivery = receive(response, deadline).await??; + let started = Instant::now(); + let (effective, _) = delivery.invocation.timing(started)?; + check_deadline(effective)?; + delivery.invocation.deadline = effective; delivery .receipt - .send(()) - .map_err(|()| SessionError::Unavailable)?; + .send(started) + .map_err(|_| SessionError::Unavailable)?; Ok(delivery.invocation) } @@ -603,13 +647,16 @@ enum Command { struct Pending { attempt: AttemptId, deadline: Instant, + timeout_error: SessionError, cancellation: watch::Receiver, response: oneshot::Sender>, } impl Pending { fn check(&self) -> Result<(), SessionError> { - check_deadline(self.deadline)?; + if Instant::now() >= self.deadline { + return Err(self.timeout_error); + } if *self.cancellation.borrow() || self.cancellation.has_changed().is_err() { Err(SessionError::Cancelled) } else { @@ -621,13 +668,13 @@ impl Pending { #[derive(Debug)] struct PollDelivery { invocation: Invocation, - receipt: oneshot::Sender<()>, + receipt: oneshot::Sender, } #[derive(Debug)] enum PhaseEvent { Cancelled, - Receipt(bool), + Receipt(Option), } #[derive(Debug, Default)] @@ -641,7 +688,7 @@ enum Phase { Offering { invocation: Invocation, pending: Pending, - receipt: oneshot::Receiver<()>, + receipt: oneshot::Receiver, }, Delivered(Pending), Closed, @@ -665,7 +712,7 @@ impl Phase { tokio::select! { biased; () = async { let _ = pending.cancellation.wait_for(|cancelled| *cancelled).await; } => PhaseEvent::Cancelled, - result = receipt => PhaseEvent::Receipt(result.is_ok()), + result = receipt => PhaseEvent::Receipt(result.ok()), } } Self::Queued { pending, .. } | Self::Delivered(pending) => { @@ -696,6 +743,14 @@ struct SessionActor { impl SessionActor { async fn run(mut self) { loop { + if *self.stop.borrow() { + self.close(SessionError::Cancelled); + break; + } + self.reconcile_receipt(); + if matches!(self.phase, Phase::Closed) { + break; + } let deadline = self .phase .pending() @@ -703,7 +758,15 @@ impl SessionActor { tokio::select! { biased; () = async { let _ = self.stop.wait_for(|stopped| *stopped).await; } => { self.close(SessionError::Cancelled); break; } - () = sleep_until(deadline) => { self.close(SessionError::Deadline); break; } + () = sleep_until(deadline) => { + self.reconcile_receipt(); + if matches!(self.phase, Phase::Closed) { break; } + let error = self.phase.pending().map_or_else( + || (Instant::now() >= self.idle_deadline).then_some(SessionError::Deadline), + |pending| pending.check().err(), + ); + if let Some(error) = error { self.close(error); break; } + } event = self.phase.event() => { match event { PhaseEvent::Cancelled => { self.close(SessionError::Cancelled); break; } @@ -763,6 +826,7 @@ impl SessionActor { let pending = Pending { attempt: invocation.attempt, deadline: invocation.deadline, + timeout_error: SessionError::Deadline, cancellation, response, }; @@ -828,8 +892,8 @@ impl SessionActor { fn reconcile_receipt(&mut self) { let received = match &mut self.phase { Phase::Offering { receipt, .. } => match receipt.try_recv() { - Ok(()) => Some(true), - Err(TryRecvError::Closed) => Some(false), + Ok(started) => Some(Some(started)), + Err(TryRecvError::Closed) => Some(None), Err(TryRecvError::Empty) => None, }, _ => None, @@ -919,29 +983,42 @@ impl SessionActor { } } - fn received(&mut self, received: bool) { + fn received(&mut self, received: Option) { let Phase::Offering { invocation, - pending, + mut pending, .. } = replace(&mut self.phase, Phase::Idle) else { return; }; + if let Some(started) = received { + match invocation.timing(started) { + Ok((deadline, error)) => { + pending.deadline = deadline; + pending.timeout_error = error; + } + Err(error) => { + let _ = pending.response.send(Err(error)); + self.close(error); + return; + } + } + self.ever_delivered = true; + } if let Err(error) = pending.check() { let _ = pending.response.send(Err(error)); self.close(error); return; } - if received { - self.ever_delivered = true; - self.phase = Phase::Delivered(pending); + self.phase = if received.is_some() { + Phase::Delivered(pending) } else { - self.phase = Phase::Queued { + Phase::Queued { invocation, pending, - }; - } + } + }; } fn take_pending(&mut self) -> Option { diff --git a/crates/rustack-lambda-runtime/src/session/tests.rs b/crates/rustack-lambda-runtime/src/session/tests.rs index 1a9551a4..f5331c65 100644 --- a/crates/rustack-lambda-runtime/src/session/tests.rs +++ b/crates/rustack-lambda-runtime/src/session/tests.rs @@ -488,6 +488,7 @@ async fn test_should_reconcile_ready_poll_receipts_before_completion_or_initiali let pending = Pending { attempt, deadline: invocation.deadline, + timeout_error: SessionError::Deadline, cancellation, response, }; @@ -497,7 +498,7 @@ async fn test_should_reconcile_ready_poll_receipts_before_completion_or_initiali pending, receipt: acknowledged, }; - receipt.send(()).unwrap(); + receipt.send(Instant::now()).unwrap(); let (reply, result) = oneshot::channel(); if initialization { assert!(!actor.command(Command::InitError { @@ -583,6 +584,7 @@ async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_d let pending = Pending { attempt, deadline: invocation.deadline, + timeout_error: SessionError::Deadline, cancellation, response, }; @@ -625,7 +627,7 @@ async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_d pending, receipt, }; - actor.received(true); + actor.received(Some(Instant::now())); } _ => panic!("unknown test case"), } @@ -635,3 +637,186 @@ async fn test_should_revalidate_cancellation_closed_owners_and_expiry_at_every_d } } } + +#[tokio::test(start_paused = true)] +async fn test_should_arm_handler_timeout_only_when_the_runtime_consumes_the_offered_event() { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation(); + call.deadline = Instant::now() + Duration::from_secs(30); + call = call.with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let ticket = handle.submit(call.clone(), deadline()).await.unwrap(); + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!handle.is_closed()); + let mut abandoned = Box::pin(handle.next(deadline())); + assert!(poll_once(&mut abandoned).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!handle.is_closed()); + drop(abandoned); + let started = Instant::now(); + let delivered = handle.next(deadline()).await.unwrap(); + assert_eq!(delivered.parts().1, started + Duration::from_secs(1)); + assert_ne!(delivered.parts().1, call.parts().1); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(ticket.wait().await, Err(SessionError::HandlerTimeout)); + assert!(handle.is_closed()); + session.shutdown().await.unwrap(); +} +#[tokio::test(start_paused = true)] +async fn test_should_keep_outer_deadline_classification_when_shorter_or_tied_with_handler_timeout() +{ + for remaining in [Duration::from_millis(500), Duration::from_secs(1)] { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation(); + call.deadline = Instant::now() + remaining; + call = call.with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let expected = call.parts().1; + let ticket = handle.submit(call, deadline()).await.unwrap(); + assert_eq!(handle.next(deadline()).await.unwrap().parts().1, expected); + tokio::time::advance(remaining).await; + assert_eq!(ticket.wait().await, Err(SessionError::Deadline)); + session.shutdown().await.unwrap(); + } +} +#[tokio::test(start_paused = true)] +async fn test_should_reject_late_handler_completion_and_delayed_consumption_receipt_without_replay() +{ + for transition in ["receipt", "complete", "receipt_after_outer"] { + let mut actor = transition_actor(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let attempt = call.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let pending = Pending { + attempt, + deadline: call.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }; + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + invocation: call, + pending, + receipt: acknowledged, + }; + receipt.send(Instant::now()).unwrap(); + if transition == "complete" { + actor.reconcile_receipt(); + } + tokio::time::advance(Duration::from_secs( + if transition == "receipt_after_outer" { + 6 + } else { + 1 + }, + )) + .await; + let (reply, result) = oneshot::channel(); + assert!(!actor.command(Command::Complete { + attempt, + invocation_id: None, + outcome: Outcome::Success, + payload: payload(), + reply + })); + assert_eq!( + result.await.unwrap(), + Err(if transition == "complete" { + SessionError::HandlerTimeout + } else { + SessionError::Unavailable + }) + ); + assert_eq!(received.await.unwrap(), Err(SessionError::HandlerTimeout)); + assert!(matches!(actor.phase, Phase::Closed)); + } +} +#[tokio::test(start_paused = true)] +async fn test_should_preserve_on_time_completion_when_the_ticket_waiter_resumes_after_handler_expiry() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let ticket = handle.submit(call, deadline()).await.unwrap(); + handle.next(deadline()).await.unwrap(); + handle + .complete( + ticket.attempt(), + None, + Outcome::Success, + payload(), + deadline(), + ) + .await + .unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(ticket.wait().await, Ok(RuntimeResult::Success(payload()))); + assert!(!handle.is_closed()); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reject_an_offered_call_consumed_after_its_outer_deadline_before_receipting_delivery() + { + let mut session = RuntimeSession::start(Duration::from_secs(60)).unwrap(); + let handle = session.handle().clone(); + let mut call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + call.deadline = Instant::now() + Duration::from_secs(2); + let ticket = handle.submit(call, deadline()).await.unwrap(); + let mut next = Box::pin(handle.next(Instant::now() + Duration::from_secs(10))); + assert!(poll_once(&mut next).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(next.await, Err(SessionError::Deadline)); + assert_eq!(ticket.wait().await, Err(SessionError::Deadline)); + tokio::time::timeout(Duration::from_secs(1), handle.closed()) + .await + .unwrap(); + session.shutdown().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn test_should_reconcile_consumed_delivery_when_receipt_and_old_outer_timer_become_ready_between_actor_polls() + { + let (commands, inbox) = mpsc::channel(MAILBOX_CAPACITY); + let (stop, stopped) = watch::channel(false); + let (bootstrap, _) = watch::channel(false); + let mut actor = SessionActor::builder() + .inbox(inbox) + .stop(stopped) + .bootstrap(bootstrap) + .initialization_error(None) + .idle_lease(Duration::from_secs(60)) + .idle_deadline(Instant::now() + Duration::from_secs(60)) + .build(); + let call = invocation() + .with_handler_timeout(HandlerTimeout::try_from(Duration::from_secs(1)).unwrap()); + let attempt = call.attempt; + let (_cancel, cancellation) = watch::channel(false); + let (response, received) = oneshot::channel(); + let (receipt, acknowledged) = oneshot::channel(); + actor.phase = Phase::Offering { + pending: Pending { + attempt, + deadline: call.deadline, + timeout_error: SessionError::Deadline, + cancellation, + response, + }, + invocation: call, + receipt: acknowledged, + }; + let mut running = Box::pin(actor.run()); + assert!(poll_once(&mut running).is_pending()); + receipt.send(Instant::now()).unwrap(); + tokio::time::advance(Duration::from_secs(6)).await; + assert!(poll_once(&mut running).is_ready()); + assert_eq!(received.await.unwrap(), Err(SessionError::HandlerTimeout)); + drop((commands, stop)); +} diff --git a/crates/rustack-lambda-runtime/tests/docker.rs b/crates/rustack-lambda-runtime/tests/docker.rs index 69486fc1..1250b572 100644 --- a/crates/rustack-lambda-runtime/tests/docker.rs +++ b/crates/rustack-lambda-runtime/tests/docker.rs @@ -347,6 +347,7 @@ impl Environment { self.send(Message::Invoke { attempt, deadline_unix_ms: deadline_unix_ms(Instant::now() + Duration::from_secs(60))?, + handler_timeout_ms: None, function_arn: "arn:aws:lambda:us-east-1:000000000000:function:spike:prod".parse()?, trace_id: local_trace_id()?, payload: Payload::try_from(Bytes::from(serde_json::to_vec(&event)?))?, diff --git a/specs/index.md b/specs/index.md index 4c352288..f599f15e 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership and bounded FIFO/warm/LRU pool dispatch. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership and bounded FIFO/warm/LRU pool dispatch, plus protocol-v2 delivery-based handler timeout and joined retirement. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until executor integration, exact invocation timing/log attribution, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until executor integration, invocation log attribution, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 31fcfffd..47696003 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -268,7 +268,7 @@ Implement the four basic 2018-06-01 routes: GET invocation/next, POST invocation Inject `AWS_LAMBDA_RUNTIME_API=127.0.0.1:9001` and revision-derived Lambda name/version/memory/region/log/task-root metadata. Reject reserved-variable overrides. User environment totals≤4 KiB with Lambda key grammar. Reject NUL values and the [reserved Lambda environment keys](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime), plus the local AWS_LAMBDA_FUNCTION_TIMEOUT injection. PATH, TZ and AWS_ENDPOINT_URL remain configurable. Inject only explicitly configured local test credentials; do not inherit host AWS secret variables. -Control frames are 4-byte big-endian length plus CBOR envelope, protocol version 1. Maximum frame is 6 MiB+64 KiB, with metadata≤64 KiB and payload encoded as CBOR bytes rather than base64. Message variants are Hello, Ready, BootstrapReady, Invoke, InvocationResult, InitError, Cancel, Close and Ack. Include environment UUID and invocation-attempt UUID. Hello advertises protocol version and architecture; mismatch retires the environment. EOF, malformed frames or attach failure invalidate it. Never reconnect and silently replay an invocation. Relay stdout carries frames only; stderr carries bounded redacted diagnostics. Collect function stdout/stderr separately from its own container. Enforce an independent subscriber filter that rejects all Bollard events more verbose than INFO regardless of RUST_LOG; the selected dependency logs HTTP request/response bodies and trailing attach bytes at trace/debug. Test secret-bearing create requests and truncated attach frames under RUST_LOG=trace. Application-level redacted Debug alone is insufficient. +Control frames are 4-byte big-endian length plus CBOR envelope, protocol version 2. Maximum frame is 6 MiB+64 KiB, with metadata≤64 KiB and payload encoded as CBOR bytes rather than base64. Message variants are Hello, Ready, BootstrapReady, Invoke, InvocationResult, InvocationTimedOut, InitError, Cancel, Close and Ack. Include environment UUID and invocation-attempt UUID. Hello advertises protocol version and architecture; mismatch retires the environment. EOF, malformed frames or attach failure invalidate it. Never reconnect and silently replay an invocation. Relay stdout carries frames only; stderr carries bounded redacted diagnostics. Collect function stdout/stderr separately from its own container. Enforce an independent subscriber filter that rejects all Bollard events more verbose than INFO regardless of RUST_LOG; the selected dependency logs HTTP request/response bodies and trailing attach bytes at trace/debug. Test secret-bearing create requests and truncated attach frames under RUST_LOG=trace. Application-level redacted Debug alone is insufficient. Create the relay with Tty=false, OpenStdin=true and StdinOnce=true. Establish bidirectional attach before starting it so the initial Hello cannot be lost; then start and await Hello/Ready before creating the function container. Demultiplex Docker stdout/stderr frames before decoding CBOR, preserve partial-frame buffering within the maximum length, and cap/control metadata before allocation. Relay protocol output never passes through a text logger. Shutdown closes the input only after Close/Ack or deadline expiry. P0 experimentally showed StdinOnce=false leaves the relay running after attach-input shutdown, while true delivers EOF and terminates the relay. Each environment owns exactly one attach connection; reconnect/replay remains forbidden. Keep attach IO in separately supervised read/write tasks so slow log collection cannot deadlock invocation completion or cancellation. @@ -298,15 +298,16 @@ Ready means the relay is listening; bootstrap-ready means the RIC issued its fir Add the host-portable library and Linux relay binary in `crates/rustack-lambda-runtime`. Lambda core depends on its library; the runtime crate never depends on lambda-core, Squib, the AWS SDK or Docker. Its protocol module owns framing and validated message types, its session module owns the single-invocation state machine, and transport adapters own HTTP/stdio IO and supervision. Deliver these contracts before the Docker consumer. -Protocol version1 encodes one definite-length CBOR map with exactly `version` (unsigned1), `environment` (canonical nonnil UUID string) and `message` (adjacently tagged `kind`/`data` map). Unit messages omit data. All identifiers are canonical lowercase hyphenated UUID strings. The message kind and data schema are: +Protocol version2 encodes one definite-length CBOR map with exactly `version` (unsigned2), `environment` (canonical nonnil UUID string) and `message` (adjacently tagged `kind`/`data` map). Unit messages omit data. All identifiers are canonical lowercase hyphenated UUID strings. The message kind and data schema are: | kind | Direction | data fields | |---|---|---| | hello | Relay→host | architecture: x86_64/arm64 | | ready | Relay→host | None | | bootstrapReady | Relay→host | None | -| invoke | Host→relay | attempt (UUID), deadlineUnixMs (nonzero u64), functionArn (validated Lambda ARN), traceId (bounded printable ASCII), payload (CBOR byte string≤6 MiB) | +| invoke | Host→relay | attempt (UUID), deadlineUnixMs (nonzero u64), functionArn (validated Lambda ARN), traceId (bounded printable ASCII), handlerTimeoutMs (omitted or integral whole-second milliseconds1,000..900,000; null rejects), payload (CBOR byte string≤6 MiB) | | invocationResult | Relay→host | attempt, outcome: success/functionError, payload | +| invocationTimedOut | Relay→host | attempt; no payload | | initError | Relay→host | payload | | cancel | Host→relay | attempt | | close | Host→relay | None | @@ -454,6 +455,37 @@ The actor's snapshot mailbox holds at most8 requests. Every envelope carries an Stop has a separate priority watch signal. `shutdown(&mut self)` signals stop and awaits the stored join by mutable borrow, retaining ownership if the waiter is cancelled; cache either the final snapshot or failure so repeated shutdown cannot report false success. Drop signals the same actor, which drops its owned socket without requiring an entered runtime. EOF marks capture closed; parser/IO/deadline failure is retained and observable through snapshots and fails joined shutdown. Drop the socket immediately on EOF/parser failure, but keep the actor available for final snapshots until owner stop, rather than losing already-captured bytes. Closing this log stream never proves container termination; environment cleanup must independently stop/wait/remove the function and relay. Unit tests cover binary order/tails, independent channel bounds/discards, record fragmentation under snapshot traffic, flood/stop responsiveness, quiet cancellation/EOF, malformed input, cancelled snapshot/shutdown waiters and cached supervision errors. Real wire qualification attaches to the function before start, captures explicit handler stdout/stderr, verifies its computed response and joins logs separately from container cleanup. This is a consumer primitive; it does not enable the production executor. +### 3.8 Delivery-based handler clock and protocol v2 + +The caller's absolute deadline continues to bound queueing, cold start, delivery and execution. A separate validated HandlerTimeout represents the captured function timeout (whole seconds1..900) as1,000..900,000 milliseconds on the control wire. Standalone Invocation defaults to no separate handler clock; `.with_handler_timeout(validated timeout)` attaches one without starting it. Docker environment calls always attach the immutable revision's timeout immediately before sending Invoke; callers cannot extend it. Native's shared session consumer validates/attaches the same captured timeout before allocating a process. This local on-demand profile excludes Managed Instances; [AWS timeout configuration](https://docs.aws.amazon.com/lambda/latest/dg/configuration-timeout.html) defines the applicable second-granularity limit. + +Delivery starts when SessionHandle.next consumes its offered invocation for the HTTP adapter, immediately before sending the synchronous consumption receipt. It does not start on submission, a pending GET, first-RIC readiness or an offer which is never consumed. Compute `effective = min(caller_absolute, consumed_at + handler_timeout)` with checked arithmetic. Return that effective deadline in the Invocation consumed by the HTTP adapter, hence in Lambda-Runtime-Deadline-Ms; the [Runtime API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html) defines this header as the function's timeout timestamp. The chosen Rustack boundary precedes HTTP response transmission; network transmission consumes the same handler budget and an uncertain/failed transmission retires the environment without replay. + +The receipt carries the monotonic consumed_at Instant. The actor retains the original unshortened Invocation while Offering; only a successful receipt installs the same effective deadline into Pending. A dropped offer restores the original queued call without starting or consuming handler time. Reconcile receipt before Complete/InitError and every expiry decision; a delayed receipt processed after its effective deadline must immediately settle as expired rather than accept a late result. Keep both absolute and handler classification: if handler expiry is strictly earlier, return SessionError::HandlerTimeout; otherwise outer expiry (including an exact tie) remains SessionError::Deadline. InvocationTicket still enforces its outer deadline when consumed; a valid result completed before effective expiry remains valid even if its waiter resumes after handler expiry but before outer expiry. Cancellation and environment stop retain their independent priority. + +Protocol v2 adds optional handlerTimeoutMs to Invoke and a distinct InvocationTimedOut(attempt) relay result. Reject version1 instead of silently running with an older relay that cannot enforce this contract; rebuild/pin the operator relay image with the host. Framing/CBOR limits and all other variants remain unchanged. Relay converts only a delivered HandlerTimeout into InvocationTimedOut, then enters Retiring and closes the Runtime API; caller deadline/transport failure is never reclassified as handler timeout. ControlClient accepts exactly one matching timeout in Busy, forbids reuse and treats it as one discardable late outcome during matching Cancel. Foreign/duplicate/idle/initializing timeouts, or a timeout without an attached handler budget, reject. InitError after a timeout is invalid because timeout proves delivery. + +EnvironmentManager maps the matching event to typed Timeout(captured duration), invalidates the attempt and completes the existing verified-ID function/control/relay retirement before replying. Pool never returns that environment to the warm LRU and does not replay the call. Native maps the shared HandlerTimeout to ExecutorError::Timeout while retaining existing cancellation, process reap and API joins. Final provider Invoke/ASL error-envelope mapping remains part of executor/integration acceptance. + +```text +Host / Environment Relay session actor Runtime API adapter / RIC + │ outer deadline + timeout │ │ + │ Invoke v2 ────────────────────▶│ Queued: outer clock only │ + │ │◀──── GET /next ───────────────│ + │ │ offer original call ─────────▶│ + │ │◀── consumed_at receipt ───────│ start handler clock + │ │ effective=min(outer,start+T) │ Deadline-Ms=effective + │ │◀──── completion ──────────────│ + │◀──── success/error ───────────│ only before effective expiry │ + │ │ handler expires first │ + │◀── InvocationTimedOut(attempt)─│ close API / Retiring │ + │ kill/wait/remove function │ │ + │ join IO / remove relay │ │ + │ Timeout after settlement │ │ +``` + +Mandatory tests: timeout range/CBOR schema/version rejection; delayed/abandoned offers do not start clocks; effective header and pending deadline agree; outer-shorter/tied deadline classification; delayed receipt/late completion rejection; successful pre-expiry completion with a later waiter; client timeout correlation/duplicate/cancel races; HTTP and actual relay/RIC timeout retirement and replacement; Native regression gates. No startup/network/probe readiness is inferred from this feature. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. @@ -516,7 +548,7 @@ One root actor owns keyed records above a JoinSet of environment workers. Each r Successful initialization is relay create/verify→attach→start→Hello/Ready→function create/verify→output attach→start→BootstrapReady. Always mutate verified container IDs, not a name that could be retargeted. Preallocated names remain the uncertainty recovery keys. Distinguish initialization payload, relay/protocol error, function early exit/OOM, cancellation and deadline errors. Init payloads are capped at32KiB. Worker states are Allocating/Initializing/Idle/Busy/Retiring/Quarantined/Removed; phase and primary bounded error are observable without payload logging. `wait_ready(deadline)` is a cancel-safe read of that state. An owner handle's Drop or explicit cancel signals retirement, independently of entered runtime. Caller loss never aborts accepted create ownership. -After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline; exact Lambda handler-timeout-at-delivery and per-invocation log attribution remain executor integration requirements and must be completed before production enablement. +After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline and attaches the immutable revision handler clock described in §3.8. Per-invocation log attribution and final executor/provider composition remain required before production enablement. Normal retirement invalidates/cancels the active attempt, then inspects/kills/waits/removes the verified function, joins log/control/bridge IO, then removes the verified relay. Keep count/memory/image/directory references until both container absence and IO settlement are established. Retry cleanup at1/2/4 seconds capped30s; failed cleanup is observable Quarantined and prevents safe readiness/snapshot success. A verified known ID's404 is terminal absence; an uncertain name's404 is not. Separate reserved cleanup concurrency prevents business saturation from starving retirement. No generic container/image deletion is permitted. @@ -550,7 +582,7 @@ Warm identity is a SHA-256 of canonical typed allocation inputs: directory owner The actor services stop and completions before new admissions and periodically expires cancelled/deadline-exhausted calls, reaps dead idle handles and revisits capacity. Dispatch FIFO head to a matching healthy Idle handle or perform one cold allocation. Known no-effect Busy admission may wait and retry; never retry a successfully admitted environment's startup or invocation. If another identity occupies idle capacity, retire the least-recently-used idle handle, await its settlement, then retry allocation. Do not evict busy environments or release their authoritative reservation early. Idle environments retain the manager's independent idle deadline. Per-identity/global warm limits apply on successful return; excess environments retire. A retired handle's identity is not transferable to another function. -Every dispatched call has a separate cancellation lane. Caller timeout/Drop, including a ready-but-unconsumed result, retires its environment. After a successful runtime result the worker waits for the public receiver's synchronous consumption receipt before returning an Idle handle to the LRU. Keep the absolute deadline and cancellation authoritative through offering and receipt waiting even when the public future remains alive but unpolled; worker-side expiry retires that environment. Receipt readiness precedes sender-closed cancellation because successful consumption synchronously acknowledges before dropping its cancellation sender. A valid FunctionError remains a business result but its environment retires. An unsuccessful invocation is not replayed. A waiting public future does not own cleanup: accepted dispatch remains with the actor until resources are reusable or retired. Supplied runtime absolute deadlines govern queue/init/invoke; delivery-based handler timeout and invocation log attribution still require the executor/runtime integration specified in §3. +Every dispatched call has a separate cancellation lane. Caller timeout/Drop, including a ready-but-unconsumed result, retires its environment. After a successful runtime result the worker waits for the public receiver's synchronous consumption receipt before returning an Idle handle to the LRU. Keep the absolute deadline and cancellation authoritative through offering and receipt waiting even when the public future remains alive but unpolled; worker-side expiry retires that environment. Receipt readiness precedes sender-closed cancellation because successful consumption synchronously acknowledges before dropping its cancellation sender. A valid FunctionError remains a business result but its environment retires. An unsuccessful invocation is not replayed. A waiting public future does not own cleanup: accepted dispatch remains with the actor until resources are reusable or retired. Supplied runtime absolute deadlines govern queue/init/invoke; the environment attaches the captured delivery-based handler clock per §3.8. Invocation log attribution and final executor/provider integration remain required. Before spawning any environment task, retain a cloneable retirement receipt (priority stop, status and cached settlement) above the JoinSet. Worker panic aborts admission and retires that receipt even though its exclusive handle was dropped. Root Drop transfers the JoinSet, all receipts and idle handles into captured-runtime emergency cleanup; abort/join workers before awaiting environment retirement. Normal shutdown closes admission, rejects queued calls, cancels dispatched calls and joins every active/idle retirement. Cancelled/timed-out shutdown waiters retain that operation; cached worker failure cannot become success after physical cleanup. Underlying quarantined resources remain accounted by EnvironmentManager and prevent clean pool shutdown. Infrastructure diagnostics never include payload, environment variables or unhashed allocation inputs. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 6498a0b3..05a0c3b9 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -123,6 +123,8 @@ Expose capability fields for workflow type/query profile/state types/integration The workflow drain interval permits already accepted executions to finish. Then cancel remaining executions with ABORTED and error=Rustack.Shutdown, preserving terminal history. This is deliberate local shutdown behavior, not cloud-style pause/resume. Revoke callback tokens, cancel owned children and await Lambda cleanup before Lambda itself quiesces. Quarantine or unfinished cleanup prevents publishing a successful consistency snapshot. Quiescence must be idempotent and share one global absolute deadline, not restart a full timeout per worker. +The host and pinned relay image must implement control protocol version2 together (§3.8 of the runtime design). An older version1 relay is incompatible and fails initialization before function allocation. Rebuild/re-pin the relay on protocol upgrades; do not silently omit handler-timeout enforcement to accept an older image. Caller absolute budgets include queue/startup/execution; the captured function handler budget arms only at consumed Runtime API delivery. Cleanup remains independently owned if the caller budget expires. + Within Lambda, stop deployment/invocation producers before retiring the pool/environment consumers. Join EnvironmentPool before EnvironmentManager, then join EnvironmentManager before revoking image leases, and join image cleanup before DockerWire shutdown. Environment count and aggregate function+relay memory remain reserved through allocating, idle, busy and quarantined states; a timed-out shutdown waiter leaves that ownership intact. A removed environment's business failure is distinct from failed supervision: known protocol/handler errors may retire cleanly, while Worker failure remains cached even after physical recovery. The lower-level owner alone is not a startup readiness receipt; historical uncertain effects and daemon/network/resource probes remain separate required gates. Snapshot coverage: diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 82c634fe..1def02f7 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -102,9 +102,11 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi `make test-lambda-environments` verifies the retained root/worker owner independently of Docker: transactional count/memory admission, scope rejection, definitive create rejection, canceled critical create,500→404→late commit, foreign label/image/name/ID quarantine, root abort recovery, revoked/connect-failed never-issued settlement, slow-health command fairness, failed versus normal queued-handle settlement, final log EOF without reuse and bounded startup under a long caller deadline. The transport regression combines an actual scope error with owner abort and requires Worker precedence/caching. Never count these scripted cases as image code execution. -`make test-lambda-pool` verifies one accepted FIFO queue across mailbox/actor waiting, canonical typed allocation identity, deadline/cancellation while waiting or offering a result, warm reuse only after consumption, zero warm retention and excess concurrent per-identity returns, one LRU eviction barrier under delayed cleanup, FIFO no-replay startup rejection, cached worker/root failure, receipt recovery and cancelled/repeated shutdown. The unpolled-live response regression requires environment removal before polling the public future again. Source inspection establishes bounds validation before fingerprint materialization; oversized-capture cases independently pin rejection. `make test-lambda-pool-docker` runs the shared four-case owner/pool real-image suite. None of these tests substitutes for full executor/provider SDK acceptance. +`make test-lambda-timeouts` runs the complete shared runtime library suite and real Native hardening fixtures. Require delayed/abandoned offers to retain the original caller deadline until consumed, no receipt/body after outer expiry, an effective Runtime API deadline matching the consumed handler budget, earlier-handler versus outer/tied classification, late receipt/completion rejection, on-time completion with a delayed ticket waiter, and strict v2 timeout/cancel correlation. A manual-poll actual actor.run regression must make the receipt and old outer timer ready between polls and require HandlerTimeout. Preserve independent Native initialization timeout classification plus process reap and separate delivered-handler timeout/quiescence. `make test-lambda-timeouts-docker` includes actual image-pool timeout retirement: a900s caller-supplied handler budget cannot override captured3s; the error is Timeout3s, allocation accounting reaches zero before that result, and replacement code starts at counter1. Rebuild/pin the version2 relay image with the host; version1 must fail protocol validation. -`make test-lambda-environments-docker` builds the required images and explicitly runs four real owner/pool cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, exact handler-timeout-at-delivery, per-invocation log attribution, startup bridge/resource probes or historical reconciliation. The pool cases use the same production owner and actual RIC to prove warm counter1/2, captured revision isolation, one-slot LRU eviction, FunctionError retirement/replacement and independently observed running-handler abandonment with joined cleanup. +`make test-lambda-pool` verifies one accepted FIFO queue across mailbox/actor waiting, canonical typed allocation identity, deadline/cancellation while waiting or offering a result, warm reuse only after consumption, zero warm retention and excess concurrent per-identity returns, one LRU eviction barrier under delayed cleanup, FIFO no-replay startup rejection, cached worker/root failure, receipt recovery and cancelled/repeated shutdown. The unpolled-live response regression requires environment removal before polling the public future again. Source inspection establishes bounds validation before fingerprint materialization; oversized-capture cases independently pin rejection. `make test-lambda-pool-docker` runs the shared five-case owner/pool real-image suite. None of these tests substitutes for full executor/provider SDK acceptance. + +`make test-lambda-environments-docker` builds the required images and explicitly runs five real owner/pool cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, per-invocation log attribution, startup bridge/resource probes or historical reconciliation. The pool cases use the same production owner and actual RIC to prove warm counter1/2, captured revision isolation, one-slot LRU eviction, FunctionError retirement/replacement and independently observed running-handler abandonment with joined cleanup. `make test-lambda-environment-plans` verifies §4.2/D30 before allocation: fixed image IDs/platforms, distinct names and ownership roles, relay/function resource/swap/tmpfs/health/log/restart policy, user selection/grammar, bounded revision/region/env/argv validation, single-key inherited→captured→runtime environment precedence with deletion markers, retained real-directory and both-image lease lifetime, and revoked availability before IO. A spawned create regression proves the wire future is Send. The actual two-case wire suite consumes EnvironmentPlan for both ordinary RIC execution and empty ImageConfig overrides: require handler-reported CPU/memory/no-swap/PIDs,512MiB tmpfs/mode, cap-drop/no-new-privileges/read-only rootfs, and configured metadata despite intentionally conflicting image defaults. Inspect real CMD/ENTRYPOINT/WORKDIR and require expected exits142/0 for inherited-empty and reset-entrypoint variants. Join control/log streams and remove both owned containers/aliases on all normal test outcomes. This is allocation-policy qualification, not lifecycle/pool, startup bridge/probe, historical reconciliation or SDK executor acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index a6ed33a2..ab200c99 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership and bounded warm/LRU pool dispatch, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker executor integration and exact invocation timing/log attribution, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership and bounded warm/LRU pool dispatch and protocol-v2 delivery-based handler timing, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker executor integration and per-invocation log attribution, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -148,6 +148,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 allocation policy | Implemented/reviewed; lifecycle owner remains | EnvironmentPlan fixes both create bodies to captured identities/platforms/revision, retained directory/image leases, validated policy and authoritative unique-key environment (§4.2/D30).54 new cases pass; W65–W67 fixed/rechecked. Core515 tests/116 doctests, workspace2,194 tests, boundary959 unit/doc passes and standard/strict/audit/deny gates pass. Both real two-case wire suites now consume plans for normal/empty ImageConfig execution, with actual resource/tmpfs/privilege/env assertions; all five shared runtime Docker regressions also pass per daemon. No production pool/executor or historical reconciliation readiness is claimed. | | 1.3 environment lifecycle | Implemented/reviewed; pool/executor integration remains | EnvironmentManager retains count/memory/plan/container/IO ownership above supervised workers, including uncertain creates, cancelled callers, panic recovery and quarantine (§4.3/D31). W68–W77 fixes and final independent recheck pass. Twenty scripted lifecycle cases plus one runtime failure-precedence regression pass. Core535 unit tests/127 doctests, workspace2,215 tests/403 gated cases, boundary800 unit tests/191 doctests and all standard/strict/audit/deny gates pass. Actual owner2, wire2 and shared runtime5 cases pass on each arm64 VZ and amd64/QEMU daemon; final owner2 cases are rechecked after the ready-reply cancellation fix. Both daemons have no owned containers/internal aliases afterward. Exact delivery-based handler timeout, invocation log attribution, warm/LRU checkout, historical reconciliation/pulls/network/probes and final executor wiring remain required before M0. | | 1.3 warm pool | Implemented/reviewed; executor composition remains | §4.4/D32 implement one bounded accepted FIFO queue, canonical validated allocation identity, receipt-gated warm return, independent worker-side receipt deadline, one-at-a-time capacity LRU eviction, zero/excess retention and retained panic/shutdown retirement. W78–W81 fixes and final independent recheck pass. Eighteen scripted pool cases pass. Core553 unit tests/132 doctests; workspace2,233 tests/405 separately gated; strict boundary818 unit tests/196 doctests (1,014 total). Standard build/test/fmt/Clippy/rustdoc, strict production checks and fresh audit/deny pass. Both arm64 VZ and amd64/QEMU explicitly pass four owner/pool real-RIC cases, including warm1/2, revision/capacity eviction, FunctionError replacement counter1 and observed running-handler abandonment; no owned containers/internal aliases remain afterward. Exact delivery-based handler timeout/log attribution and production Executor/provider/startup composition remain required. | +| 1.3 delivery clock | Implemented/reviewed; executor envelope mapping remains | §3.8/D33 implement validated relative handler budgets, consumed monotonic receipts/effective HTTP headers, strict earlier-handler versus outer/tie classification, protocol-v2 compatibility/timeout correlation, immutable captured override and joined environment retirement; Native attaches the shared clock and preserves independent initialization expiry. W82–W84 fixes and final independent recheck pass. Runtime119 unit tests/51 doctests; core553 unit tests/132 doctests. Workspace2,255 tests/406 gated; strict boundary839 unit tests/199 doctests (1,038 total); standard build/test/fmt/Clippy/rustdoc, strict production, audit and deny pass. Timeout target131 passes includes12 real Native fixtures. On each arm64 VZ and amd64/QEMU daemon, owner/pool5 + wire2 + shared runtime5 cases explicitly pass (12 per daemon); captured3s defeats caller-supplied900s, confirmed retirement releases capacity, replacement counter1 and no owned resources afterward. Six actual Native/DryRun provider scenarios pass separately; the gated Squib case is not VM evidence. Per-invocation log attribution and final Executor/provider/startup composition remain required before M0; P2–P5 remain pending. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Next implement exact handler timeout from delivery and invocation log attribution, then the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Next implement per-invocation log attribution and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 65bf6ad4..bcbc0a3e 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -364,6 +364,23 @@ Four explicit owner/pool real-RIC cases pass with zero ignored on each retained Exact handler timeout from Runtime API delivery, per-invocation log attribution, historical reconciliation, pulls/network/resource probes and real Executor/provider composition remain mandatory M0 work. P2–P5 remain required by the complete objective. +## P1 task1.3 delivery-based handler clock review + +Independent design and implementation review covers §3.8/D33: typed timeout bounds, protocol-v2 compatibility, original Offering ownership, consumed monotonic receipts, effective HTTP metadata, earlier-handler versus outer classification, strict timeout correlation and joined retirement. The final recheck has no remaining in-slice finding. Final Docker Executor/provider and ASL envelope mapping remain mandatory integration work. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W82 | P2; `crates/rustack-lambda-runtime/src/session.rs:761`, timer selection | A receipt could become ready after initial polling but before the old outer timer; biased timer selection then used stale classification. Reconcile again inside the timer branch and recheck current pending timing | Actual actor.run future manually polled before receipt, then both receipt/outer timer made ready between polls; result must be HandlerTimeout | +| W83 | P2; `crates/rustack-lambda-core/src/executor/instance.rs:477`, initialization errors | Reusing invocation Deadline mapping mislabeled independent initialization expiry as caller DeadlineExceeded. Dedicated initialization mapping checks caller cancellation/deadline, otherwise preserves Timeout(init budget) at API/process/readiness sites | Real no-poll initialization requires2s error category and joined child reap; delivered-handler fixture requires1s category, reap and quiescence | +| W84 | P3; runtime §3.1 protocol table | Optional timeout notation must not imply null support while structural preflight rejects null. Document omission-only and skip absent budget serialization | Typed round trips accept omitted/valid millisecond values; explicit null, invalid ranges and version1 reject | + +Final-source standard workspace build/test/fmt/Clippy/rustdoc all pass with2,255 tests, zero failures and406 separately resource-gated cases. Runtime has119 unit tests/51 doctests; Lambda core has553 unit tests/132 doctests. The six-library strict boundary target passes839 unit tests/199 doctests (1,038 total),12 separately gated Docker cases and all strict production checks. The timeout target passes119 runtime cases plus12 actual Native hardening fixtures. Fresh audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources all pass. No dependency/version/advisory exception was introduced. + +Final production-relay images were rebuilt for protocol v2 on both retained Engine29.5.2 daemons. Owner/pool5, wire2 and shared runtime5 cases explicitly pass with zero ignored on each arm64 VZ and amd64/QEMU daemon (12 per daemon). Actual code execution proves captured3s cannot be extended to900s by caller metadata, allocation is released before the handler-timeout result, and replacement code starts with counter1. Existing near6MiB/warm/FunctionError/cancel/EOF/Close regressions pass under v2. Post-run scoped queries find no owner/test-labeled containers or internal image aliases. Native provider regression separately passes its six actual Native/DryRun scenarios; the seventh gated Squib test is not claimed as VM execution. QEMU is not native-x86 platform acceptance. + +Per-invocation log attribution, real Docker Executor/provider envelope mapping, historical reconciliation, allowed pulls/network/resource probes and final startup acceptance remain mandatory M0 work. The ASL engine and P2–P5 remain required by the full objective. + + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 57a7201b..cb82bf63 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -290,3 +290,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: queue, execution, warm retention and cleanup have explicit bounded owners; underlying EnvironmentManager reservations remain authoritative through quarantine. - **Pinned by**: [runtime §4.4](./rustack-workflows-20-lambda-runtime-design.md#44-warm-checkout-and-retained-pool-dispatch), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. + +## D33 — Arm handler timeout at consumed Runtime API delivery + +- **Context**: A warm RIC may not poll immediately; queue/startup time must not consume the function handler timeout or extend the caller's absolute deadline. +- **Alternatives considered**: starting at host dispatch charges queue/transport time prematurely; starting on offer charges abandoned polls; retaining protocol v1 permits an older relay to silently omit timeout enforcement. +- **Decision**: version2 carries a validated relative timeout; the session's consumed poll receipt arms the effective monotonic deadline and a distinct correlated timeout result forces environment retirement. +- **Why**: HTTP metadata and actor enforcement share one delivery instant, abandoned offers remain recoverable, and handler versus outer timeout preserves error semantics without replay. +- **Pinned by**: [runtime §3.8](./rustack-workflows-20-lambda-runtime-design.md#38-delivery-based-handler-clock-and-protocol-v2), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. diff --git a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs index b791d20e..ed2b4bd9 100644 --- a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs +++ b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs @@ -8,6 +8,7 @@ use std::{ use base64::{Engine, engine::general_purpose::STANDARD}; use rustack_lambda_core::{ config::LambdaConfig, + error::LambdaServiceError, executor::ExecutorBackend, provider::{InvokeKind, InvokeOutcome, RustackLambda}, storage::FunctionStore, @@ -174,11 +175,12 @@ async fn test_should_time_out_and_reap_a_bootstrap_that_never_polls_the_runtime_ }); provider.create_function(input).await.unwrap(); let started = tokio::time::Instant::now(); + let error = provider + .invoke("revision-test", None, b"{}", InvokeKind::RequestResponse) + .await + .unwrap_err(); assert!( - provider - .invoke("revision-test", None, b"{}", InvokeKind::RequestResponse) - .await - .is_err() + matches!(error, LambdaServiceError::Internal { message } if message=="function timed out after 2s") ); assert!(started.elapsed() < Duration::from_secs(4)); let pid = tokio::fs::read_to_string(marker).await.unwrap(); @@ -614,3 +616,48 @@ async fn test_should_reject_native_event_architecture_mismatch_before_acceptance provider.quiesce(Duration::from_secs(1)).await.unwrap(); provider.shutdown().await; } + +#[cfg(unix)] +#[tokio::test] +async fn test_should_preserve_delivered_native_handler_timeout_and_join_process_retirement() { + let root = tempfile::tempdir().unwrap(); + let marker = root.path().join("started"); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Native, + init_timeout: Duration::from_secs(5), + ..Default::default() + }, + ); + let mut input = create("handler-timeout"); + input.timeout = Some(1); + input.environment = Some(Environment { + variables: Some(HashMap::from([ + ("SLEEP_SECS".into(), "60".into()), + ("STARTED_FILE".into(), marker.to_string_lossy().into_owned()), + ])), + }); + provider.create_function(input).await.unwrap(); + let error = provider + .invoke("revision-test", None, b"{}", InvokeKind::RequestResponse) + .await + .unwrap_err(); + assert!( + matches!(error,LambdaServiceError::Internal{message} if message=="function timed out after 1s") + ); + let pid = tokio::fs::read_to_string(marker).await.unwrap(); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(pid.trim()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!( + !status.success(), + "handler timeout returned before child reap" + ); + provider.quiesce(Duration::from_secs(5)).await.unwrap(); + provider.shutdown().await; +} From 9a74c85a7e22202045ad03d022fe0bf60ed30901 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 03:02:10 -0700 Subject: [PATCH 026/106] phase 1: isolate invocation log windows from diagnostic tails Implement runtime section 3.9 and D34 with opaque revocable windows, actor-retained mutation consumption receipts/deadlines, bounded independent tails and Tail/None environment integration. Resolve W85 and independently review ownership, framing, fairness and EOF behavior; document observed-stream timing limits. All standard gates pass with 2,273 workspace tests, 1,058 strict-boundary unit/doc tests and 36 log cases. Audit and deny pass. Thirteen explicit owner/pool/wire/runtime cases pass per arm64 VZ and amd64/QEMU daemon, including requested/uncaptured/silent warm calls and empty owned resources afterward. B05 initialization-error classification is recorded for the next M0 composition work. Real executor/provider response wiring, reconciliation/pulls/network/probes and P2-P5 remain required; M0 is not closed. --- Makefile | 9 + .../src/docker/environment/io.rs | 22 +- .../src/docker/environment/mod.rs | 2 +- .../src/docker/environment/worker.rs | 7 +- crates/rustack-lambda-core/src/docker/logs.rs | 215 ++++++++- .../src/docker/logs/window.rs | 130 +++++ .../src/docker/logs_tests.rs | 443 ++++++++++++++++++ crates/rustack-lambda-core/src/docker/mod.rs | 2 +- .../tests/docker_environment.rs | 36 ++ specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 33 +- .../rustack-workflows-60-operations-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 6 +- specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 15 + specs/rustack-workflows-99-decisions.md | 9 + 16 files changed, 912 insertions(+), 28 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/logs/window.rs diff --git a/Makefile b/Makefile index b9fa0e8c..0ab8a2a2 100644 --- a/Makefile +++ b/Makefile @@ -908,3 +908,12 @@ test-lambda-timeouts: test-lambda-timeouts-docker: test-lambda-environments-docker .PHONY: test-lambda-timeouts test-lambda-timeouts-docker + +# Invocation log capabilities/receipts and independent diagnostic capture. +test-lambda-log-windows: + @cargo test -p rustack-lambda-core --lib docker::logs::tests + +# Includes requested/silent/uncaptured warm-call log isolation on actual images. +test-lambda-log-windows-docker: test-lambda-environments-docker + +.PHONY: test-lambda-log-windows test-lambda-log-windows-docker diff --git a/crates/rustack-lambda-core/src/docker/environment/io.rs b/crates/rustack-lambda-core/src/docker/environment/io.rs index b5bdc871..0704e531 100644 --- a/crates/rustack-lambda-core/src/docker/environment/io.rs +++ b/crates/rustack-lambda-core/src/docker/environment/io.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use rustack_lambda_runtime::{ client::{ClientConfig, ControlClient}, diagnostics::runtime_dispatch, - protocol::{Architecture, EnvironmentId}, + protocol::{Architecture, AttemptId, EnvironmentId}, relay::RelayError, }; use tokio::{ @@ -21,7 +21,7 @@ use tracing::instrument::WithSubscriber; use super::{EnvironmentConfig, EnvironmentError, actor::Signals}; use crate::{ docker::{ - DockerAttachment, DockerError, FunctionLogs, OutputChannel, + DockerAttachment, DockerError, FunctionLogs, LogSnapshot, LogWindow, OutputChannel, connection::stopped as wait_for_stop, }, executor::ExecutorError, @@ -124,6 +124,24 @@ impl IoGuard { pub(super) async fn tail(&mut self) -> Result { let logs = self.streams.logs.as_ref().ok_or(EnvironmentError::Worker)?; let snapshot = logs.snapshot(self.config.deadline()).await?; + self.observe(snapshot) + } + pub(super) async fn begin_logs( + &mut self, + attempt: AttemptId, + ) -> Result { + let logs = self.streams.logs.as_mut().ok_or(EnvironmentError::Worker)?; + Ok(logs.begin(attempt, self.config.deadline()).await?) + } + pub(super) async fn finish_logs( + &mut self, + window: LogWindow, + ) -> Result { + let logs = self.streams.logs.as_mut().ok_or(EnvironmentError::Worker)?; + let snapshot = logs.finish(window, self.config.deadline()).await?; + self.observe(snapshot) + } + fn observe(&mut self, snapshot: LogSnapshot) -> Result { if let Some(error) = snapshot.failure { return Err(error.into()); } diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index 32f2207f..9f7551a6 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -182,7 +182,7 @@ pub struct EnvironmentCapacity { pub struct EnvironmentResponse { /// Unmodified handler success or `FunctionError` payload. pub result: RuntimeResult, - /// Combined captured final 4 KiB when requested; per-invocation attribution is separate. + /// Combined observed invocation-window tail when requested, at most 4 KiB. pub logs: Bytes, } impl fmt::Debug for EnvironmentResponse { diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs index 45822a4e..c2a654a8 100644 --- a/crates/rustack-lambda-core/src/docker/environment/worker.rs +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -278,6 +278,11 @@ impl Worker { if *call.cancellation.borrow() || call.reply.is_closed() { return Err(EnvironmentError::Cancelled); } + let log_window = io.begin_logs(attempt).await?; + self.check(deadline)?; + if *call.cancellation.borrow() || call.reply.is_closed() { + return Err(EnvironmentError::Cancelled); + } io.client()? .invoke( call.invocation.clone().with_handler_timeout( @@ -298,7 +303,7 @@ impl Worker { attempt: returned, result, }) if returned == attempt => { - let tail = io.tail().await?; + let tail = io.finish_logs(log_window).await?; let logs = if call.capture_logs { tail } else { diff --git a/crates/rustack-lambda-core/src/docker/logs.rs b/crates/rustack-lambda-core/src/docker/logs.rs index b666b9d3..231f53b2 100644 --- a/crates/rustack-lambda-core/src/docker/logs.rs +++ b/crates/rustack-lambda-core/src/docker/logs.rs @@ -1,9 +1,11 @@ //! Independent function output capture; socket reads never wait for a consumer. +mod window; + use std::{collections::VecDeque, fmt, time::Duration}; use bytes::Bytes; -use rustack_lambda_runtime::diagnostics::runtime_dispatch; +use rustack_lambda_runtime::{diagnostics::runtime_dispatch, protocol::AttemptId}; use tokio::{ io::AsyncRead, runtime::Handle, @@ -12,6 +14,8 @@ use tokio::{ time::{Instant, timeout_at}, }; use tracing::instrument::WithSubscriber; +pub use window::LogWindow; +use window::{ActiveWindow, PendingReceipt, ReceiptGuard, WindowKey, changed}; use super::{ DockerAttachment, DockerError, DockerOutputReader, OutputChannel, OutputChunk, check_deadline, @@ -48,7 +52,13 @@ impl fmt::Debug for LogSnapshot { } } -struct SnapshotRequest { +enum Action { + Snapshot, + Begin(ActiveWindow, PendingReceipt), + Finish(WindowKey, PendingReceipt), +} +struct Request { + action: Action, deadline: Instant, reply: oneshot::Sender>, } @@ -56,10 +66,11 @@ struct SnapshotRequest { /// Retained log actor; its environment owns the independent container lifetime. #[non_exhaustive] pub struct FunctionLogs { - requests: mpsc::Sender, + requests: mpsc::Sender, stop: watch::Sender, task: Option>>, settled: Option>, + window_mutating: bool, } impl fmt::Debug for FunctionLogs { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -67,6 +78,7 @@ impl fmt::Debug for FunctionLogs { .debug_struct("FunctionLogs") .field("stopping", &*self.stop.borrow()) .field("settled", &self.settled) + .field("window_mutation_uncertain", &self.window_mutating) .finish_non_exhaustive() } } @@ -109,6 +121,7 @@ impl FunctionLogs { stop, task: Some(task), settled: None, + window_mutating: false, }) } @@ -127,12 +140,84 @@ impl FunctionLogs { if let Some(result) = &self.settled { return result.clone(); } + self.request(Action::Snapshot, deadline).await + } + + /// Open an empty invocation window before sending its runtime event. + /// + /// # Errors + /// Returns expired/invalid budget, overload, overlapping/uncertain mutation, + /// stopped/closed capture or retained IO failure. An accepted mutation's + /// failure or cancellation forbids further window mutations; retire the owner. + /// + /// ```no_run + /// # async fn begin(logs: &mut rustack_lambda_core::docker::FunctionLogs) { + /// let window = logs.begin(Default::default(), tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn begin( + &mut self, + attempt: AttemptId, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + if self.window_mutating { + return Err(DockerError::Stopped); + } + self.window_mutating = true; + let (window, active) = LogWindow::new(attempt); + let (mut receipt, pending) = ReceiptGuard::new(deadline); + self.request(Action::Begin(active, pending), deadline) + .await?; + receipt.consume(deadline)?; + self.window_mutating = false; + Ok(window) + } + + /// Close exactly that window and return its bounded observed tail/status. + /// + /// # Errors + /// Returns budget/admission failure, foreign/stale capability, uncertain + /// mutation or retained capture failure. Caller cancellation requires owner + /// retirement. Environment diagnostic discard counters remain cumulative. + /// + /// ```no_run + /// # async fn finish(logs: &mut rustack_lambda_core::docker::FunctionLogs, window: rustack_lambda_core::docker::LogWindow) { + /// let tail = logs.finish(window, tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn finish( + &mut self, + mut window: LogWindow, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + if self.window_mutating { + return Err(DockerError::Stopped); + } + self.window_mutating = true; + let (mut receipt, pending) = ReceiptGuard::new(deadline); + let snapshot = self + .request(Action::Finish(window.key, pending), deadline) + .await?; + receipt.consume(deadline)?; + window.complete(); + self.window_mutating = false; + Ok(snapshot) + } + + async fn request(&self, action: Action, deadline: Instant) -> Result { + check_deadline(deadline)?; if *self.stop.borrow() { return Err(DockerError::Stopped); } let (reply, response) = oneshot::channel(); self.requests - .try_send(SnapshotRequest { deadline, reply }) + .try_send(Request { + action, + deadline, + reply, + }) .map_err(|error| match error { mpsc::error::TrySendError::Full(_) => DockerError::Busy, mpsc::error::TrySendError::Closed(_) => DockerError::Worker, @@ -219,6 +304,9 @@ struct Capture { stdout: Ring, stderr: Ring, tail: Ring, + window_tail: Ring, + active: Option, + receipt: Option, closed: bool, failure: Option, } @@ -228,6 +316,9 @@ impl Capture { stdout: Ring::new(capacity)?, stderr: Ring::new(capacity)?, tail: Ring::new(TAIL)?, + window_tail: Ring::new(TAIL)?, + active: None, + receipt: None, closed: false, failure: None, }) @@ -237,34 +328,107 @@ impl Capture { OutputChannel::Stdout => self.stdout.push(&chunk.bytes)?, OutputChannel::Stderr => self.stderr.push(&chunk.bytes)?, } - self.tail.push(&chunk.bytes) + self.tail.push(&chunk.bytes)?; + if self.active.is_some() { + self.window_tail.push(&chunk.bytes)?; + } + Ok(()) } fn snapshot(&self) -> LogSnapshot { + self.snapshot_of(&self.tail) + } + fn window_snapshot(&self) -> LogSnapshot { + self.snapshot_of(&self.window_tail) + } + fn snapshot_of(&self, ring: &Ring) -> LogSnapshot { LogSnapshot { - tail: Bytes::from(self.tail.bytes.iter().copied().collect::>()), + tail: Bytes::from(ring.bytes.iter().copied().collect::>()), stdout_discarded: self.stdout.discarded, stderr_discarded: self.stderr.discarded, closed: self.closed, failure: self.failure.clone(), } } - fn reply(&self, request: SnapshotRequest) { + fn fail_window(&mut self, error: DockerError) { + self.active = None; + self.receipt = None; + if self.failure.is_none() { + self.failure = Some(error); + } + } + fn reconcile(&mut self) -> bool { + if let Some(receipt) = &self.receipt { + match receipt.consumed() { + Ok(true) => { + self.receipt = None; + } + Ok(false) => {} + Err(error) => { + self.fail_window(error); + return true; + } + } + } + if self + .active + .as_ref() + .is_some_and(|window| *window.revoked.borrow() || window.revoked.has_changed().is_err()) + { + self.fail_window(DockerError::Protocol); + true + } else { + false + } + } + fn revocation(&self) -> Option> { + self.active.as_ref().map(|window| window.revoked.clone()) + } + fn reply(&mut self, request: Request) { + self.reconcile(); if request.reply.is_closed() { return; } + let mut committed = false; let result = if Instant::now() >= request.deadline { Err(DockerError::Deadline) } else { - Ok(self.snapshot()) + match request.action { + Action::Snapshot => Ok(self.snapshot()), + _ if self.failure.is_some() => { + Err(self.failure.clone().unwrap_or(DockerError::Worker)) + } + _ if self.receipt.is_some() => Err(DockerError::Busy), + Action::Begin(_, _) if self.closed => Err(DockerError::Stopped), + Action::Begin(_, _) if self.active.is_some() => Err(DockerError::Busy), + Action::Begin(window, receipt) => { + self.window_tail.bytes.clear(); + self.window_tail.discarded = 0; + self.active = Some(window); + self.receipt = Some(receipt); + committed = true; + Ok(self.window_snapshot()) + } + Action::Finish(key, receipt) + if self.active.as_ref().map(|window| window.key) == Some(key) => + { + self.active = None; + self.receipt = Some(receipt); + committed = true; + Ok(self.window_snapshot()) + } + Action::Finish(_, _) => Err(DockerError::Protocol), + } }; - let _ = request.reply.send(result); + if request.reply.send(result).is_err() && committed { + self.fail_window(DockerError::Protocol); + } } } async fn capture( reader: DockerOutputReader, mut state: Capture, - mut requests: mpsc::Receiver, + mut requests: mpsc::Receiver, mut stop: watch::Receiver, ) -> Result { let mut reader = Some(reader); @@ -273,16 +437,22 @@ async fn capture( if *stop.borrow() { break; } + if state.reconcile() { + reader.take(); + } if state.closed || state.failure.is_some() { + let receipt = state.receipt.clone(); + let revocation = state.revocation(); tokio::select! { biased; () = super::connection::stopped(&mut stop) => break, + () = changed(receipt, revocation) => { state.reconcile(); }, request = requests.recv() => match request { Some(request) => state.reply(request), None => break }, } continue; } let result = match &mut reader { - Some(reader) => read_next(reader, &state, &mut requests, &mut stop).await, + Some(reader) => read_next(reader, &mut state, &mut requests, &mut stop).await, None => return Err(DockerError::Worker), }; match result { @@ -307,8 +477,13 @@ async fn capture( drop(reader); requests.close(); while let Ok(request) = requests.try_recv() { - state.reply(request); + if matches!(request.action, Action::Snapshot) { + state.reply(request); + } else { + let _ = request.reply.send(Err(DockerError::Stopped)); + } } + state.reconcile(); match state.failure.clone() { Some(error) => Err(error), None => Ok(state.snapshot()), @@ -317,23 +492,33 @@ async fn capture( async fn read_next( reader: &mut DockerOutputReader, - state: &Capture, - requests: &mut mpsc::Receiver, + state: &mut Capture, + requests: &mut mpsc::Receiver, stop: &mut watch::Receiver, ) -> Option, DockerError>> { - // This future survives snapshot traffic. Only terminal stop drops it. + // Requests retain this read; only stop or terminal capture failure drops it. let next = reader.next(Instant::now() + SILENCE); tokio::pin!(next); loop { if *stop.borrow() { return None; } + state.reconcile(); + if let Some(error) = &state.failure { + return Some(Err(error.clone())); + } if let Ok(request) = requests.try_recv() { state.reply(request); } + if let Some(error) = &state.failure { + return Some(Err(error.clone())); + } + let receipt = state.receipt.clone(); + let revocation = state.revocation(); tokio::select! { biased; () = super::connection::stopped(stop) => return None, + () = changed(receipt, revocation) => { state.reconcile(); }, result = &mut next => return Some(result), request = requests.recv() => match request { Some(request) => state.reply(request), None => return None }, } diff --git a/crates/rustack-lambda-core/src/docker/logs/window.rs b/crates/rustack-lambda-core/src/docker/logs/window.rs new file mode 100644 index 00000000..856336a5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/logs/window.rs @@ -0,0 +1,130 @@ +//! Window revocation and mutation consumption receipts; neither Drop claims IO joins. + +use std::future::pending; + +use rustack_lambda_runtime::protocol::AttemptId; +use tokio::{ + sync::watch, + time::{Instant, sleep_until}, +}; +use uuid::Uuid; + +use super::DockerError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct WindowKey { + id: Uuid, + attempt: AttemptId, +} +impl WindowKey { + pub(super) fn new(attempt: AttemptId) -> Self { + Self { + id: Uuid::new_v4(), + attempt, + } + } +} + +/// Opaque ownership of one invocation's observed log window; consume with finish. +#[derive(Debug)] +#[non_exhaustive] +pub struct LogWindow { + pub(super) key: WindowKey, + revoke: watch::Sender, + complete: bool, +} +impl LogWindow { + pub(super) fn new(attempt: AttemptId) -> (Self, ActiveWindow) { + let key = WindowKey::new(attempt); + let (revoke, revoked) = watch::channel(false); + ( + Self { + key, + revoke, + complete: false, + }, + ActiveWindow { key, revoked }, + ) + } + pub(super) fn complete(&mut self) { + self.complete = true; + } +} +impl Drop for LogWindow { + fn drop(&mut self) { + if !self.complete { + self.revoke.send_replace(true); + } + } +} +pub(super) struct ActiveWindow { + pub(super) key: WindowKey, + pub(super) revoked: watch::Receiver, +} + +#[derive(Clone, Copy)] +enum ReceiptState { + Pending, + Consumed(Instant), + Revoked, +} +pub(super) struct ReceiptGuard { + sender: watch::Sender, + complete: bool, +} +#[derive(Clone)] +pub(super) struct PendingReceipt { + receiver: watch::Receiver, + deadline: Instant, +} +impl ReceiptGuard { + pub(super) fn new(deadline: Instant) -> (Self, PendingReceipt) { + let (sender, receiver) = watch::channel(ReceiptState::Pending); + ( + Self { + sender, + complete: false, + }, + PendingReceipt { receiver, deadline }, + ) + } + pub(super) fn consume(&mut self, deadline: Instant) -> Result<(), DockerError> { + let consumed = Instant::now(); + if consumed >= deadline { + return Err(DockerError::Deadline); + } + self.sender.send_replace(ReceiptState::Consumed(consumed)); + self.complete = true; + Ok(()) + } +} +impl Drop for ReceiptGuard { + fn drop(&mut self) { + if !self.complete { + self.sender.send_replace(ReceiptState::Revoked); + } + } +} +impl PendingReceipt { + pub(super) fn consumed(&self) -> Result { + match *self.receiver.borrow() { + ReceiptState::Consumed(at) if at < self.deadline => Ok(true), + ReceiptState::Consumed(_) => Err(DockerError::Deadline), + ReceiptState::Revoked => Err(DockerError::Protocol), + ReceiptState::Pending if Instant::now() >= self.deadline => Err(DockerError::Deadline), + ReceiptState::Pending if self.receiver.has_changed().is_err() => { + Err(DockerError::Protocol) + } + ReceiptState::Pending => Ok(false), + } + } +} +pub(super) async fn changed( + mut receipt: Option, + mut revocation: Option>, +) { + tokio::select! { biased; + ()=async {match &mut revocation { Some(receiver)=>{let _=receiver.wait_for(|revoked|*revoked).await;},None=>pending().await,}}=>{}, + ()=async {match &mut receipt {Some(receipt)=>tokio::select! {biased;_=receipt.receiver.changed()=>{},()=sleep_until(receipt.deadline)=>{}},None=>pending().await,}}=>{}, + } +} diff --git a/crates/rustack-lambda-core/src/docker/logs_tests.rs b/crates/rustack-lambda-core/src/docker/logs_tests.rs index bedf0534..e33568e9 100644 --- a/crates/rustack-lambda-core/src/docker/logs_tests.rs +++ b/crates/rustack-lambda-core/src/docker/logs_tests.rs @@ -357,3 +357,446 @@ async fn test_should_use_output_only_attach_without_replay_and_transfer_driver_o assert!(logs.shutdown().await.is_ok()); daemon.close().await; } + +async fn observed(logs: &FunctionLogs, suffix: &[u8]) { + timeout(Duration::from_secs(3), async { + while !logs + .snapshot(deadline()) + .await + .unwrap() + .tail + .ends_with(suffix) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn probe(sender: &mpsc::Sender) -> LogSnapshot { + let (reply, response) = oneshot::channel(); + sender + .try_send(Request { + action: Action::Snapshot, + deadline: deadline(), + reply, + }) + .unwrap_or_else(|_| panic!("test probe admission")); + timeout(Duration::from_secs(3), response) + .await + .unwrap() + .unwrap() + .unwrap() +} +#[tokio::test] +async fn test_should_keep_diagnostics_separate_and_never_seed_a_silent_invocation_with_prior_logs() +{ + let (mut writer, reader) = duplex(8192); + let mut logs = logs(reader); + writer.write_all(&record(1, b"idle-before")).await.unwrap(); + observed(&logs, b"idle-before").await; + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&record(1, b"first\0\xff")).await.unwrap(); + writer.write_all(&record(2, b"stderr")).await.unwrap(); + observed(&logs, b"stderr").await; + let first = logs.finish(first, deadline()).await.unwrap(); + assert_eq!(first.tail.as_ref(), b"first\0\xffstderr"); + writer.write_all(&record(2, b"idle-after")).await.unwrap(); + observed(&logs, b"idle-after").await; + let silent = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!( + logs.finish(silent, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + let third = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&record(1, b"third")).await.unwrap(); + observed(&logs, b"third").await; + assert_eq!( + logs.finish(third, deadline()).await.unwrap().tail.as_ref(), + b"third" + ); + assert!( + logs.snapshot(deadline()) + .await + .unwrap() + .tail + .starts_with(b"idle-before") + ); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_bound_binary_window_tail_and_allow_matching_finish_after_clean_eof() { + let (mut writer, reader) = duplex(16384); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer + .write_all(&record(1, &vec![0xff; TAIL + 17])) + .await + .unwrap(); + writer.write_all(&record(2, b"end\0")).await.unwrap(); + writer.shutdown().await.unwrap(); + let diagnostic = closed(&logs).await; + let finished = logs.finish(window, deadline()).await.unwrap(); + assert_eq!(finished.tail.len(), TAIL); + assert_eq!(finished.tail, diagnostic.tail); + assert!(finished.tail.ends_with(b"end\0")); + assert_eq!(finished.stdout_discarded, 17); + assert!(finished.closed); + assert!(!format!("{finished:?}").contains("end\\0")); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_revoke_capture_and_close_its_quiet_socket_when_an_open_capability_is_dropped() +{ + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + drop(window); + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(matches!( + closed(&logs).await.failure, + Some(DockerError::Protocol) + )); + assert!(logs.begin(AttemptId::default(), deadline()).await.is_err()); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_reject_cross_owner_capabilities_and_revoke_the_originating_window() { + let (_first_writer, first_reader) = duplex(8); + let (_second_writer, second_reader) = duplex(8); + let mut first = logs(first_reader); + let mut second = logs(second_reader); + let attempt = AttemptId::default(); + let foreign = first.begin(attempt, deadline()).await.unwrap(); + let own = second.begin(attempt, deadline()).await.unwrap(); + assert!(matches!( + second.finish(foreign, deadline()).await, + Err(DockerError::Protocol) + )); + assert!(matches!( + closed(&first).await.failure, + Some(DockerError::Protocol) + )); + assert!(matches!( + second.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + drop(own); + assert!(closed(&second).await.failure.is_some()); + assert!(first.shutdown().await.is_err()); + assert!(second.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_reject_a_stale_finish_without_closing_the_current_window() { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let old = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let key = old.key; + logs.finish(old, deadline()).await.unwrap(); + let current = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let (_guard, receipt) = ReceiptGuard::new(deadline()); + assert!(matches!( + logs.request(Action::Finish(key, receipt), deadline()).await, + Err(DockerError::Protocol) + )); + assert!( + logs.finish(current, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + logs.shutdown().await.unwrap(); +} +#[tokio::test] +async fn test_should_poison_window_mutation_after_an_overlapping_begin() { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Busy) + )); + assert!(matches!( + logs.finish(first, deadline()).await, + Err(DockerError::Stopped) + )); + assert!(closed(&logs).await.failure.is_some()); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_preserve_partial_record_io_across_begin_and_finish_commands() { + let (mut writer, reader) = duplex(1); + let mut logs = logs(reader); + let frame = record(1, b"fragmented"); + writer.write_all(frame.get(..4).unwrap()).await.unwrap(); + let first = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + assert!( + logs.finish(first, deadline()) + .await + .unwrap() + .tail + .is_empty() + ); + let second = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(frame.get(4..).unwrap()).await.unwrap(); + writer.shutdown().await.unwrap(); + closed(&logs).await; + assert_eq!( + logs.finish(second, deadline()).await.unwrap().tail.as_ref(), + b"fragmented" + ); + logs.shutdown().await.unwrap(); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test] +async fn test_should_retire_a_mutation_whose_ready_reply_is_abandoned(#[case] finishing: bool) { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let sender = logs.requests.clone(); + let window = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + if let Some(window) = window { + let mut finish = Box::pin(logs.finish(window, deadline())); + assert!( + finish + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + drop(finish); + } else { + let mut begin = Box::pin(logs.begin(AttemptId::default(), deadline())); + assert!( + begin + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + drop(begin); + } + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(closed(&logs).await.failure.is_some()); + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + assert!(logs.shutdown().await.is_err()); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test(start_paused = true)] +async fn test_should_expire_a_ready_mutation_receipt_while_its_public_future_remains_unpolled( + #[case] finishing: bool, +) { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let sender = logs.requests.clone(); + let until = Instant::now() + Duration::from_secs(1); + let window = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + if let Some(window) = window { + let mut finish = Box::pin(logs.finish(window, until)); + assert!( + finish + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + advance(Duration::from_secs(1)).await; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(probe(&sender).await.failure.is_some()); + assert!(matches!(finish.await, Err(DockerError::Deadline))); + } else { + let mut begin = Box::pin(logs.begin(AttemptId::default(), until)); + assert!( + begin + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + probe(&sender).await; + advance(Duration::from_secs(1)).await; + assert_eq!( + timeout(Duration::from_secs(1), writer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!(probe(&sender).await.failure.is_some()); + assert!(matches!(begin.await, Err(DockerError::Deadline))); + } + assert!(matches!( + logs.begin(AttemptId::default(), deadline()).await, + Err(DockerError::Stopped) + )); + assert!(logs.shutdown().await.is_err()); +} +#[tokio::test] +async fn test_should_retain_capture_failure_when_finishing_an_active_window() { + let (mut writer, reader) = duplex(8); + let mut logs = logs(reader); + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + writer.write_all(&[3, 0, 0, 0, 0, 0, 0, 0]).await.unwrap(); + assert!(closed(&logs).await.failure.is_some()); + assert!(matches!( + logs.finish(window, deadline()).await, + Err(DockerError::Protocol) + )); + assert!(logs.shutdown().await.is_err()); + assert!(logs.shutdown().await.is_err()); +} + +#[tokio::test(start_paused = true)] +async fn test_should_preserve_an_on_time_consumption_receipt_processed_after_its_deadline() { + let mut state = Capture::new(TAIL).unwrap(); + let (window, active) = LogWindow::new(AttemptId::default()); + let until = Instant::now() + Duration::from_secs(1); + let (mut guard, receipt) = ReceiptGuard::new(until); + let (reply, response) = oneshot::channel(); + state.reply(Request { + action: Action::Begin(active, receipt), + deadline: until, + reply, + }); + response.await.unwrap().unwrap(); + guard.consume(until).unwrap(); + advance(Duration::from_secs(2)).await; + assert!(!state.reconcile()); + assert!(state.failure.is_none()); + assert!(state.active.is_some()); + drop(window); + assert!(state.reconcile()); +} +#[rstest::rstest] +#[case::expired(false)] +#[case::abandoned(true)] +#[tokio::test] +async fn test_should_reject_unissued_window_requests_without_mutating_capture( + #[case] abandoned: bool, +) { + let mut state = Capture::new(TAIL).unwrap(); + let (window, active) = LogWindow::new(AttemptId::default()); + let until = if abandoned { + deadline() + } else { + Instant::now() + }; + let (_guard, receipt) = ReceiptGuard::new(until); + let (reply, response) = oneshot::channel(); + let response = if abandoned { + drop(response); + None + } else { + Some(response) + }; + state.reply(Request { + action: Action::Begin(active, receipt), + deadline: until, + reply, + }); + if let Some(response) = response { + assert!(matches!( + response.await.unwrap(), + Err(DockerError::Deadline) + )); + } + assert!(state.active.is_none()); + assert!(state.receipt.is_none()); + assert!(state.failure.is_none()); + drop(window); +} +#[rstest::rstest] +#[case::begin(false)] +#[case::finish(true)] +#[tokio::test] +async fn test_should_reject_queued_mutations_when_priority_shutdown_wins(#[case] finishing: bool) { + let (_writer, reader) = duplex(8); + let mut logs = logs(reader); + let (window, active) = LogWindow::new(AttemptId::default()); + let own = if finishing { + Some(logs.begin(AttemptId::default(), deadline()).await.unwrap()) + } else { + None + }; + let (_guard, receipt) = ReceiptGuard::new(deadline()); + let action = if let Some(own) = &own { + Action::Finish(own.key, receipt) + } else { + Action::Begin(active, receipt) + }; + let (reply, response) = oneshot::channel(); + logs.requests + .try_send(Request { + action, + deadline: deadline(), + reply, + }) + .unwrap_or_else(|_| panic!("test mutation admission")); + logs.shutdown().await.unwrap(); + assert!(matches!(response.await.unwrap(), Err(DockerError::Stopped))); + drop((window, own)); +} +#[tokio::test] +async fn test_should_serve_bounded_window_commands_without_starving_flooded_output_or_shutdown() { + let (mut writer, reader) = duplex(64 * 1024); + let mut logs = logs(reader); + let producer = tokio::spawn(async move { + let frame = record(1, &vec![b'x'; 16384]); + while writer.write_all(&frame).await.is_ok() { + tokio::task::yield_now().await; + } + }); + for _ in 0..16 { + let window = logs.begin(AttemptId::default(), deadline()).await.unwrap(); + let tail = logs.finish(window, deadline()).await.unwrap(); + assert!(tail.tail.len() <= TAIL); + } + timeout(Duration::from_secs(1), logs.shutdown()) + .await + .unwrap() + .unwrap(); + timeout(Duration::from_secs(1), producer) + .await + .unwrap() + .unwrap(); +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index bb8a4ea6..b6be6d5f 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -39,7 +39,7 @@ use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; pub use image::ImageResolver; pub use lease::{ImageLeaseConfig, ImageLeaseManager, ImageLeasePort, ImageLeaseStatus}; -pub use logs::{FunctionLogs, LogSnapshot}; +pub use logs::{FunctionLogs, LogSnapshot, LogWindow}; use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs index 092b943a..59ce4499 100644 --- a/crates/rustack-lambda-core/tests/docker_environment.rs +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -477,3 +477,39 @@ async fn test_should_enforce_captured_handler_timeout_and_replace_the_retired_po let cleanup = fixture.cleanup().await; result.and(retired.map_err(Into::into)).and(cleanup) } + +#[tokio::test] +#[ignore = "requires explicit Docker environment images/socket"] +async fn test_should_isolate_requested_log_tails_across_logging_silent_and_uncaptured_warm_calls() +-> Result<()> { + let mut fixture = Fixture::new().await?; + let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; + let result = async { + let revision = fixture.revision()?; + let port = pool.port(); + for (capture, emit, counter) in [(true, true, 1), (false, true, 2), (true, false, 3), (true, true, 4)] { + let response = port.invoke( + fixture.plan_revision(revision.clone())?, + invocation(&json!({"value":35,"logs":emit,"response_bytes":if emit {1024*1024} else {0}}))?, + capture, + ).await?; + let RuntimeResult::Success(payload) = response.result else { + anyhow::bail!("logging fixture invocation failed"); + }; + let value: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(value["computed"] == 42 && value["counter"] == counter, "log windows prevented actual warm reuse"); + ensure!(response.logs.len() <= 4096, "log window exceeds response bound"); + if capture && emit { + for marker in [b"WIRE_STDOUT_CAPTURE".as_slice(), b"WIRE_STDERR_CAPTURE".as_slice()] { + ensure!(response.logs.windows(marker.len()).any(|bytes| bytes == marker), "observed invocation marker missing"); + } + } else { + ensure!(response.logs.is_empty(), "silent or uncaptured invocation inherited earlier logs"); + } + } + Ok(()) + }.await; + let retired = pool.shutdown(deadline()).await; + let cleanup = fixture.cleanup().await; + result.and(retired.map_err(Into::into)).and(cleanup) +} diff --git a/specs/index.md b/specs/index.md index f599f15e..ba9ca6b6 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 runtime foundations are implemented and independently reviewed. Qualified components include shared RuntimeSession/HTTP/relay/client, bounded Docker wire, immutable image/default resolution, pin/deployment owners and provider control-plane settlement, persistent directory leases, independent function logs, immutable allocation plans and retained environment lifecycle ownership and bounded FIFO/warm/LRU pool dispatch, plus protocol-v2 delivery-based handler timeout and joined retirement. B01 dependency security, B03 strict production boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. B01 dependency security, B03 strict boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until executor integration, invocation log attribution, historical reconciliation, allowed pulls, network/probes and final wiring pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until Executor/provider response wiring, B05 initialization-error classification, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 47696003..1f322f92 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -486,6 +486,35 @@ Host / Environment Relay session actor Runtime API adapter / R Mandatory tests: timeout range/CBOR schema/version rejection; delayed/abandoned offers do not start clocks; effective header and pending deadline agree; outer-shorter/tied deadline classification; delayed receipt/late completion rejection; successful pre-expiry completion with a later waiter; client timeout correlation/duplicate/cancel races; HTTP and actual relay/RIC timeout retirement and replacement; Native regression gates. No startup/network/probe readiness is inferred from this feature. +### 3.9 Invocation-scoped log windows + +The capture actor retains environment-wide diagnostic rings, while an invocation result uses a separate4KiB window. FunctionLogs::begin(attempt, deadline) opens exactly one window before the environment sends Invoke and returns an opaque, noncloneable LogWindow capability with a fresh window UUID and the validated attempt. FunctionLogs::finish(window, deadline) validates the exact capability, copies that window's final combined bytes and closes it. A new window starts empty; it never copies the environment diagnostic tail. Windows run for every invocation, including LogType=None, so a later Tail call cannot inherit an earlier invocation's collected bytes. The existing snapshot(deadline) remains an environment-wide health/diagnostic view; it cannot serve as the invocation response. + +Membership is defined by the capture actor's observed stdout/stderr chunk order between successful Begin and Finish commands. This extends §3.7's observed-order contract. stdout/stderr and relay completion arrive on independent streams: an uninstrumented image supplies no cross-stream flush barrier or producer-side invocation labels. Buffered/background output arriving in a later window can therefore reflect earlier work; output observed outside a window stays in diagnostic rings only. Advertise this local timing limitation, retain function/revision isolation, and never claim causality or complete transport flushing. [AWS Invoke](https://docs.aws.amazon.com/lambda/latest/api/API_Invoke.html) defines the base64 final4KiB response; the executor encodes the bounded window bytes once. No handler rewrite, synthetic stdout marker, sleep-based flush interval or per-call Docker log replay is introduced. + +Reserve the extra4KiB ring fallibly when starting FunctionLogs; Begin clears/reuses that allocation. Keep one active capability at most, with no growing attempt/token history. Reject overlapping Begin, unmatched/foreign/stale Finish, Begin after EOF/failure and expired requests before mutation. Matching Finish after clean EOF may return the final window with closed=true; parser/IO failure still rejects and remains cached for joined shutdown. Snapshot discard counters retain their existing environment-wide meaning. Debug exposes IDs/counts/classes rather than bytes. LogWindow is not serializable or publicly constructible. Its Drop signals a window-specific revocation lane; losing a successfully opened capability (including cross-owner misuse) invalidates its originating capture without claiming an asynchronous join. Matching Finish disarms the capability only after its deadline-validated reply. + +The same bounded eight-request mailbox carries Snapshot/Begin/Finish. Keep the single partial-record read future pinned across all commands; byte ownership uses completed observed chunks, including fragments completed while a window is active. Preserve stop priority, request/read fairness and chunk yielding. Every command has an absolute deadline. Read-only snapshot cancellation is safe. For Begin/Finish, set a local pending/failed-mutation latch before enqueue/await and clear it only after a successful, deadline-validated receipt. Dropping/timing out a mutation future, a failed mutation or losing its capability prevents future window mutations on that handle; shutdown remains available. The actor does not mutate for an already abandoned request. A mutation reply carries a separate consumption receipt: retain it in the actor until the public future synchronously acknowledges before its absolute deadline. Watch that deadline even if a ready public future remains alive but unpolled. If a committed reply is lost, its guard is dropped or its receipt expires, fail capture and close its socket rather than permit an unacknowledged lifecycle change to appear healthy. An on-time consumption receipt remains valid if processed later. Stop rejects queued mutation commands and preserves already observable mutation failures during joined shutdown. + +The environment owns the capability inside its retained call worker. Open before control send, finish after a matching Success/FunctionError and sample closed/failure regardless of LogType. Preserve the valid business result with a clean-EOF tail but retire that environment. Any send/result/window failure, caller abandonment, timeout or worker failure follows the existing retained container/IO cleanup path; never return a window with another attempt's data or reopen after an uncertain mutation. No artificial flush-delay timers, unbounded log copies or additional Docker streams are needed. + +```text +Environment worker Function log actor Docker stdout/stderr + │ validate attempt │ │ + │ Begin(attempt) ────────────────▶│ fresh opaque window / clear4KiB │ + │◀── acknowledged LogWindow ────│ │ + │ Invoke → relay/RIC │◀── observed binary chunks ─────│ + │ │ diagnostic rings + active tail │ + │ matching result │ │ + │ Finish(LogWindow) ────────────▶│ verify capability / close window│ + │◀── current window + status ───│ │ + │ Tail or discard bytes │◀── outside-window chunks ──────│ diagnostics only + │ failure / abandoned mutation │ │ + │ retire / join ────────────────▶│ stop + joined socket cleanup │ +``` + +Required tests: silent next invocation returns no collected prior tail; idle diagnostics never seed a window; bounded binary stdout/stderr ordering and overflow; invalid/stale/cross-owner capabilities; expired/abandoned/ready-but-unconsumed mutation receipts; partial records survive window commands; EOF/failure and cached shutdown; diagnostic/window independence under flooding. Actual image-pool qualification must request logs, perform a silent warm call, then a later logging call, verifying observed markers and absence of previously collected markers without weakening real payload/error/cancel/timeout cleanup gates. This closes runtime attribution mechanics; production Executor/provider SDK response wiring remains required. + ## 4. Environment states, pooling and cancellation State enum: Allocating → Initializing → Idle → Busy(attempt) → Idle. Any stage can become Retiring → Removed; failed cleanup becomes Quarantined. PoolActor owns its map and LRU. Bounded Docker IO futures run in a supervised JoinSet; never hold a shared-state lock across IO. Give cancellation a separate priority lane or reserved capacity. @@ -548,7 +577,7 @@ One root actor owns keyed records above a JoinSet of environment workers. Each r Successful initialization is relay create/verify→attach→start→Hello/Ready→function create/verify→output attach→start→BootstrapReady. Always mutate verified container IDs, not a name that could be retargeted. Preallocated names remain the uncertainty recovery keys. Distinguish initialization payload, relay/protocol error, function early exit/OOM, cancellation and deadline errors. Init payloads are capped at32KiB. Worker states are Allocating/Initializing/Idle/Busy/Retiring/Quarantined/Removed; phase and primary bounded error are observable without payload logging. `wait_ready(deadline)` is a cancel-safe read of that state. An owner handle's Drop or explicit cancel signals retirement, independently of entered runtime. Caller loss never aborts accepted create ownership. -After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline and attaches the immutable revision handler clock described in §3.8. Per-invocation log attribution and final executor/provider composition remain required before production enablement. +After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline and attaches the immutable revision handler clock described in §3.8. Invocation-window ownership follows §3.9; final executor/provider composition remains required before production enablement. Normal retirement invalidates/cancels the active attempt, then inspects/kills/waits/removes the verified function, joins log/control/bridge IO, then removes the verified relay. Keep count/memory/image/directory references until both container absence and IO settlement are established. Retry cleanup at1/2/4 seconds capped30s; failed cleanup is observable Quarantined and prevents safe readiness/snapshot success. A verified known ID's404 is terminal absence; an uncertain name's404 is not. Separate reserved cleanup concurrency prevents business saturation from starving retirement. No generic container/image deletion is permitted. @@ -582,7 +611,7 @@ Warm identity is a SHA-256 of canonical typed allocation inputs: directory owner The actor services stop and completions before new admissions and periodically expires cancelled/deadline-exhausted calls, reaps dead idle handles and revisits capacity. Dispatch FIFO head to a matching healthy Idle handle or perform one cold allocation. Known no-effect Busy admission may wait and retry; never retry a successfully admitted environment's startup or invocation. If another identity occupies idle capacity, retire the least-recently-used idle handle, await its settlement, then retry allocation. Do not evict busy environments or release their authoritative reservation early. Idle environments retain the manager's independent idle deadline. Per-identity/global warm limits apply on successful return; excess environments retire. A retired handle's identity is not transferable to another function. -Every dispatched call has a separate cancellation lane. Caller timeout/Drop, including a ready-but-unconsumed result, retires its environment. After a successful runtime result the worker waits for the public receiver's synchronous consumption receipt before returning an Idle handle to the LRU. Keep the absolute deadline and cancellation authoritative through offering and receipt waiting even when the public future remains alive but unpolled; worker-side expiry retires that environment. Receipt readiness precedes sender-closed cancellation because successful consumption synchronously acknowledges before dropping its cancellation sender. A valid FunctionError remains a business result but its environment retires. An unsuccessful invocation is not replayed. A waiting public future does not own cleanup: accepted dispatch remains with the actor until resources are reusable or retired. Supplied runtime absolute deadlines govern queue/init/invoke; the environment attaches the captured delivery-based handler clock per §3.8. Invocation log attribution and final executor/provider integration remain required. +Every dispatched call has a separate cancellation lane. Caller timeout/Drop, including a ready-but-unconsumed result, retires its environment. After a successful runtime result the worker waits for the public receiver's synchronous consumption receipt before returning an Idle handle to the LRU. Keep the absolute deadline and cancellation authoritative through offering and receipt waiting even when the public future remains alive but unpolled; worker-side expiry retires that environment. Receipt readiness precedes sender-closed cancellation because successful consumption synchronously acknowledges before dropping its cancellation sender. A valid FunctionError remains a business result but its environment retires. An unsuccessful invocation is not replayed. A waiting public future does not own cleanup: accepted dispatch remains with the actor until resources are reusable or retired. Supplied runtime absolute deadlines govern queue/init/invoke; the environment attaches the captured delivery-based handler clock per §3.8. Invocation windows follow §3.9; final executor/provider integration remains required. Before spawning any environment task, retain a cloneable retirement receipt (priority stop, status and cached settlement) above the JoinSet. Worker panic aborts admission and retires that receipt even though its exclusive handle was dropped. Root Drop transfers the JoinSet, all receipts and idle handles into captured-runtime emergency cleanup; abort/join workers before awaiting environment retirement. Normal shutdown closes admission, rejects queued calls, cancels dispatched calls and joins every active/idle retirement. Cancelled/timed-out shutdown waiters retain that operation; cached worker failure cannot become success after physical cleanup. Underlying quarantined resources remain accounted by EnvironmentManager and prevent clean pool shutdown. Infrastructure diagnostics never include payload, environment variables or unhashed allocation inputs. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index 05a0c3b9..ffbe93d3 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -125,6 +125,8 @@ The workflow drain interval permits already accepted executions to finish. Then The host and pinned relay image must implement control protocol version2 together (§3.8 of the runtime design). An older version1 relay is incompatible and fails initialization before function allocation. Rebuild/re-pin the relay on protocol upgrades; do not silently omit handler-timeout enforcement to accept an older image. Caller absolute budgets include queue/startup/execution; the captured function handler budget arms only at consumed Runtime API delivery. Cleanup remains independently owned if the caller budget expires. +LogType=Tail uses the invocation's bounded observed log window; environment-wide diagnostic tails remain separate. Window boundaries follow actor-observed completed chunks across independent stdout/stderr and control streams. Buffered/background output can arrive in a later window; this local runtime does not claim a causal producer flush barrier. A silent invocation never copies the prior collected diagnostic tail. Lost capabilities or unconsumed/expired mutation receipts invalidate capture and require environment retirement. No flush sleeps or extra Docker log replay are used. + Within Lambda, stop deployment/invocation producers before retiring the pool/environment consumers. Join EnvironmentPool before EnvironmentManager, then join EnvironmentManager before revoking image leases, and join image cleanup before DockerWire shutdown. Environment count and aggregate function+relay memory remain reserved through allocating, idle, busy and quarantined states; a timed-out shutdown waiter leaves that ownership intact. A removed environment's business failure is distinct from failed supervision: known protocol/handler errors may retire cleanly, while Worker failure remains cached even after physical recovery. The lower-level owner alone is not a startup readiness receipt; historical uncertain effects and daemon/network/resource probes remain separate required gates. Snapshot coverage: diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 1def02f7..35c07e2f 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -102,11 +102,13 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi `make test-lambda-environments` verifies the retained root/worker owner independently of Docker: transactional count/memory admission, scope rejection, definitive create rejection, canceled critical create,500→404→late commit, foreign label/image/name/ID quarantine, root abort recovery, revoked/connect-failed never-issued settlement, slow-health command fairness, failed versus normal queued-handle settlement, final log EOF without reuse and bounded startup under a long caller deadline. The transport regression combines an actual scope error with owner abort and requires Worker precedence/caching. Never count these scripted cases as image code execution. +`make test-lambda-log-windows` verifies diagnostics remain environment-wide while each invocation window starts empty: binary order/overflow, silent next call, partial records across Begin/Finish, matching EOF finish, capture failure, stale/cross-owner capabilities and direct capability Drop. Mutation receipts require direct tests where a ready reply is abandoned or remains live/unpolled through deadline: the quiet socket must close before any later diagnostic request, and the local handle must forbid another mutation. Also cover on-time receipts processed after deadline, no-effect expired/abandoned commands, priority-stop rejection and flooded-output fairness. `make test-lambda-log-windows-docker` runs the six-case owner/pool suite: the window case alternates Tail/logging, None/logging, Tail/silent and Tail/logging through actual warm counters1..4. Require computed payloads, expected observed stdout/stderr markers and empty silent/None tails; never use the diagnostic tail as the response. The independent-stream timing limitation in runtime §3.9 remains explicit. + `make test-lambda-timeouts` runs the complete shared runtime library suite and real Native hardening fixtures. Require delayed/abandoned offers to retain the original caller deadline until consumed, no receipt/body after outer expiry, an effective Runtime API deadline matching the consumed handler budget, earlier-handler versus outer/tied classification, late receipt/completion rejection, on-time completion with a delayed ticket waiter, and strict v2 timeout/cancel correlation. A manual-poll actual actor.run regression must make the receipt and old outer timer ready between polls and require HandlerTimeout. Preserve independent Native initialization timeout classification plus process reap and separate delivered-handler timeout/quiescence. `make test-lambda-timeouts-docker` includes actual image-pool timeout retirement: a900s caller-supplied handler budget cannot override captured3s; the error is Timeout3s, allocation accounting reaches zero before that result, and replacement code starts at counter1. Rebuild/pin the version2 relay image with the host; version1 must fail protocol validation. -`make test-lambda-pool` verifies one accepted FIFO queue across mailbox/actor waiting, canonical typed allocation identity, deadline/cancellation while waiting or offering a result, warm reuse only after consumption, zero warm retention and excess concurrent per-identity returns, one LRU eviction barrier under delayed cleanup, FIFO no-replay startup rejection, cached worker/root failure, receipt recovery and cancelled/repeated shutdown. The unpolled-live response regression requires environment removal before polling the public future again. Source inspection establishes bounds validation before fingerprint materialization; oversized-capture cases independently pin rejection. `make test-lambda-pool-docker` runs the shared five-case owner/pool real-image suite. None of these tests substitutes for full executor/provider SDK acceptance. +`make test-lambda-pool` verifies one accepted FIFO queue across mailbox/actor waiting, canonical typed allocation identity, deadline/cancellation while waiting or offering a result, warm reuse only after consumption, zero warm retention and excess concurrent per-identity returns, one LRU eviction barrier under delayed cleanup, FIFO no-replay startup rejection, cached worker/root failure, receipt recovery and cancelled/repeated shutdown. The unpolled-live response regression requires environment removal before polling the public future again. Source inspection establishes bounds validation before fingerprint materialization; oversized-capture cases independently pin rejection. `make test-lambda-pool-docker` runs the shared six-case owner/pool real-image suite. None of these tests substitutes for full executor/provider SDK acceptance. -`make test-lambda-environments-docker` builds the required images and explicitly runs five real owner/pool cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance, per-invocation log attribution, startup bridge/resource probes or historical reconciliation. The pool cases use the same production owner and actual RIC to prove warm counter1/2, captured revision isolation, one-slot LRU eviction, FunctionError retirement/replacement and independently observed running-handler abandonment with joined cleanup. +`make test-lambda-environments-docker` builds the required images and explicitly runs six real owner/pool cases per selected daemon: relay/function readiness, near6MiB computed result, warm counter, allocation saturation, actual FunctionError and an observed running handler canceled by abandoning its call. Require an actual canceled task join (not any JoinError), joined observer/control/log IO, no retained allocation budget, empty owner/run container queries and joined image cleanup. Keep setup under a cleanup owner before image preparation; preserve namespace files on every unresolved setup/cleanup path. A missing image/socket fails this explicit suite. These owner primitives still do not establish provider/SDK executor acceptance and base64 log-header wiring, startup bridge/resource probes or historical reconciliation. The pool cases use the same production owner and actual RIC to prove warm counter1/2, captured revision isolation, one-slot LRU eviction, FunctionError retirement/replacement and independently observed running-handler abandonment with joined cleanup. `make test-lambda-environment-plans` verifies §4.2/D30 before allocation: fixed image IDs/platforms, distinct names and ownership roles, relay/function resource/swap/tmpfs/health/log/restart policy, user selection/grammar, bounded revision/region/env/argv validation, single-key inherited→captured→runtime environment precedence with deletion markers, retained real-directory and both-image lease lifetime, and revoked availability before IO. A spawned create regression proves the wire future is Send. The actual two-case wire suite consumes EnvironmentPlan for both ordinary RIC execution and empty ImageConfig overrides: require handler-reported CPU/memory/no-swap/PIDs,512MiB tmpfs/mode, cap-drop/no-new-privileges/read-only rootfs, and configured metadata despite intentionally conflicting image defaults. Inspect real CMD/ENTRYPOINT/WORKDIR and require expected exits142/0 for inherited-empty and reset-entrypoint variants. Join control/log streams and remove both owned containers/aliases on all normal test outcomes. This is allocation-policy qualification, not lifecycle/pool, startup bridge/probe, historical reconciliation or SDK executor acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index ab200c99..fc4e93da 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: Lambda provider/control plane, object-safe Executor, Native/Squib, immutable revision/artifact capture, invocation admission/cancellation, shared RuntimeSession/HTTP/relay/control client, bounded DockerWire, platform-aware local image resolution and owned image preparation/leases, persistent directory leases, bounded function logs, immutable environment allocation plans and retained environment lifecycle ownership and bounded warm/LRU pool dispatch and protocol-v2 delivery-based handler timing, plus existing gateway/YAML/runtime/snapshot infrastructure. Provider asynchronous image publication/status/CAS and leased invocation capture are implemented. Missing: Docker executor integration and per-invocation log attribution, historical-effect reconciliation and startup network/resource probes, allowed pulls and final configuration/capabilities, Step Functions protocol/model/core, ASL engine, task integrations, workflow snapshots and complete acceptance fixtures. The execution ledger below distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows are implemented. Missing: real Docker Executor/provider response composition and B05 initialization-error classification; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -149,6 +149,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 environment lifecycle | Implemented/reviewed; pool/executor integration remains | EnvironmentManager retains count/memory/plan/container/IO ownership above supervised workers, including uncertain creates, cancelled callers, panic recovery and quarantine (§4.3/D31). W68–W77 fixes and final independent recheck pass. Twenty scripted lifecycle cases plus one runtime failure-precedence regression pass. Core535 unit tests/127 doctests, workspace2,215 tests/403 gated cases, boundary800 unit tests/191 doctests and all standard/strict/audit/deny gates pass. Actual owner2, wire2 and shared runtime5 cases pass on each arm64 VZ and amd64/QEMU daemon; final owner2 cases are rechecked after the ready-reply cancellation fix. Both daemons have no owned containers/internal aliases afterward. Exact delivery-based handler timeout, invocation log attribution, warm/LRU checkout, historical reconciliation/pulls/network/probes and final executor wiring remain required before M0. | | 1.3 warm pool | Implemented/reviewed; executor composition remains | §4.4/D32 implement one bounded accepted FIFO queue, canonical validated allocation identity, receipt-gated warm return, independent worker-side receipt deadline, one-at-a-time capacity LRU eviction, zero/excess retention and retained panic/shutdown retirement. W78–W81 fixes and final independent recheck pass. Eighteen scripted pool cases pass. Core553 unit tests/132 doctests; workspace2,233 tests/405 separately gated; strict boundary818 unit tests/196 doctests (1,014 total). Standard build/test/fmt/Clippy/rustdoc, strict production checks and fresh audit/deny pass. Both arm64 VZ and amd64/QEMU explicitly pass four owner/pool real-RIC cases, including warm1/2, revision/capacity eviction, FunctionError replacement counter1 and observed running-handler abandonment; no owned containers/internal aliases remain afterward. Exact delivery-based handler timeout/log attribution and production Executor/provider/startup composition remain required. | | 1.3 delivery clock | Implemented/reviewed; executor envelope mapping remains | §3.8/D33 implement validated relative handler budgets, consumed monotonic receipts/effective HTTP headers, strict earlier-handler versus outer/tie classification, protocol-v2 compatibility/timeout correlation, immutable captured override and joined environment retirement; Native attaches the shared clock and preserves independent initialization expiry. W82–W84 fixes and final independent recheck pass. Runtime119 unit tests/51 doctests; core553 unit tests/132 doctests. Workspace2,255 tests/406 gated; strict boundary839 unit tests/199 doctests (1,038 total); standard build/test/fmt/Clippy/rustdoc, strict production, audit and deny pass. Timeout target131 passes includes12 real Native fixtures. On each arm64 VZ and amd64/QEMU daemon, owner/pool5 + wire2 + shared runtime5 cases explicitly pass (12 per daemon); captured3s defeats caller-supplied900s, confirmed retirement releases capacity, replacement counter1 and no owned resources afterward. Six actual Native/DryRun provider scenarios pass separately; the gated Squib case is not VM evidence. Per-invocation log attribution and final Executor/provider/startup composition remain required before M0; P2–P5 remain pending. | +| 1.3 invocation logs | Implemented/reviewed; Executor/provider response composition remains | §3.9/D34 separate bounded per-attempt windows from diagnostic rings, with opaque revocable capabilities, actor-retained consumption receipts/deadlines, exact ownership, pinned partial reads and Tail/None integration. W85 self-review correction and final independent recheck pass.36 log cases (18 added) pass; core571 tests/134 doctests; workspace2,273 tests/407 gated; strict boundary857 unit tests/201 doctests (1,058 total). Standard build/test/fmt/Clippy/rustdoc, strict production, audit/deny pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes owner/pool6 + wire2 + shared runtime5 cases (13 per daemon), including real requested/uncaptured/silent log windows with warm counters1..4. Post-run owned resource queries are empty. Observed-stream timing limits remain explicit; B05 and final Executor/provider/base64 response wiring remain mandatory before M0. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Next implement per-invocation log attribution and the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. Next fix B05 initialization-error classification and compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index bcbc0a3e..fecc72c3 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -381,6 +381,20 @@ Final production-relay images were rebuilt for protocol v2 on both retained Engi Per-invocation log attribution, real Docker Executor/provider envelope mapping, historical reconciliation, allowed pulls/network/resource probes and final startup acceptance remain mandatory M0 work. The ASL engine and P2–P5 remain required by the full objective. +## P1 task1.3 invocation log window review + +Independent design and implementation review covers §3.9/D34: separate bounded diagnostic/window tails, opaque capabilities and direct revocation, consumption receipts/deadlines, exact window identity, partial-read retention/fairness, EOF handling and environment Tail/None lifecycle integration. The final read-only implementation review has no concrete in-slice finding. Preflight explicitly required capability Drop and cross-owner misuse to revoke the originating capture. Self-review corrected the unconsumed-reply case below before final review. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W85 | P2; self-review, `crates/rustack-lambda-core/src/docker/logs/window.rs:71`, mutation receipts | Sending a reply cannot prove consumption, especially after Finish removed the active capability. Retain a separate actor-owned consumption receipt and deadline until the public mutation acknowledges | Ready-but-unconsumed Begin/Finish abandonment and live/unpolled expiry require quiet-socket closure before another diagnostic request; on-time ACK processed later remains valid | + +Final-source standard workspace build/test/fmt/Clippy/rustdoc pass with2,273 tests, zero failures and407 separately resource-gated cases. Lambda core has571 unit tests/134 doctests; the log target passes36 cases (18 additions). The six-library strict boundary target passes857 unit tests/201 doctests (1,058 total),13 separately gated Docker cases and all strict production checks. Fresh audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources all pass. No dependency/version or advisory exception was introduced. + +Each retained Engine29.5.2 daemon explicitly passes owner/pool6, wire2 and shared runtime5 cases with zero ignored (13 per daemon), on arm64 VZ and amd64/QEMU. Actual image code alternates Tail/logging, None/logging, Tail/silent and Tail/logging, retaining warm counters1..4, computed42 responses, observed stdout/stderr markers, bounded tails and empty silent/uncaptured output. Existing timeout/replacement/FunctionError/cancel/large-frame/Close regressions pass with joined log/container/image cleanup. Read-only post-run queries find no owner/test-labeled containers or internal image aliases. QEMU is not native-x86 platform acceptance. + +Window attribution follows observed completed chunks, with the independent-stream timing limit advertised in runtime/operations specs. Production Executor/provider base64 header wiring, historical reconciliation, allowed pulls/network/probes and M0 SDK acceptance remain required; P2–P5 remain pending. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | @@ -389,6 +403,7 @@ Per-invocation log attribution, real Docker Executor/provider envelope mapping, | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; Lambda model/HTTP/core and shared auth/core/runtime production libraries | **Closed:** the six-library strict target passes without new ignores after correcting123 fresh baseline findings, including typed MAC initialization errors, guarded indexing and method-specific Errors contracts | Retain `make test-lambda-boundaries` through P1 acceptance; this is not global workspace strict-lint or Docker executor acceptance | | B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | +| B05 | P2; `crates/rustack-lambda-core/src/docker/environment/worker.rs:318`, call/idle event mapping | **Open M0 integration gate:** BootstrapReady does not prove event consumption; after an abandoned poll a first queued invocation may legally receive InitError. Busy/idle fallback currently erases that bounded initialization class into Protocol | Before Executor/provider acceptance, preserve <=32KiB InitError as EnvironmentError::Initialization in both paths, retain joined retirement and add a post-bootstrap/pre-delivery regression | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index cb82bf63..8ce2b8b3 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -299,3 +299,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: HTTP metadata and actor enforcement share one delivery instant, abandoned offers remain recoverable, and handler versus outer timeout preserves error semantics without replay. - **Pinned by**: [runtime §3.8](./rustack-workflows-20-lambda-runtime-design.md#38-delivery-based-handler-clock-and-protocol-v2), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. + +## D34 — Separate invocation log windows from environment diagnostics + +- **Context**: Warm reuse must not return already collected logs from an earlier call; independent stdout/control streams lack a causal flush barrier. +- **Alternatives considered**: returning the global tail leaks prior collected bytes; clearing diagnostic rings loses health evidence; inserting sleeps or rewriting images cannot establish a reliable producer-side boundary. +- **Decision**: use acknowledged Begin/Finish actor commands and an opaque per-attempt capability around one bounded reusable4KiB window, alongside existing diagnostic rings. +- **Why**: current-call observed output has bounded explicit ownership, silent calls cannot inherit the cached tail, and uncertain mutations forbid reuse. The advertised stream timing limit preserves honest compatibility. +- **Pinned by**: [runtime §3.9](./rustack-workflows-20-lambda-runtime-design.md#39-invocation-scoped-log-windows), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. From 2e92c3074f0dc47c39c2a1ac1597bddcfdd6dd58 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 03:22:03 -0700 Subject: [PATCH 027/106] phase 1: preserve pre-delivery initialization errors through retirement Classify bounded InitError consistently before and after BootstrapReady. Compose actual session, HTTP, relay, client and environment serving in four boundary regressions; a controlled known-container absence barrier proves active replies follow retirement. B05 is closed with independent review and standard plus strict boundary checks. Production Docker executor/startup and the full P1-P5 exits remain required. --- .../src/docker/environment/tests.rs | 2 + .../environment/tests/initialization.rs | 290 ++++++++++++++++++ .../src/docker/environment/worker.rs | 40 ++- specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 2 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 2 +- 8 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs diff --git a/crates/rustack-lambda-core/src/docker/environment/tests.rs b/crates/rustack-lambda-core/src/docker/environment/tests.rs index ed8768fd..30eb6e31 100644 --- a/crates/rustack-lambda-core/src/docker/environment/tests.rs +++ b/crates/rustack-lambda-core/src/docker/environment/tests.rs @@ -7,6 +7,8 @@ use crate::docker::{ tests::Daemon, }; +mod initialization; + fn deadline() -> Instant { Instant::now() + Duration::from_secs(10) } diff --git a/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs b/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs new file mode 100644 index 00000000..7a4a3d69 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/environment/tests/initialization.rs @@ -0,0 +1,290 @@ +//! Post-bootstrap initialization failures through the actual runtime stack. + +use std::{ + future::Future, + net::SocketAddr, + task::{Context, Wake, Waker}, +}; + +use rustack_lambda_runtime::{ + client::{ClientConfig, ClientEvent, ControlClient}, + http, + metadata::local_trace_id, + protocol::{Architecture, AttemptId}, + relay::{self, RelayConfig}, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, duplex, split}, + net::TcpStream, + sync::{Notify, Semaphore}, +}; + +use super::*; +use crate::docker::{DockerOutputReader, FunctionLogs, plan::ContainerRole}; + +struct Offered(Notify); +impl Wake for Offered { + fn wake(self: Arc) { + self.0.notify_one(); + } + fn wake_by_ref(self: &Arc) { + self.0.notify_one(); + } +} + +#[rstest::rstest] +#[case::idle_bounded(false, 32768)] +#[case::idle_oversized(false, 32769)] +#[case::busy_bounded(true, 32768)] +#[case::busy_oversized(true, 32769)] +#[tokio::test] +async fn test_should_preserve_init_error_after_abandoned_bootstrap_poll_and_join_retirement( + #[case] busy: bool, + #[case] bytes: usize, +) { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let planned = Arc::new(plan(&directory())); + let RuntimeFixture { + address, + session, + mut relay, + client, + } = runtime(planned.environment_id()).await; + + let offered = Arc::new(Offered(Notify::new())); + let waker = Waker::from(offered.clone()); + let mut observer = Box::pin(session.next(deadline())); + if busy { + assert!( + observer + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + } + let signals = Signals::new(); + signals.phase(EnvironmentPhase::Idle); + let (mut task, io_task, calls, mut quiet) = + start_worker(wire.port(), planned.clone(), client, signals.clone()); + let inspected = function_inspection(&planned); + let daemon_signals = signals.clone(); + let (retiring, observed) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let container_cleanup = tokio::spawn(async move { + loop { + let request = daemon.next().await; + assert!(request.head.starts_with("GET ")); + assert!(request.head.contains(&"a".repeat(64))); + if daemon_signals.status.borrow().phase == EnvironmentPhase::Retiring { + retiring.send(()).unwrap(); + timeout_at(deadline(), released).await.unwrap().unwrap(); + request.reply(404, b"{}").await; + break; + } + request.reply(200, &inspected).await; + } + daemon.close().await; + }); + let (stop, cancellation) = watch::channel(false); + let (reply, mut response) = oneshot::channel(); + if busy { + calls + .send(Call { + invocation: Invocation::new( + AttemptId::default(), + deadline(), + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture" + .parse() + .unwrap(), + local_trace_id().unwrap(), + Payload::try_from(Bytes::from_static(b"{}")).unwrap(), + ) + .unwrap(), + capture_logs: false, + cancellation, + reply, + }) + .await + .unwrap(); + timeout_at(deadline(), offered.0.notified()).await.unwrap(); + assert_eq!(signals.status.borrow().phase, EnvironmentPhase::Busy); + } else { + drop(reply); + } + // A wake proves an offer exists. Never repoll: that would acknowledge delivery. + drop(observer); + let marker = b"private initialization cause"; + let mut payload = vec![b'x'; bytes]; + payload + .get_mut(..marker.len()) + .unwrap() + .copy_from_slice(marker); + post_init_error(address, &payload).await; + timeout_at(deadline(), observed).await.unwrap().unwrap(); + assert!(!task.is_finished()); + if busy { + assert!(matches!( + response.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + } + release.send(()).unwrap(); + timeout_at(deadline(), &mut task).await.unwrap().unwrap(); + io_task.wait().await.unwrap(); + timeout_at(deadline(), relay.wait()).await.unwrap().unwrap(); + container_cleanup.await.unwrap(); + let error = signals.status.borrow().error.clone().unwrap(); + assert_class(&error, &payload); + if busy { + assert_class(&response.await.unwrap().unwrap_err(), &payload); + } + assert!(!format!("{error:?} {error}").contains(std::str::from_utf8(marker).unwrap())); + assert_eq!(quiet.read(&mut [0; 1]).await.unwrap(), 0); + assert!(TcpStream::connect(address).await.is_err()); + drop(stop); + drop(calls); + wire.shutdown().await.unwrap(); +} + +fn start_worker( + docker: DockerPort, + planned: Arc, + client: ControlClient, + signals: Signals, +) -> ( + tokio::task::JoinHandle<()>, + Arc, + mpsc::Sender, + tokio::io::DuplexStream, +) { + let (quiet, reader) = duplex(8); + let logs = FunctionLogs::start_reader( + DockerOutputReader::new(reader, Duration::from_secs(1)).unwrap(), + 4096, + ) + .unwrap(); + let config = EnvironmentConfig::default(); + let io_task = Arc::new(CleanupTask::default()); + let progress = Arc::new(cleanup::Progress::new()); + progress + .phase(ContainerRole::Function) + .send_replace(cleanup::Phase::Known("a".repeat(64).parse().unwrap())); + let worker = worker::Worker::builder() + .docker(docker) + .config(config) + .plan(planned) + .progress(progress) + .cleanup(Arc::new(Semaphore::new(2))) + .health(Arc::new(Health::default())) + .signals(signals.clone()) + .global(watch::channel(false).0) + .io(io_task.clone()) + .runtime(Handle::current()) + .build(); + let mut io = io::IoGuard::new(Handle::current(), io_task.clone(), config, signals); + io.test_streams(client, logs); + let (calls, mut mailbox) = mpsc::channel(4); + let task = tokio::spawn(async move { + worker.serve_and_retire_test(&mut io, &mut mailbox).await; + }); + (task, io_task, calls, quiet) +} + +fn function_inspection(plan: &EnvironmentPlan) -> Vec { + let mut owned = inspected(plan); + owned["Name"] = json!(format!("/{}", plan.function_key().as_str())); + owned["Config"]["Labels"]["rustack.lambda.role"] = json!("function"); + owned["State"]["Running"] = json!(true); + serde_json::to_vec(&owned).unwrap() +} + +async fn post_init_error(address: SocketAddr, payload: &[u8]) { + timeout_at(deadline(), async { + let mut stream = TcpStream::connect(address).await.unwrap(); + let head = format!( + "POST /2018-06-01/runtime/init/error HTTP/1.1\r\nHost: localhost\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n", + payload.len() + ); + stream.write_all(head.as_bytes()).await.unwrap(); + stream.write_all(payload).await.unwrap(); + let mut response = Vec::new(); + stream.take(16384).read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 202 "), "{response:?}"); + }) + .await + .unwrap(); +} + +fn assert_class(error: &EnvironmentError, payload: &[u8]) { + if payload.len() <= 32768 { + let EnvironmentError::Initialization(received) = error else { + panic!("{error:?}"); + }; + assert_eq!(received.as_ref(), payload); + } else { + assert!( + matches!(error, EnvironmentError::Docker(DockerError::Protocol)), + "{error:?}" + ); + } +} + +struct RuntimeFixture { + address: SocketAddr, + session: rustack_lambda_runtime::session::SessionHandle, + relay: relay::RelayHandle, + client: ControlClient, +} +async fn runtime(environment: EnvironmentId) -> RuntimeFixture { + let api = http::start().await.unwrap(); + let address = api.addr(); + let session = api.session().clone(); + let (host, peer) = duplex(65536); + let (reader, writer) = split(peer); + let relay = relay::start( + api, + RelayConfig::new( + environment, + Architecture::Arm64, + Duration::from_secs(600), + Duration::from_secs(1), + Duration::from_secs(10), + ) + .unwrap(), + reader, + writer, + ) + .unwrap(); + let (reader, writer) = split(host); + let mut client = ControlClient::start( + ClientConfig::new(environment, Architecture::Arm64, Duration::from_secs(1)).unwrap(), + reader, + writer, + ) + .unwrap(); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::RelayReady + ); + let mut first_poll = Box::pin(session.next(deadline())); + assert!( + first_poll + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + assert_eq!( + client.next(deadline()).await.unwrap(), + ClientEvent::BootstrapReady + ); + drop(first_poll); + + RuntimeFixture { + address, + session, + relay, + client, + } +} diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs index c2a654a8..f2f5b7c8 100644 --- a/crates/rustack-lambda-core/src/docker/environment/worker.rs +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -49,6 +49,16 @@ pub(super) type FinalReply = ( Result, ); impl Worker { + #[cfg(test)] + pub(super) async fn serve_and_retire_test( + &self, + io: &mut IoGuard, + mailbox: &mut mpsc::Receiver, + ) { + let cleaner = self.cleaner(); + let pending = self.serve(io, &cleaner, mailbox).await; + self.finish(io, &cleaner, mailbox, pending).await; + } #[cfg(test)] pub(super) async fn serve_test( &self, @@ -81,16 +91,25 @@ impl Worker { None } }; + self.finish(&mut io, &cleaner, &mut mailbox, pending).await; + Ok(()) + } + async fn finish( + &self, + io: &mut IoGuard, + cleaner: &Cleaner, + mailbox: &mut mpsc::Receiver, + pending: Option, + ) { self.signals.phase(EnvironmentPhase::Retiring); mailbox.close(); while let Ok(call) = mailbox.try_recv() { let _ = call.reply.send(Err(EnvironmentError::Stopped)); } - cleaner.retire(&self.io, Some(&mut io)).await; + cleaner.retire(&self.io, Some(io)).await; if let Some((reply, result)) = pending { let _ = reply.send(result); } - Ok(()) } fn cleaner(&self) -> Cleaner { Cleaner::builder() @@ -182,10 +201,7 @@ impl Worker { .await?; match self.next(io, deadline, Some(&self.cleaner()), None).await? { ClientEvent::BootstrapReady => Ok(()), - ClientEvent::InitError(payload) if payload.as_ref().len() <= 32768 => { - Err(EnvironmentError::Initialization(payload)) - } - _ => Err(DockerError::Protocol.into()), + event => Err(Self::event_failure(event)), } } async fn next( @@ -234,7 +250,7 @@ impl Worker { ()=wait_for_stop(&mut stopped)=>return None, ()=sleep_until(idle)=>return None, _=tick.tick()=>{let result=async{cleaner.inspect_health().await?;io.healthy().await}.await;tick.reset();if let Err(error)=result{self.signals.error(error);return None;}continue;}, - result=io.client().ok()?.next(idle)=>{self.signals.error(result.err().map_or(EnvironmentError::Docker(DockerError::Protocol),EnvironmentError::from));return None;}, + result=io.client().ok()?.next(idle)=>{self.signals.error(match result {Ok(event)=>Self::event_failure(event),Err(error)=>error.into()});return None;}, call=mailbox.recv()=>call, }; let mut call = action?; @@ -315,7 +331,7 @@ impl Worker { } Ok(EnvironmentResponse { result, logs }) } - Ok(_) => Err(DockerError::Protocol.into()), + Ok(event) => Err(Self::event_failure(event)), Err(error) => { if io .client()? @@ -329,4 +345,12 @@ impl Worker { } } } + fn event_failure(event: ClientEvent) -> EnvironmentError { + match event { + ClientEvent::InitError(payload) if payload.as_ref().len() <= 32768 => { + EnvironmentError::Initialization(payload) + } + _ => DockerError::Protocol.into(), + } + } } diff --git a/specs/index.md b/specs/index.md index ba9ca6b6..2c831045 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. B01 dependency security, B03 strict boundaries and B04 deployment validation are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until Executor/provider response wiring, B05 initialization-error classification, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until Executor/provider response wiring, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 1f322f92..95879a9a 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -577,6 +577,8 @@ One root actor owns keyed records above a JoinSet of environment workers. Each r Successful initialization is relay create/verify→attach→start→Hello/Ready→function create/verify→output attach→start→BootstrapReady. Always mutate verified container IDs, not a name that could be retargeted. Preallocated names remain the uncertainty recovery keys. Distinguish initialization payload, relay/protocol error, function early exit/OOM, cancellation and deadline errors. Init payloads are capped at32KiB. Worker states are Allocating/Initializing/Idle/Busy/Retiring/Quarantined/Removed; phase and primary bounded error are observable without payload logging. `wait_ready(deadline)` is a cancel-safe read of that state. An owner handle's Drop or explicit cancel signals retirement, independently of entered runtime. Caller loss never aborts accepted create ownership. +BootstrapReady proves a valid Runtime API poll occurred, not that an event was consumed. If that poll is abandoned, InitError remains legal in Idle or during the first queued invocation until delivery. Classify every accepted InitError of at most32KiB as EnvironmentError::Initialization in initialization, idle and busy paths. Oversized InitError and other unexpected events remain Protocol failures. Preserve redacted payload ownership and the same joined retirement before publishing an active call's failure; idle failures remain observable in status and cannot enter the warm pool. + After readiness one worker owns the ordered ControlClient, function log capture and bounded control-output bridge. Invocation commands contain an already-validated runtime Invocation and a cancellation guard. Only one active invocation is allowed; a cancellation guard retains the environment stop signal through response receipt and post-receipt deadline validation. Abandoning even a ready-but-unconsumed reply retires the environment; a successfully consumed result disarms that guard. Results remain raw success/FunctionError bytes with an optional bounded captured tail. Successful calls return to Idle; FunctionError, protocol failure, timeout, cancellation, early exit and idle expiry retire. Selecting/cancelling ControlClient.next is safe; never cancel/replay an invocation send to reuse an uncertain session. Poll bounded container/log/bridge health while initializing/idle/busy rather than inspecting per call. Schedule the next health interval after a completed check, so slow IO cannot accumulate ready timer ticks and starve queued invocation commands. This primitive enforces the supplied invocation absolute deadline and attaches the immutable revision handler clock described in §3.8. Invocation-window ownership follows §3.9; final executor/provider composition remains required before production enablement. Normal retirement invalidates/cancels the active attempt, then inspects/kills/waits/removes the verified function, joins log/control/bridge IO, then removes the verified relay. Keep count/memory/image/directory references until both container absence and IO settlement are established. Retry cleanup at1/2/4 seconds capped30s; failed cleanup is observable Quarantined and prevents safe readiness/snapshot success. A verified known ID's404 is terminal absence; an uncertain name's404 is not. Separate reserved cleanup concurrency prevents business saturation from starving retirement. No generic container/image deletion is permitted. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 35c07e2f..44330144 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -102,6 +102,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi `make test-lambda-environments` verifies the retained root/worker owner independently of Docker: transactional count/memory admission, scope rejection, definitive create rejection, canceled critical create,500→404→late commit, foreign label/image/name/ID quarantine, root abort recovery, revoked/connect-failed never-issued settlement, slow-health command fairness, failed versus normal queued-handle settlement, final log EOF without reuse and bounded startup under a long caller deadline. The transport regression combines an actual scope error with owner abort and requires Worker precedence/caching. Never count these scripted cases as image code execution. +The B05 regression composes the actual session/HTTP/relay/client with an environment worker after BootstrapReady and an abandoned poll. For an active call, observe an offered but unconsumed event with a recording waker, abandon that offer, then POST InitError. Test Idle and Busy at32KiB and32KiB+1, require the bounded initialization class versus Protocol, redacted diagnostics and joined control/log/API retirement. Scripted container absence remains distinct from real Docker code execution. + `make test-lambda-log-windows` verifies diagnostics remain environment-wide while each invocation window starts empty: binary order/overflow, silent next call, partial records across Begin/Finish, matching EOF finish, capture failure, stale/cross-owner capabilities and direct capability Drop. Mutation receipts require direct tests where a ready reply is abandoned or remains live/unpolled through deadline: the quiet socket must close before any later diagnostic request, and the local handle must forbid another mutation. Also cover on-time receipts processed after deadline, no-effect expired/abandoned commands, priority-stop rejection and flooded-output fairness. `make test-lambda-log-windows-docker` runs the six-case owner/pool suite: the window case alternates Tail/logging, None/logging, Tail/silent and Tail/logging through actual warm counters1..4. Require computed payloads, expected observed stdout/stderr markers and empty silent/None tails; never use the diagnostic tail as the response. The independent-stream timing limitation in runtime §3.9 remains explicit. `make test-lambda-timeouts` runs the complete shared runtime library suite and real Native hardening fixtures. Require delayed/abandoned offers to retain the original caller deadline until consumed, no receipt/body after outer expiry, an effective Runtime API deadline matching the consumed handler budget, earlier-handler versus outer/tied classification, late receipt/completion rejection, on-time completion with a delayed ticket waiter, and strict v2 timeout/cancel correlation. A manual-poll actual actor.run regression must make the receipt and old outer timer ready between polls and require HandlerTimeout. Preserve independent Native initialization timeout classification plus process reap and separate delivered-handler timeout/quiescence. `make test-lambda-timeouts-docker` includes actual image-pool timeout retirement: a900s caller-supplied handler budget cannot override captured3s; the error is Timeout3s, allocation accounting reaches zero before that result, and replacement code starts at counter1. Rebuild/pin the version2 relay image with the host; version1 must fail protocol validation. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index fc4e93da..3e789487 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows are implemented. Missing: real Docker Executor/provider response composition and B05 initialization-error classification; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows are implemented. Missing: real Docker Executor/provider response composition; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -150,6 +150,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 warm pool | Implemented/reviewed; executor composition remains | §4.4/D32 implement one bounded accepted FIFO queue, canonical validated allocation identity, receipt-gated warm return, independent worker-side receipt deadline, one-at-a-time capacity LRU eviction, zero/excess retention and retained panic/shutdown retirement. W78–W81 fixes and final independent recheck pass. Eighteen scripted pool cases pass. Core553 unit tests/132 doctests; workspace2,233 tests/405 separately gated; strict boundary818 unit tests/196 doctests (1,014 total). Standard build/test/fmt/Clippy/rustdoc, strict production checks and fresh audit/deny pass. Both arm64 VZ and amd64/QEMU explicitly pass four owner/pool real-RIC cases, including warm1/2, revision/capacity eviction, FunctionError replacement counter1 and observed running-handler abandonment; no owned containers/internal aliases remain afterward. Exact delivery-based handler timeout/log attribution and production Executor/provider/startup composition remain required. | | 1.3 delivery clock | Implemented/reviewed; executor envelope mapping remains | §3.8/D33 implement validated relative handler budgets, consumed monotonic receipts/effective HTTP headers, strict earlier-handler versus outer/tie classification, protocol-v2 compatibility/timeout correlation, immutable captured override and joined environment retirement; Native attaches the shared clock and preserves independent initialization expiry. W82–W84 fixes and final independent recheck pass. Runtime119 unit tests/51 doctests; core553 unit tests/132 doctests. Workspace2,255 tests/406 gated; strict boundary839 unit tests/199 doctests (1,038 total); standard build/test/fmt/Clippy/rustdoc, strict production, audit and deny pass. Timeout target131 passes includes12 real Native fixtures. On each arm64 VZ and amd64/QEMU daemon, owner/pool5 + wire2 + shared runtime5 cases explicitly pass (12 per daemon); captured3s defeats caller-supplied900s, confirmed retirement releases capacity, replacement counter1 and no owned resources afterward. Six actual Native/DryRun provider scenarios pass separately; the gated Squib case is not VM evidence. Per-invocation log attribution and final Executor/provider/startup composition remain required before M0; P2–P5 remain pending. | | 1.3 invocation logs | Implemented/reviewed; Executor/provider response composition remains | §3.9/D34 separate bounded per-attempt windows from diagnostic rings, with opaque revocable capabilities, actor-retained consumption receipts/deadlines, exact ownership, pinned partial reads and Tail/None integration. W85 self-review correction and final independent recheck pass.36 log cases (18 added) pass; core571 tests/134 doctests; workspace2,273 tests/407 gated; strict boundary857 unit tests/201 doctests (1,058 total). Standard build/test/fmt/Clippy/rustdoc, strict production, audit/deny pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes owner/pool6 + wire2 + shared runtime5 cases (13 per daemon), including real requested/uncaptured/silent log windows with warm counters1..4. Post-run owned resource queries are empty. Observed-stream timing limits remain explicit; B05 and final Executor/provider/base64 response wiring remain mandatory before M0. | +| 1.3 post-bootstrap initialization | Implemented/reviewed; full M0 remains open | Shared bounded InitError classification preserves initialization failure after an abandoned BootstrapReady poll, including an offered-but-unconsumed first event. Four actual session/HTTP/relay/client/worker composition regressions prove Idle/Busy32KiB boundaries, no active reply before a scripted known-container absence barrier, redaction and joined IO/API retirement. Independent read-only review has no findings. Environment target25 cases, workspace2,277 tests/407 separately gated, strict boundary1,062 unit+doc passes/13 Docker-gated; build/test/fmt/Clippy/rustdoc and strict production checks pass. Evidence: `/tmp/rustack-workflows-b05-{environments,build,tests,fmt,clippy,docs,boundaries}.log`. B05 closed; this is not production Executor/startup or actual Docker acceptance. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. Next fix B05 initialization-error classification and compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. Next compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index fecc72c3..1cba133d 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -403,7 +403,7 @@ Window attribution follows observed completed chunks, with the independent-strea | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; Lambda model/HTTP/core and shared auth/core/runtime production libraries | **Closed:** the six-library strict target passes without new ignores after correcting123 fresh baseline findings, including typed MAC initialization errors, guarded indexing and method-specific Errors contracts | Retain `make test-lambda-boundaries` through P1 acceptance; this is not global workspace strict-lint or Docker executor acceptance | | B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | -| B05 | P2; `crates/rustack-lambda-core/src/docker/environment/worker.rs:318`, call/idle event mapping | **Open M0 integration gate:** BootstrapReady does not prove event consumption; after an abandoned poll a first queued invocation may legally receive InitError. Busy/idle fallback currently erases that bounded initialization class into Protocol | Before Executor/provider acceptance, preserve <=32KiB InitError as EnvironmentError::Initialization in both paths, retain joined retirement and add a post-bootstrap/pre-delivery regression | +| B05 | P2; `crates/rustack-lambda-core/src/docker/environment/worker.rs`, call/idle event mapping | **Closed:** BootstrapReady does not prove event consumption; shared event classification now preserves <=32KiB InitError as Initialization in initialization/Idle/Busy, with oversized/unexpected events remaining Protocol | Four composed session/HTTP/relay/client/worker regressions cover abandoned polling/offers, Idle/Busy32KiB/32KiB+1, a blocked container-absence barrier before reply, redacted diagnostics and joined control/log/API retirement. Independent read-only review reports no findings; no actual Docker execution is claimed by these scripted-container cases | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. From c18456426bd0565239cf292a00d6b5333e36efd1 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 03:31:42 -0700 Subject: [PATCH 028/106] phase 1: retain resumable per-call pool cleanup settlement Expose start, cancel-safe response waiting and independent cancel_and_wait receipts for the Docker Executor consumer. Retain settlement senders beside Retirement records above dispatch JoinSet and through worker/root recovery. D35 and runtime section 4.5 specify the contract. Nine new pool cases and seven real-image cases on each retained daemon pass; standard and strict boundary checks pass with final independent review. Production executor/startup and complete P1-P5 exits remain required. --- crates/rustack-lambda-core/src/docker/mod.rs | 2 +- .../src/docker/pool/actor.rs | 129 ++++++-- .../src/docker/pool/mod.rs | 158 ++++++++-- .../src/docker/pool/tests.rs | 2 + .../src/docker/pool/tests/settlement.rs | 298 ++++++++++++++++++ .../tests/docker_environment.rs | 68 +++- specs/index.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 26 ++ .../rustack-workflows-70-verification-plan.md | 4 +- specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 16 + specs/rustack-workflows-99-decisions.md | 10 + 12 files changed, 662 insertions(+), 58 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index b6be6d5f..eeaccdf3 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -48,7 +48,7 @@ pub use ownership::{DirectoryLease, DirectoryLeaseError}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; pub use pin::{ImagePin, ImagePinScope}; pub use plan::{ContainerPolicy, EnvironmentPlan, NetworkSelection, RelayBudgets}; -pub use pool::{EnvironmentPool, PoolConfig, PoolPort, PoolSettings, PoolStatus}; +pub use pool::{EnvironmentPool, PoolConfig, PoolInvocation, PoolPort, PoolSettings, PoolStatus}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; diff --git a/crates/rustack-lambda-core/src/docker/pool/actor.rs b/crates/rustack-lambda-core/src/docker/pool/actor.rs index da569fc5..f7753816 100644 --- a/crates/rustack-lambda-core/src/docker/pool/actor.rs +++ b/crates/rustack-lambda-core/src/docker/pool/actor.rs @@ -37,6 +37,29 @@ pub(super) struct Completion { warm: Option, failed: bool, } +pub(super) struct Record { + retirement: Retirement, + settlement: Option>>>, + failed: bool, +} +impl Record { + fn maintenance(retirement: Retirement) -> Self { + Self { + retirement, + settlement: None, + failed: false, + } + } + fn publish(self, failed: bool) { + if let Some(settlement) = self.settlement { + settlement.send_replace(Some(if self.failed || failed { + Err(EnvironmentError::Worker) + } else { + Ok(()) + })); + } + } +} #[derive(TypedBuilder)] pub(super) struct Actor { environments: EnvironmentPort, @@ -54,7 +77,7 @@ pub(super) struct Actor { #[builder(default)] jobs: JoinSet, #[builder(default)] - receipts: HashMap, + receipts: HashMap, #[builder(default)] settled: bool, #[builder(default)] @@ -177,10 +200,14 @@ impl Actor { match environment { Ok(environment) => { call.queue.take(); - let receipt = environment.retirement(); + let record = Record { + retirement: environment.retirement(), + settlement: Some(call.settlement.clone()), + failed: false, + }; let stop = self.stop.subscribe(); let job = self.jobs.spawn(invoke(environment, key, call, stop)); - self.receipts.insert(job.id(), receipt); + self.receipts.insert(job.id(), record); } Err(EnvironmentError::Busy) => { self.pending.push_front(call); @@ -202,9 +229,12 @@ impl Actor { self.retire_receipt(receipt) } fn retire_receipt(&mut self, receipt: Retirement) -> Id { - receipt.cancel(); - let job = self.jobs.spawn(retire(receipt.clone())); - self.receipts.insert(job.id(), receipt); + self.retire_record(Record::maintenance(receipt)) + } + fn retire_record(&mut self, record: Record) -> Id { + record.retirement.cancel(); + let job = self.jobs.spawn(retire(record.retirement.clone())); + self.receipts.insert(job.id(), record); job.id() } fn completed(&mut self, result: Result<(Id, Completion), tokio::task::JoinError>) { @@ -232,10 +262,14 @@ impl Actor { self.retire(warm.environment); } } + if let Some(record) = receipt { + record.publish(completion.failed); + } } else { self.fail(); - if let Some(receipt) = receipt { - self.retire_receipt(receipt); + if let Some(mut record) = receipt { + record.failed = true; + self.retire_record(record); } } } @@ -257,11 +291,12 @@ impl Drop for Actor { let mut jobs = mem::take(&mut self.jobs); let mut receipts: Vec<_> = mem::take(&mut self.receipts).into_values().collect(); for warm in self.warm.drain(..) { - receipts.push(warm.environment.retirement()); + receipts.push(Record::maintenance(warm.environment.retirement())); drop(warm); } - for receipt in &receipts { - receipt.cancel(); + for record in &mut receipts { + record.failed = true; + record.retirement.cancel(); } jobs.abort_all(); self.emergency.start_on(&self.runtime, move || async move { @@ -270,32 +305,34 @@ impl Drop for Actor { }); } } -async fn recover(receipts: Vec) -> Result<(), ExecutorError> { +async fn recover(receipts: Vec) -> Result<(), ExecutorError> { let mut cleanup = JoinSet::new(); let mut index = HashMap::new(); let mut failed = false; - for receipt in receipts { - let job = cleanup.spawn(retire(receipt.clone())); - index.insert(job.id(), receipt); + for record in receipts { + let job = cleanup.spawn(retire(record.retirement.clone())); + index.insert(job.id(), record); } while let Some(result) = cleanup.join_next_with_id().await { let id = match &result { Ok((id, _)) => *id, Err(error) => error.id(), }; - let Some(receipt) = index.remove(&id) else { + let Some(mut record) = index.remove(&id) else { continue; }; if let Ok((_, completion)) = result { failed |= completion.failed; - if receipt.removed() { + if record.retirement.removed() { + record.publish(completion.failed); continue; } } else { failed = true; + record.failed = true; } - let job = cleanup.spawn(retire(receipt.clone())); - index.insert(job.id(), receipt); + let job = cleanup.spawn(retire(record.retirement.clone())); + index.insert(job.id(), record); } if failed { Err(ExecutorError::CleanupFailed( @@ -388,14 +425,52 @@ pub(super) fn testing_start( config: PoolConfig, warm: Vec<(String, EnvironmentHandle)>, ) -> super::EnvironmentPool { - testing_start_inner(environments, config, warm, false) + testing_start_inner(environments, config, warm, false, None) } #[cfg(test)] pub(super) fn testing_failed_worker_start( environments: EnvironmentPort, warm: Vec<(String, EnvironmentHandle)>, ) -> super::EnvironmentPool { - testing_start_inner(environments, PoolConfig::default(), warm, true) + testing_start_inner(environments, PoolConfig::default(), warm, true, None) +} +#[cfg(test)] +pub(super) fn testing_failed_call_start( + environments: EnvironmentPort, + plan: crate::docker::EnvironmentPlan, + invocation: rustack_lambda_runtime::session::Invocation, + environment: EnvironmentHandle, +) -> (super::EnvironmentPool, super::PoolInvocation) { + let deadline = invocation.parts().1; + let (stop, cancellation) = watch::channel(false); + let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); + let key = plan.warm_key().to_owned(); + let call = Call { + plan: Arc::new(plan), + invocation, + capture_logs: false, + cancellation, + reply, + queue: None, + settlement, + }; + let pool = testing_start_inner( + environments, + PoolConfig::default(), + vec![(key, environment)], + true, + Some(call), + ); + let invocation = super::PoolInvocation { + deadline, + stop, + response: Some(response), + settlement: observed, + failed: pool.port.failed.clone(), + consumed: false, + }; + (pool, invocation) } #[cfg(test)] fn testing_start_inner( @@ -403,6 +478,7 @@ fn testing_start_inner( config: PoolConfig, warm: Vec<(String, EnvironmentHandle)>, failed_worker: bool, + failed_call: Option, ) -> super::EnvironmentPool { let runtime = Handle::current(); let maximum = usize::from(config.0.queue); @@ -429,11 +505,20 @@ fn testing_start_inner( if failed_worker { let warm = actor.warm.pop_front().unwrap(); let receipt = warm.environment.retirement(); + let settlement = failed_call.as_ref().map(|call| call.settlement.clone()); let job = actor.jobs.spawn(async move { let _owned = warm; + let _call = failed_call; std::future::pending::().await }); - actor.receipts.insert(job.id(), receipt); + actor.receipts.insert( + job.id(), + Record { + retirement: receipt, + settlement, + failed: false, + }, + ); job.abort(); } let port = super::PoolPort { diff --git a/crates/rustack-lambda-core/src/docker/pool/mod.rs b/crates/rustack-lambda-core/src/docker/pool/mod.rs index 0246027f..3fd926fc 100644 --- a/crates/rustack-lambda-core/src/docker/pool/mod.rs +++ b/crates/rustack-lambda-core/src/docker/pool/mod.rs @@ -20,7 +20,8 @@ use tokio::{ use typed_builder::TypedBuilder; use super::{ - EnvironmentError, EnvironmentPlan, EnvironmentPort, EnvironmentResponse, check_deadline, + DockerError, EnvironmentError, EnvironmentPlan, EnvironmentPort, EnvironmentResponse, + check_deadline, }; use crate::invocation::CleanupTask; @@ -85,6 +86,7 @@ struct Call { cancellation: watch::Receiver, reply: oneshot::Sender, queue: Option, + settlement: watch::Sender>>, } impl Call { fn deadline(&self) -> Instant { @@ -103,17 +105,122 @@ impl Call { } } fn reject(self, error: EnvironmentError) { + // No environment has been allocated for a rejected queued call. + self.settlement.send_replace(Some(Ok(()))); let _ = self.reply.send(Offer { result: Err(error), receipt: None, }); } } -struct CancelOnDrop { +/// Exclusive response and retained cleanup receipt for one accepted pool call. +/// +/// Drop requests cancellation; only `cancel_and_wait` proves settlement. +#[non_exhaustive] +pub struct PoolInvocation { + deadline: Instant, stop: watch::Sender, + response: Option>, + settlement: watch::Receiver>>, + failed: Arc, consumed: bool, } -impl Drop for CancelOnDrop { +impl fmt::Debug for PoolInvocation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PoolInvocation") + .field("deadline", &self.deadline) + .field("consumed", &self.consumed) + .field("settled", &self.settlement.borrow().is_some()) + .finish_non_exhaustive() + } +} +impl PoolInvocation { + /// Consume one result while retaining the receiver across cancelled waiters. + /// + /// # Errors + /// Returns invocation failure, deadline, or unavailable supervision. A closed + /// response lane is not cleanup proof; use `cancel_and_wait` before releasing + /// higher-level work permits after cancellation or uncertain failure. + /// + /// ```no_run + /// # async fn result(call: &mut rustack_lambda_core::docker::PoolInvocation) { + /// let result = call.wait().await; + /// # } + /// ``` + pub async fn wait(&mut self) -> Result { + if self.consumed || self.response.is_none() { + return Err(EnvironmentError::Configuration); + } + let response = self + .response + .as_mut() + .ok_or(EnvironmentError::Configuration)?; + let received = timeout_at(self.deadline, response) + .await + .map_err(|_| EnvironmentError::Deadline)?; + self.response.take(); + let offer = received.map_err(|_| { + if self.failed.load(Ordering::Acquire) { + EnvironmentError::Worker + } else { + EnvironmentError::Stopped + } + })?; + // Remove a completed receiver even when the deadline check rejects its + // offer. Dropping that offer revokes any unconsumed warm-return receipt. + if Instant::now() >= self.deadline { + return Err(EnvironmentError::Deadline); + } + if let Some(receipt) = offer.receipt { + receipt.send(()).map_err(|()| EnvironmentError::Worker)?; + } + self.consumed = true; + offer.result + } + + /// Request cancellation and await independently owned physical settlement. + /// + /// A cancelled/expired waiter can resume with a fresh cleanup deadline. + /// Consumed success belongs to the pool and cannot be cancelled by this call. + /// + /// # Errors + /// Returns Deadline while cleanup remains retained, or Worker when recovery + /// was uncertain or the settlement producer disappeared without proof. + /// Cleanup deadlines over ten minutes return Configuration after signalling. + /// + /// ```no_run + /// # async fn stop(call: &mut rustack_lambda_core::docker::PoolInvocation) { + /// let settled = call.cancel_and_wait(tokio::time::Instant::now() + /// + std::time::Duration::from_secs(60)).await; + /// # } + /// ``` + pub async fn cancel_and_wait(&mut self, deadline: Instant) -> Result<(), EnvironmentError> { + if !self.consumed { + self.stop.send_replace(true); + self.response.take(); + } + Self::check_cleanup_deadline(deadline)?; + loop { + let result = self.settlement.borrow().clone(); + if let Some(result) = result { + return result; + } + timeout_at(deadline, self.settlement.changed()) + .await + .map_err(|_| EnvironmentError::Deadline)? + .map_err(|_| EnvironmentError::Worker)?; + Self::check_cleanup_deadline(deadline)?; + } + } + fn check_cleanup_deadline(deadline: Instant) -> Result<(), EnvironmentError> { + check_deadline(deadline).map_err(|error| match error { + DockerError::Deadline => EnvironmentError::Deadline, + DockerError::Configuration => EnvironmentError::Configuration, + error => error.into(), + }) + } +} +impl Drop for PoolInvocation { fn drop(&mut self) { if !self.consumed { self.stop.send_replace(true); @@ -156,6 +263,25 @@ impl PoolPort { invocation: Invocation, capture_logs: bool, ) -> Result { + self.start(plan, invocation, capture_logs)?.wait().await + } + /// Admit once and retain separate response/cancellation-settlement ownership. + /// + /// # Errors + /// Rejects expired calls, unavailable image pins, overload or closed admission + /// before enqueueing. Scope validation also occurs before physical allocation. + /// + /// ```no_run + /// # fn start(port: &rustack_lambda_core::docker::PoolPort, plan: rustack_lambda_core::docker::EnvironmentPlan, event: rustack_lambda_runtime::session::Invocation) { + /// let call = port.start(plan, event, false); + /// # } + /// ``` + pub fn start( + &self, + plan: EnvironmentPlan, + invocation: Invocation, + capture_logs: bool, + ) -> Result { let deadline = invocation.parts().1; if Instant::now() >= deadline { return Err(EnvironmentError::Deadline); @@ -170,11 +296,8 @@ impl PoolPort { .try_acquire_owned() .map_err(|_| EnvironmentError::Busy)?; let (stop, cancellation) = watch::channel(false); - let mut cancel = CancelOnDrop { - stop, - consumed: false, - }; let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); self.sender .try_send(Call { plan: Arc::new(plan), @@ -183,23 +306,20 @@ impl PoolPort { cancellation, reply, queue: Some(queue), + settlement, }) .map_err(|error| match error { mpsc::error::TrySendError::Full(_) => EnvironmentError::Busy, mpsc::error::TrySendError::Closed(_) => self.closed_error(), })?; - let offer = timeout_at(deadline, response) - .await - .map_err(|_| EnvironmentError::Deadline)? - .map_err(|_| self.closed_error())?; - if Instant::now() >= deadline { - return Err(EnvironmentError::Deadline); - } - if let Some(receipt) = offer.receipt { - let _ = receipt.send(()); - } - cancel.consumed = true; - offer.result + Ok(PoolInvocation { + deadline, + stop, + response: Some(response), + settlement: observed, + failed: self.failed.clone(), + consumed: false, + }) } fn closed_error(&self) -> EnvironmentError { if self.failed.load(Ordering::Acquire) { diff --git a/crates/rustack-lambda-core/src/docker/pool/tests.rs b/crates/rustack-lambda-core/src/docker/pool/tests.rs index 17f70dcb..de3dd324 100644 --- a/crates/rustack-lambda-core/src/docker/pool/tests.rs +++ b/crates/rustack-lambda-core/src/docker/pool/tests.rs @@ -7,6 +7,8 @@ use crate::docker::{ tests::Daemon, }; +mod settlement; + fn deadline() -> Instant { Instant::now() + Duration::from_secs(10) } diff --git a/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs b/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs new file mode 100644 index 00000000..e10c0c1d --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/pool/tests/settlement.rs @@ -0,0 +1,298 @@ +use std::{ + future::Future, + sync::atomic::AtomicUsize, + task::{Context, Waker}, +}; + +use super::*; +use crate::docker::environment::test_peer; + +#[rstest::rstest] +#[case::cancel(false, false)] +#[case::expired(true, false)] +#[case::root_abort(false, true)] +#[tokio::test] +async fn test_should_retain_per_call_settlement_across_expiry_waiter_drop_and_root_recovery( + #[case] expired: bool, + #[case] abort: bool, +) { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let plan = plan(&lease); + let (handle, peer) = test_peer::pair(); + let physical = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let serving = tokio::spawn(peer.serve_gated(count.clone(), gate, stopping)); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(plan.warm_key().into(), handle)], + ); + let until = if expired { + Instant::now() + Duration::from_millis(250) + } else { + deadline() + }; + let mut call = pool.port().start(plan, invocation(until), false).unwrap(); + timeout_at(deadline(), async { + while count.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + if abort { + pool.task.as_ref().unwrap().abort(); + } else if expired { + tokio::time::sleep_until(until).await; + assert!(matches!(call.wait().await, Err(EnvironmentError::Deadline))); + } + let mut waiting = Box::pin(call.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + timeout_at(deadline(), stopped).await.unwrap().unwrap(); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(!physical.removed()); + assert!(matches!( + call.cancel_and_wait(Instant::now() + Duration::from_millis(20)) + .await, + Err(EnvironmentError::Deadline) + )); + assert!(!physical.removed()); + release.send(()).unwrap(); + let result = call.cancel_and_wait(deadline()).await; + assert!(physical.removed()); + if abort { + assert!(matches!(result, Err(EnvironmentError::Worker))); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + } else { + result.unwrap(); + call.cancel_and_wait(deadline()).await.unwrap(); + pool.shutdown(deadline()).await.unwrap(); + } + serving.await.unwrap(); + assert_eq!(count.load(Ordering::Acquire), 1); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_resume_pending_response_wait_and_leave_consumed_success_warm() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let revision = Arc::new(revision()); + let make = || { + EnvironmentPlan::new( + lease.clone(), + image_with("1000", vec![]), + revision.clone(), + policy(), + ) + .unwrap() + }; + let (handle, peer) = test_peer::pair(); + let physical = handle.retirement(); + let count = Arc::new(AtomicUsize::new(0)); + let serving = tokio::spawn(peer.serve(count.clone())); + let mut pool = actor::testing_start( + environments.port(), + PoolConfig::default(), + vec![(make().warm_key().into(), handle)], + ); + let port = pool.port(); + for expected in 1..=2 { + let mut call = port.start(make(), invocation(deadline()), false).unwrap(); + let mut waiting = Box::pin(call.wait()); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + call.wait().await.unwrap(); + assert!(matches!( + call.wait().await, + Err(EnvironmentError::Configuration) + )); + call.cancel_and_wait(deadline()).await.unwrap(); + drop(call); + idle(&port).await; + assert!(!physical.removed()); + assert_eq!(count.load(Ordering::Acquire), expected); + } + pool.shutdown(deadline()).await.unwrap(); + serving.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_settle_cancelled_queued_call_without_waiting_for_an_unrelated_allocation() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = EnvironmentManager::start( + wire.port(), + lease.clone(), + crate::docker::EnvironmentSettings::builder() + .environments(1) + .build() + .try_into() + .unwrap(), + ) + .unwrap(); + let mut occupied = environments.port().spawn(plan(&lease), deadline()).unwrap(); + let held = daemon.next().await; + let mut pool = EnvironmentPool::start(environments.port(), PoolConfig::default()).unwrap(); + let mut call = pool + .port() + .start(plan(&lease), invocation(deadline()), false) + .unwrap(); + call.cancel_and_wait(deadline()).await.unwrap(); + assert_eq!(environments.port().status().allocated, 1); + assert!(daemon.requests.try_recv().is_err()); + pool.shutdown(deadline()).await.unwrap(); + held.reply(400, b"{}").await; + occupied.shutdown(deadline()).await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_retain_call_receipt_above_a_failed_dispatch_worker() { + let daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory(); + let mut environments = + EnvironmentManager::start(wire.port(), lease.clone(), EnvironmentConfig::default()) + .unwrap(); + let (environment, peer) = test_peer::pair(); + let physical = environment.retirement(); + let (release, gate) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let serving = tokio::spawn(peer.serve_gated(Arc::new(AtomicUsize::new(0)), gate, stopping)); + let (mut pool, mut call) = actor::testing_failed_call_start( + environments.port(), + plan(&lease), + invocation(deadline()), + environment, + ); + timeout_at(deadline(), stopped).await.unwrap().unwrap(); + assert!(matches!(call.wait().await, Err(EnvironmentError::Worker))); + let mut waiting = Box::pin(call.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(waiting); + assert!(!physical.removed()); + release.send(()).unwrap(); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(physical.removed()); + assert!(matches!( + pool.shutdown(deadline()).await, + Err(EnvironmentError::Worker) + )); + serving.await.unwrap(); + environments.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +async fn test_should_treat_a_closed_response_and_missing_settlement_as_uncertain() { + let (reply, response) = oneshot::channel(); + let (settlement, observed) = watch::channel(None); + drop(reply); + drop(settlement); + let mut call = PoolInvocation { + deadline: deadline(), + stop: watch::channel(false).0, + response: Some(response), + settlement: observed, + failed: Arc::new(AtomicBool::new(false)), + consumed: false, + }; + assert!(matches!(call.wait().await, Err(EnvironmentError::Stopped))); + assert!(matches!( + call.wait().await, + Err(EnvironmentError::Configuration) + )); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); + assert!(matches!( + call.cancel_and_wait(deadline()).await, + Err(EnvironmentError::Worker) + )); +} + +#[rstest::rstest] +#[case::expired(true)] +#[case::excessive(false)] +#[tokio::test] +async fn test_should_signal_cancellation_before_rejecting_invalid_cleanup_deadline( + #[case] expired: bool, +) { + let (reply, response) = oneshot::channel(); + let (_settled, observed) = watch::channel(Some(Ok(()))); + let mut call = PoolInvocation { + deadline: deadline(), + stop: watch::channel(false).0, + response: Some(response), + settlement: observed, + failed: Arc::new(AtomicBool::new(false)), + consumed: false, + }; + let until = if expired { + Instant::now() + } else { + Instant::now() + Duration::from_secs(601) + }; + let error = call.cancel_and_wait(until).await.unwrap_err(); + if expired { + assert!(matches!(error, EnvironmentError::Deadline)); + } else { + assert!(matches!(error, EnvironmentError::Configuration)); + } + assert!(*call.stop.borrow()); + assert!(reply.is_closed()); + call.cancel_and_wait(deadline()).await.unwrap(); +} diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs index 59ce4499..eecc18b0 100644 --- a/crates/rustack-lambda-core/tests/docker_environment.rs +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -370,10 +370,14 @@ async fn test_should_pool_real_warm_calls_and_evict_capacity_across_revision_cha result.and(retired.map_err(Into::into)).and(cleanup) } +#[rstest::rstest] +#[case::abandoned(false)] +#[case::awaited_settlement(true)] #[tokio::test] #[ignore = "requires explicit Docker environment images/socket"] -async fn test_should_join_pool_retirement_after_abandoning_an_observed_running_handler() --> Result<()> { +async fn test_should_join_pool_retirement_after_cancelling_an_observed_running_handler( + #[case] awaited: bool, +) -> Result<()> { let mut fixture = Fixture::new().await?; let mut pool = EnvironmentPool::start(fixture.owner.port(), PoolConfig::default())?; let result = async { @@ -388,14 +392,20 @@ async fn test_should_join_pool_retirement_after_abandoning_an_observed_running_h matches!(initial.result, RuntimeResult::Success(_)), "warm setup failed" ); - let plan = fixture.plan_revision(revision)?; - let invocation = invocation(&json!({"value":1,"sleep":true}))?; + let plan = fixture.plan_revision(revision.clone())?; + let event = invocation(&json!({"value":1,"sleep":true}))?; let mut observed = FunctionLogs::start( fixture.wire.port().attach_logs(&key, deadline()).await?, Duration::from_secs(60), 262_144, )?; - let call = tokio::spawn(async move { port.invoke(plan, invocation, false).await }); + let mut call = match port.start(plan, event, false) { + Ok(call) => call, + Err(error) => { + observed.shutdown().await?; + return Err(error.into()); + } + }; let observation = timeout(Duration::from_secs(30), async { loop { let snapshot = observed.snapshot(deadline()).await?; @@ -411,17 +421,51 @@ async fn test_should_join_pool_retirement_after_abandoning_an_observed_running_h }) .await .context("pooled handler did not start"); - call.abort(); - let joined = call.await; - let retired = pool.shutdown(deadline()).await; + let cancellation = if awaited { + call.cancel_and_wait(deadline()).await + } else { + drop(call); + pool.shutdown(deadline()).await + }; let logs = observed.shutdown().await; observation??; + cancellation?; + logs?; ensure!( - matches!(joined,Err(ref error) if error.is_cancelled()), - "pooled invocation task was not cancelled" + fixture.owner.port().status().allocated == 0, + "call returned before environment retirement" ); - retired?; - logs?; + ensure!( + matches!( + fixture + .wire + .port() + .inspect_container(&key, deadline()) + .await, + Err(rustack_lambda_core::docker::DockerError::Status( + http::StatusCode::NOT_FOUND + )) + ), + "cancelled function still exists" + ); + if awaited { + ensure!( + port.status().accepting, + "per-call cancellation stopped the pool" + ); + let replacement = port + .invoke( + fixture.plan_revision(revision)?, + invocation(&json!({"value":2}))?, + false, + ) + .await?; + let RuntimeResult::Success(payload) = replacement.result else { + anyhow::bail!("replacement handler failed"); + }; + let output: Value = serde_json::from_slice(payload.as_ref())?; + ensure!(output["counter"] == 1, "cancelled environment was reused"); + } Ok(()) } .await; diff --git a/specs/index.md b/specs/index.md index 2c831045..49fed672 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,7 +4,7 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. Production Docker selection remains unavailable until Executor/provider response wiring, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 95879a9a..242b57e8 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -634,6 +634,32 @@ Caller Pool actor / retained registry Environment owner Tests must pin identity isolation/order normalization, queue+active bounds, FIFO capacity waiting, LRU and per-identity retention, dead idle handles, cancellation before/after dispatch and at ready-result receipt, no replay after startup failure, root/worker abort recovery and cancelled/repeated shutdown. Actual image-RIC qualification must prove warm counter reuse, independent-function/revision cold counters, bounded capacity eviction and no owned allocations after joined shutdown. This pool does not establish startup reconciliation, registry/network/resource probes or production backend readiness. +### 4.5 Supervised pool invocation receipt + +The Executor consumer needs a cancellation settlement port distinct from a response future. `PoolPort::start(plan, invocation, capture_logs)` synchronously admits once and returns an exclusive `PoolInvocation`; immediate overload/readiness errors and actor-side scope rejections allocate nothing. Existing `PoolPort::invoke` delegates to start plus wait and retains its response behavior. No method replays an invocation. + +`PoolInvocation::wait(&mut self)` is cancel-safe while pending. It preserves the response receiver across a dropped waiter, validates the absolute invocation deadline before synchronously acknowledging a successful result, and disarms cancellation only after consuming a valid offer. A ready-but-unconsumed result still belongs to the retained dispatch worker. A consumed response cannot be consumed twice. Success already proves the environment reusable; FunctionError and ordinary failure offers are published only after physical retirement or a no-allocation rejection. Closed response channels alone prove neither result nor cleanup. + +`cancel_and_wait(&mut self, cleanup_deadline)` requests cancellation and closes the unconsumed response lane before validating the independent cleanup deadline. It then awaits a cached per-call settlement receipt. An expired invocation deadline cannot shorten this cleanup budget. A timed-out or dropped cleanup waiter retains its receiver and may resume; Drop signals cancellation without claiming cleanup. After a consumed successful response, this operation only waits for bookkeeping settlement and cannot cancel an environment returned to another call. + +Keep each dispatched call's settlement sender alongside its environment Retirement record **above** the pool JoinSet. Normal completion publishes clean only after reusable/retired proof; a failed worker/root carries Worker uncertainty through retained recovery and publishes only after owned physical retirement. Never derive clean from sender Drop, response-channel closure, aborted workers or a cleanup timeout. Queued calls rejected before allocation publish clean settlement. Maintenance retirement has no call receipt. The Executor must retain the Lambda invocation work future/permits through this independent settlement, returning CleanupFailed for unresolved or uncertain ownership. Startup readiness, production enablement and Executor response mapping remain separate mandatory consumers. + +```text +Lambda work owner PoolInvocation Pool actor registry Environment owner + │ │ [Retirement + receipt] │ + │ start ──────────────────▶│ ── bounded admission ─────▶│ │ + │ │ │ allocate/checkout ──────▶│ + │ wait (cancel-safe) ──────▶│◀──── result offer ────────│◀── actual result ───────│ + │ cancellation │ │ │ + │ cancel_and_wait ────────▶│ ── stop/drop offer ──────▶│ ── retire ─────────────▶│ + │ keeps permits │ │ waits physical + IO ◀───│ + │ │◀── cached settlement ─────│ │ + │ release only on proof ◀──│ │ │ + └─ waiter/root loss: registry transfers receipt + retirement into retained recovery ─┘ +``` + +Tests must hold retirement behind a channel barrier and prove cancellation does not return early, including invocation expiry and dropped cleanup waiters. Cover queued no-allocation rejection, consumed-success bookkeeping without retiring warm reuse, root/worker abort recovery and closed-receipt uncertainty. These ownership tests do not establish actual Docker/provider acceptance. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 44330144..9dda3987 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -104,7 +104,9 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi The B05 regression composes the actual session/HTTP/relay/client with an environment worker after BootstrapReady and an abandoned poll. For an active call, observe an offered but unconsumed event with a recording waker, abandon that offer, then POST InitError. Test Idle and Busy at32KiB and32KiB+1, require the bounded initialization class versus Protocol, redacted diagnostics and joined control/log/API retirement. Scripted container absence remains distinct from real Docker code execution. -`make test-lambda-log-windows` verifies diagnostics remain environment-wide while each invocation window starts empty: binary order/overflow, silent next call, partial records across Begin/Finish, matching EOF finish, capture failure, stale/cross-owner capabilities and direct capability Drop. Mutation receipts require direct tests where a ready reply is abandoned or remains live/unpolled through deadline: the quiet socket must close before any later diagnostic request, and the local handle must forbid another mutation. Also cover on-time receipts processed after deadline, no-effect expired/abandoned commands, priority-stop rejection and flooded-output fairness. `make test-lambda-log-windows-docker` runs the six-case owner/pool suite: the window case alternates Tail/logging, None/logging, Tail/silent and Tail/logging through actual warm counters1..4. Require computed payloads, expected observed stdout/stderr markers and empty silent/None tails; never use the diagnostic tail as the response. The independent-stream timing limitation in runtime §3.9 remains explicit. +The pool settlement consumer tests exercise `start`/cancel-safe `wait`/independent `cancel_and_wait`. Hold retirement behind an explicit barrier; require pending settlement after invocation expiry, after a dropped cleanup waiter, and during root-abort recovery. Queued rejection must settle without allocation, consumed success must preserve warm reuse, and closed or supervision-failed receipts must never report clean. + +`make test-lambda-log-windows` verifies diagnostics remain environment-wide while each invocation window starts empty: binary order/overflow, silent next call, partial records across Begin/Finish, matching EOF finish, capture failure, stale/cross-owner capabilities and direct capability Drop. Mutation receipts require direct tests where a ready reply is abandoned or remains live/unpolled through deadline: the quiet socket must close before any later diagnostic request, and the local handle must forbid another mutation. Also cover on-time receipts processed after deadline, no-effect expired/abandoned commands, priority-stop rejection and flooded-output fairness. `make test-lambda-log-windows-docker` runs the seven-case owner/pool suite: the window case alternates Tail/logging, None/logging, Tail/silent and Tail/logging through actual warm counters1..4. Require computed payloads, expected observed stdout/stderr markers and empty silent/None tails; never use the diagnostic tail as the response. The independent-stream timing limitation in runtime §3.9 remains explicit. `make test-lambda-timeouts` runs the complete shared runtime library suite and real Native hardening fixtures. Require delayed/abandoned offers to retain the original caller deadline until consumed, no receipt/body after outer expiry, an effective Runtime API deadline matching the consumed handler budget, earlier-handler versus outer/tied classification, late receipt/completion rejection, on-time completion with a delayed ticket waiter, and strict v2 timeout/cancel correlation. A manual-poll actual actor.run regression must make the receipt and old outer timer ready between polls and require HandlerTimeout. Preserve independent Native initialization timeout classification plus process reap and separate delivered-handler timeout/quiescence. `make test-lambda-timeouts-docker` includes actual image-pool timeout retirement: a900s caller-supplied handler budget cannot override captured3s; the error is Timeout3s, allocation accounting reaches zero before that result, and replacement code starts at counter1. Rebuild/pin the version2 relay image with the host; version1 must fail protocol validation. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 3e789487..481df094 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows are implemented. Missing: real Docker Executor/provider response composition; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Missing: real Docker Executor/provider response composition; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -151,6 +151,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 delivery clock | Implemented/reviewed; executor envelope mapping remains | §3.8/D33 implement validated relative handler budgets, consumed monotonic receipts/effective HTTP headers, strict earlier-handler versus outer/tie classification, protocol-v2 compatibility/timeout correlation, immutable captured override and joined environment retirement; Native attaches the shared clock and preserves independent initialization expiry. W82–W84 fixes and final independent recheck pass. Runtime119 unit tests/51 doctests; core553 unit tests/132 doctests. Workspace2,255 tests/406 gated; strict boundary839 unit tests/199 doctests (1,038 total); standard build/test/fmt/Clippy/rustdoc, strict production, audit and deny pass. Timeout target131 passes includes12 real Native fixtures. On each arm64 VZ and amd64/QEMU daemon, owner/pool5 + wire2 + shared runtime5 cases explicitly pass (12 per daemon); captured3s defeats caller-supplied900s, confirmed retirement releases capacity, replacement counter1 and no owned resources afterward. Six actual Native/DryRun provider scenarios pass separately; the gated Squib case is not VM evidence. Per-invocation log attribution and final Executor/provider/startup composition remain required before M0; P2–P5 remain pending. | | 1.3 invocation logs | Implemented/reviewed; Executor/provider response composition remains | §3.9/D34 separate bounded per-attempt windows from diagnostic rings, with opaque revocable capabilities, actor-retained consumption receipts/deadlines, exact ownership, pinned partial reads and Tail/None integration. W85 self-review correction and final independent recheck pass.36 log cases (18 added) pass; core571 tests/134 doctests; workspace2,273 tests/407 gated; strict boundary857 unit tests/201 doctests (1,058 total). Standard build/test/fmt/Clippy/rustdoc, strict production, audit/deny pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes owner/pool6 + wire2 + shared runtime5 cases (13 per daemon), including real requested/uncaptured/silent log windows with warm counters1..4. Post-run owned resource queries are empty. Observed-stream timing limits remain explicit; B05 and final Executor/provider/base64 response wiring remain mandatory before M0. | | 1.3 post-bootstrap initialization | Implemented/reviewed; full M0 remains open | Shared bounded InitError classification preserves initialization failure after an abandoned BootstrapReady poll, including an offered-but-unconsumed first event. Four actual session/HTTP/relay/client/worker composition regressions prove Idle/Busy32KiB boundaries, no active reply before a scripted known-container absence barrier, redaction and joined IO/API retirement. Independent read-only review has no findings. Environment target25 cases, workspace2,277 tests/407 separately gated, strict boundary1,062 unit+doc passes/13 Docker-gated; build/test/fmt/Clippy/rustdoc and strict production checks pass. Evidence: `/tmp/rustack-workflows-b05-{environments,build,tests,fmt,clippy,docs,boundaries}.log`. B05 closed; this is not production Executor/startup or actual Docker acceptance. | +| 1.3 supervised pool invocation | Implemented/reviewed; Executor consumer remains required | §4.5/D35 expose synchronous start, cancel-safe response wait and separate resumable cancel_and_wait with an independent cleanup deadline. Per-call settlement senders stay beside Retirement records above JoinSet through worker/root recovery. Nine new regressions,27 total pool cases, prove delayed/expired cleanup, waiter loss, no-allocation rejection, worker/root failure, warm success and missing-producer uncertainty. W86 observer rejection cleanup is fixed; final independent review has no remaining finding. Workspace2,286 tests/408 separately gated; strict boundary870 units+204 docs (1,074 total)/14 Docker-gated; standard build/test/fmt/Clippy/rustdoc and strict production checks pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes7 owner/pool real-image cases, including per-call awaited retirement before global shutdown, container404 and replacement counter1 while admission stays open. Post-run owned containers/internal aliases are absent. This does not close Executor/provider/startup or M0 acceptance. | | 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. Next compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. Next compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 1cba133d..34606dce 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -395,6 +395,22 @@ Each retained Engine29.5.2 daemon explicitly passes owner/pool6, wire2 and share Window attribution follows observed completed chunks, with the independent-stream timing limit advertised in runtime/operations specs. Production Executor/provider base64 header wiring, historical reconciliation, allowed pulls/network/probes and M0 SDK acceptance remain required; P2–P5 remain pending. +## P1 task1.3 supervised pool invocation review + +Independent read-only review covers runtime §4.5/D35, PoolPort::start and PoolInvocation response/settlement ownership, actor registry records above JoinSet, worker/root recovery, queued no-allocation rejection, independent cleanup deadlines and the real running-handler cancellation test. The final source review has no remaining in-slice finding. One test-fixture lifecycle finding was corrected before final verification. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W86 | P3; `crates/rustack-lambda-core/tests/docker_environment.rs`, log observer followed by synchronous pool admission | A start rejection after observer attachment could skip the explicit observer join. Match admission failure and await observed.shutdown before returning the original error | Final review verifies both admission failure and normal observation/cancellation branches join the observer; no detached observer is accepted as cleanup proof | + +The pool target passes27 cases (nine additions): gated cancellation and invocation expiry, cancelled/timed-out cleanup waiters, retained worker/root-abort receipts, queued no-allocation rejection, resumable pending response wait and consumed-success warm reuse, missing settlement producer, and cancellation before invalid cleanup deadline rejection. Closed response channels cannot grant cleanup proof. No dependency/version or advisory exception is introduced. + +Final standard workspace build/test/fmt/Clippy/rustdoc pass with2,286 tests, zero failures and408 separately resource-gated cases. The six-library strict boundary target passes870 unit tests and204 doctests (1,074 total),14 separately gated Docker cases and all strict production checks. Evidence is retained in `/tmp/rustack-workflows-settlement-verified-{build,tests,fmt,clippy,docs,boundaries}.log`. + +The final owner/pool suite explicitly passes7 actual-image cases with zero ignored on each retained Engine29.5.2 daemon: arm64 VZ and amd64/QEMU. The new awaited cancellation case proves owner allocation0 and container404 before pool shutdown, keeps pool admission open and obtains replacement handler counter1. The separate abandoned-call path and prior warm/revision/error/timeout/log cases remain green. Post-run scoped queries find no owner/test-labeled containers or internal image aliases. QEMU is not native-x86 platform acceptance. Logs: `/tmp/rustack-workflows-settlement-verified-{arm64,amd64}.log`. + +Production Executor/provider composition must consume this cancellation settlement before releasing its work permits. Startup reconciliation, allowed pulls, network/resource probes, M0 SDK acceptance and P2–P5 remain required. The receipt alone does not enable Docker or complete M0. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 8ce2b8b3..20d50b15 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -308,3 +308,13 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: current-call observed output has bounded explicit ownership, silent calls cannot inherit the cached tail, and uncertain mutations forbid reuse. The advertised stream timing limit preserves honest compatibility. - **Pinned by**: [runtime §3.9](./rustack-workflows-20-lambda-runtime-design.md#39-invocation-scoped-log-windows), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. + + +## D35 — Keep per-call pool cleanup settlement above dispatch workers + +- **Context**: the Docker Executor must retain provider invocation permits through cancellation cleanup, including response-channel closure and pool worker/root failure. +- **Alternatives considered**: dropping an invoke future only requests cancellation; using a response-channel close as cleanup proof loses physical ownership; worker-local settlement senders disappear during panic; reusing an invocation deadline can skip cleanup once that deadline expires. +- **Decision**: expose an exclusive PoolInvocation with cancel-safe response waiting and a separate resumable cancellation-settlement operation; retain the sender beside the environment Retirement record above JoinSet and through emergency recovery. +- **Why**: upper-layer admission can await a positive reusable/retired proof without owning or aborting lower-layer resources. Failed supervision remains uncertain even after physical recovery; queued no-allocation rejection can settle cleanly. +- **Pinned by**: [runtime §4.5](./rustack-workflows-20-lambda-runtime-design.md#45-supervised-pool-invocation-receipt), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. This receipt does not enable production Docker or complete Executor/provider integration. From e5c4dc365b5d16365a122b98cfdd12cfb7f887ba Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 03:53:39 -0700 Subject: [PATCH 029/106] phase 1: compose real Docker Lambda execution through provider and HTTP Implement DockerExecutor over retained pool invocation settlement with trusted startup readiness injection, raw outcomes, immutable metadata, resolved versions and bounded log tails. Propagate validated HTTP LogType through provider admission; reject duplicate or malformed scalar Invoke headers. Preserve quota versus post-effect infrastructure Busy classes and resumable shutdown ownership. Ordinary pinned boto3 executes actual image deploy, publish, alias, Invoke, FunctionError, captured timeout and replacement on both retained daemons. Final standard and strict boundary gates, audit and deny pass; independent review findings W87-W91 are closed. Production CLI startup qualification and complete P1-P5 acceptance remain required. --- Makefile | 10 + .../src/docker/environment/mod.rs | 4 + .../src/docker/environment/test_peer.rs | 16 + .../src/docker/executor.rs | 384 ++++++++++++++++ .../src/docker/executor/tests.rs | 410 ++++++++++++++++++ crates/rustack-lambda-core/src/docker/mod.rs | 5 + .../src/docker/pool/actor.rs | 2 +- .../src/docker/pool/mod.rs | 3 +- .../rustack-lambda-core/src/executor/error.rs | 14 + .../rustack-lambda-core/src/executor/mod.rs | 8 +- .../rustack-lambda-core/src/executor/types.rs | 3 +- crates/rustack-lambda-core/src/handler.rs | 93 +++- crates/rustack-lambda-core/src/provider.rs | 42 +- crates/rustack-lambda-core/src/work.rs | 2 +- .../tests/common/docker.rs | 131 ++++++ .../rustack-lambda-core/tests/common/mod.rs | 1 + .../tests/docker_environment.rs | 136 +----- .../tests/docker_executor.rs | 182 ++++++++ specs/index.md | 4 +- specs/rustack-workflows-00-prd.md | 2 +- ...tack-workflows-20-lambda-runtime-design.md | 35 ++ .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 7 +- specs/rustack-workflows-93-review.md | 22 + specs/rustack-workflows-99-decisions.md | 10 + tests/workflows/docker_lambda_sdk.py | 116 +++++ tools/workflows-spikes/handler.py | 1 + 27 files changed, 1502 insertions(+), 143 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/executor.rs create mode 100644 crates/rustack-lambda-core/src/docker/executor/tests.rs create mode 100644 crates/rustack-lambda-core/tests/common/docker.rs create mode 100644 crates/rustack-lambda-core/tests/common/mod.rs create mode 100644 crates/rustack-lambda-core/tests/docker_executor.rs create mode 100644 tests/workflows/docker_lambda_sdk.py diff --git a/Makefile b/Makefile index 0ab8a2a2..a5afd920 100644 --- a/Makefile +++ b/Makefile @@ -899,6 +899,16 @@ test-lambda-pool-docker: test-lambda-environments-docker .PHONY: test-lambda-pool test-lambda-pool-docker +# Actual Docker Executor/provider/HTTP composition via pinned ordinary boto3. +test-lambda-executor: + @cargo test -p rustack-lambda-core --lib docker::executor::tests + @cargo test -p rustack-lambda-core --lib handler::tests + +test-lambda-executor-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --test docker_executor -- --ignored --test-threads=1 + +.PHONY: test-lambda-executor test-lambda-executor-docker + # Delivery clocks, protocol-v2 correlation and actual Native process retirement. test-lambda-timeouts: @cargo test -p rustack-lambda-runtime --lib diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index 9f7551a6..3658c603 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -404,6 +404,10 @@ impl fmt::Debug for EnvironmentPort { } } impl EnvironmentPort { + pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { + self.directory.owner_id() == directory.owner_id() + && self.directory.run_id() == directory.run_id() + } /// Reserve count/memory and register a plan before any container operation. /// /// # Errors diff --git a/crates/rustack-lambda-core/src/docker/environment/test_peer.rs b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs index e294870b..b88a9254 100644 --- a/crates/rustack-lambda-core/src/docker/environment/test_peer.rs +++ b/crates/rustack-lambda-core/src/docker/environment/test_peer.rs @@ -33,6 +33,22 @@ pub(in crate::docker) fn pair() -> (EnvironmentHandle, Peer) { ) } impl Peer { + pub(in crate::docker) async fn serve_pending( + mut self, + entered: oneshot::Sender, + release: oneshot::Receiver<()>, + stopping: oneshot::Sender<()>, + ) { + let call = self.calls.recv().await.unwrap(); + entered.send(call.invocation.clone()).unwrap(); + let mut stop = self.signals.stop.subscribe(); + let _ = stop.wait_for(|stop| *stop).await; + let _ = stopping.send(()); + let _ = release.await; + self.signals.phase(EnvironmentPhase::Removed); + self.done.send_replace(Some(Ok(()))); + drop(call); + } pub(in crate::docker) async fn serve(self, count: Arc) { self.serve_inner(count, None, None).await; } diff --git a/crates/rustack-lambda-core/src/docker/executor.rs b/crates/rustack-lambda-core/src/docker/executor.rs new file mode 100644 index 00000000..2ac726d5 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/executor.rs @@ -0,0 +1,384 @@ +//! Trusted runtime composition; startup qualification remains coordinator-owned. +//! +//! The synchronous readiness trait is object-safe for trusted in-process startup +//! injection. Executor uses async-trait for the existing dynamic backend boundary. + +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; +use parking_lot::Mutex; +use rustack_lambda_runtime::{ + metadata::{MetadataError, local_trace_id}, + protocol::{FunctionArn, Payload, ProtocolError}, + session::{Invocation, RuntimeResult, SessionError}, +}; +use thiserror::Error; +use tokio::{ + runtime::Handle, + time::{Instant, timeout_at}, +}; +use typed_builder::TypedBuilder; + +use super::{ + ContainerPolicy, DirectoryLease, DockerError, EnvironmentError, EnvironmentPlan, + EnvironmentPool, EnvironmentPort, EnvironmentResponse, PoolConfig, PoolPort, +}; +use crate::{ + artifact::PinnedImage, + executor::{Executor, ExecutorError, InvokeRequest, InvokeResponse}, + invocation::{CleanupTask, InvocationContext}, +}; + +/// Redacted typed failures at the Docker Executor composition boundary. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum DockerExecutorError { + /// Invalid trusted settings, runtime scope or captured invocation metadata. + #[error("invalid Docker executor configuration")] + Configuration, + /// Startup qualification was not granted or admission has been revoked. + #[error("Docker execution is not ready")] + Unavailable, + /// Invalid bounded invocation payload, ARN or attempt identity. + #[error("invalid Docker invocation metadata")] + Request(#[source] Arc), + /// Wall-clock tracing metadata could not be generated. + #[error("Docker invocation clock failed")] + Clock(#[from] MetadataError), + /// Runtime session rejected construction before pool admission. + #[error("Docker invocation session failed")] + Session(#[from] SessionError), + /// Retained environment preserves initialization, exit/OOM and wire classes. + #[error("Docker environment failed: {0}")] + Environment(#[from] EnvironmentError), +} +impl fmt::Debug for DockerExecutorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl From for DockerExecutorError { + fn from(error: ProtocolError) -> Self { + Self::Request(Arc::new(error)) + } +} + +/// Trusted startup evidence port for the exact runtime inputs injected below. +/// +/// Implementations must make bounded, nonblocking checks without IO. The startup +/// coordinator owns actual reconciliation and probe evidence; this trait alone +/// cannot establish it. Revocation rejects new calls; shutdown retires active work. +pub trait DockerReadiness: fmt::Debug + Send + Sync + 'static { + /// Check coordinator-owned qualification for this runtime. + /// + /// # Errors + /// Returns Unavailable or a typed qualification failure after revocation. + /// + /// ```no_run + /// # fn ready(evidence: &dyn rustack_lambda_core::executor::DockerReadiness) { + /// let qualified = evidence.check().is_ok(); + /// # } + /// ``` + fn check(&self) -> Result<(), DockerExecutorError>; +} + +/// Validated pool and independent public cleanup-wait budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct DockerExecutorSettings { + pool: PoolConfig, + cleanup: Duration, +} +impl Default for DockerExecutorSettings { + fn default() -> Self { + Self { + pool: PoolConfig::default(), + cleanup: Duration::from_secs(10), + } + } +} +impl DockerExecutorSettings { + /// Validate a whole-second cleanup budget in1..60s. + /// + /// # Errors + /// Rejects zero, fractional or excessive cleanup duration. + /// + /// ``` + /// let settings = rustack_lambda_core::executor::DockerExecutorSettings::new( + /// Default::default(), std::time::Duration::from_secs(10))?; + /// # Ok::<(), rustack_lambda_core::executor::DockerExecutorError>(()) + /// ``` + pub fn new(pool: PoolConfig, cleanup: Duration) -> Result { + if cleanup.subsec_nanos() != 0 || !(1..=60).contains(&cleanup.as_secs()) { + return Err(DockerExecutorError::Configuration); + } + Ok(Self { pool, cleanup }) + } +} + +/// Trusted startup inputs; readiness must qualify this exact scope/relay/policy. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct DockerExecutorConfig { + /// Persistent namespace capability retained through cleanup. + pub directory: DirectoryLease, + /// Immutable live startup relay image. + pub relay: PinnedImage, + /// Validated operator allocation policy. + pub policy: ContainerPolicy, + /// Retained environment service for the same directory owner and run. + pub environments: EnvironmentPort, + /// Trusted startup evidence, never supplied by an AWS request. + pub readiness: Arc, + /// Validated dispatch/cleanup budgets. + #[builder(default)] + pub settings: DockerExecutorSettings, +} + +#[derive(Debug)] +struct PoolOwner { + root: Mutex>, + port: PoolPort, + cleanup: CleanupTask, + runtime: Handle, + stopping: AtomicBool, +} + +/// Actual image Lambda Executor over retained pool/environment ownership. +#[derive(Debug)] +#[non_exhaustive] +pub struct DockerExecutor { + config: DockerExecutorConfig, + pool: PoolOwner, +} +impl DockerExecutor { + /// Construct dispatch over matching live runtime inputs without daemon IO. + /// + /// This does not qualify production startup. Retain the environment/image + /// managers until shutdown has joined this executor's pool. + /// + /// # Errors + /// Rejects scope mismatch, revoked relay or missing Tokio supervision. + /// + /// ```no_run + /// # async fn compose(config: rustack_lambda_core::executor::DockerExecutorConfig) -> Result<(), rustack_lambda_core::executor::ExecutorError> { + /// use rustack_lambda_core::executor::{DockerExecutor, Executor}; + /// let executor = DockerExecutor::new(config)?; + /// executor.shutdown().await?; + /// # Ok(()) } + /// ``` + pub fn new(config: DockerExecutorConfig) -> Result { + if !config.environments.same_scope(&config.directory) || !config.relay.is_live() { + return Err(DockerExecutorError::Configuration); + } + let runtime = Handle::try_current().map_err(|_| DockerExecutorError::Unavailable)?; + let root = EnvironmentPool::start(config.environments.clone(), config.settings.pool)?; + let port = root.port(); + Ok(Self::with_pool(config, root, port, runtime)) + } + fn with_pool( + config: DockerExecutorConfig, + root: EnvironmentPool, + port: PoolPort, + runtime: Handle, + ) -> Self { + Self { + config, + pool: PoolOwner { + root: Mutex::new(Some(root)), + port, + cleanup: CleanupTask::default(), + runtime, + stopping: AtomicBool::new(false), + }, + } + } + fn plan(&self, request: &InvokeRequest) -> Result { + let plan = EnvironmentPlan::new( + self.config.directory.clone(), + self.config.relay.clone(), + request.revision.clone(), + self.config.policy.clone(), + ) + .map_err(EnvironmentError::from)?; + let _: FunctionArn = request.function_arn.parse()?; + let (_, function) = request + .function_arn + .split_once(":function:") + .ok_or(DockerExecutorError::Configuration)?; + let (name, qualifier) = function + .split_once(':') + .map_or((function, None), |(name, q)| (name, Some(q))); + let version = request.revision.qualifier.as_str(); + let matches = match qualifier { + None | Some("$LATEST") => version == "$LATEST", + Some(value) if value.bytes().all(|b| b.is_ascii_digit()) => value == version, + Some(_) => version != "$LATEST", + }; + let region = request.function_arn.split(':').nth(3); + if name != request.revision.function_name + || !matches + || region + != request + .revision + .environment + .get("AWS_REGION") + .map(String::as_str) + { + return Err(DockerExecutorError::Configuration); + } + Payload::try_from(request.payload.clone())?; + Ok(plan) + } + fn start_cleanup(&self) { + self.pool.stopping.store(true, Ordering::Release); + self.pool.cleanup.start_on(&self.pool.runtime, || { + let root = self.pool.root.lock().take(); + async move { + let Some(mut root) = root else { + return Err(cleanup_error()); + }; + loop { + match root + .shutdown(Instant::now() + Duration::from_secs(60)) + .await + { + Ok(()) => return Ok(()), + Err( + EnvironmentError::Deadline + | EnvironmentError::Docker(DockerError::Deadline), + ) => {} + Err(_) => return Err(cleanup_error()), + } + } + } + }); + } +} +impl Drop for DockerExecutor { + fn drop(&mut self) { + self.start_cleanup(); + } +} + +#[async_trait] +impl Executor for DockerExecutor { + fn image_available(&self) -> Result<(), ExecutorError> { + self.config.readiness.check()?; + let environment = self.config.environments.status(); + let pool = self.pool.port.status(); + if environment.worker_failed || pool.worker_failed { + return Err(DockerExecutorError::Environment(EnvironmentError::Worker).into()); + } + if self.pool.stopping.load(Ordering::Acquire) + || !self.config.relay.is_live() + || !environment.accepting + || !pool.accepting + { + return Err(DockerExecutorError::Unavailable.into()); + } + Ok(()) + } + fn available(&self, request: &InvokeRequest) -> Result<(), ExecutorError> { + self.image_available()?; + self.plan(request)?; + Ok(()) + } + async fn invoke( + &self, + request: InvokeRequest, + context: InvocationContext, + ) -> Result { + self.image_available()?; + context.check()?; + let plan = self.plan(&request)?; + let invocation = Invocation::new( + context + .id() + .to_string() + .parse() + .map_err(DockerExecutorError::from)?, + context.deadline(), + request + .function_arn + .parse() + .map_err(DockerExecutorError::from)?, + local_trace_id().map_err(DockerExecutorError::from)?, + Payload::try_from(request.payload.clone()).map_err(DockerExecutorError::from)?, + ) + .map_err(|error| match error { + SessionError::Deadline => ExecutorError::DeadlineExceeded, + error => DockerExecutorError::Session(error).into(), + })?; + context.check()?; + let mut call = self + .pool + .port + .start(plan, invocation, request.capture_logs) + .map_err(environment_error)?; + let result = tokio::select! { biased; + reason = context.cancelled() => Err(reason.error()), + result = call.wait() => result.map_err(environment_error), + }; + call.cancel_and_wait(Instant::now() + self.config.settings.cleanup) + .await + .map_err(|_| cleanup_error())?; + context.check()?; + response(result?, &request) + } + async fn shutdown(&self) -> Result<(), ExecutorError> { + self.start_cleanup(); + timeout_at( + Instant::now() + self.config.settings.cleanup, + self.pool.cleanup.wait(), + ) + .await + .map_err(|_| cleanup_error())? + } +} + +fn cleanup_error() -> ExecutorError { + ExecutorError::CleanupFailed("Docker resource settlement is not confirmed".into()) +} +fn environment_error(error: EnvironmentError) -> ExecutorError { + match error { + EnvironmentError::Cancelled => ExecutorError::Cancelled, + EnvironmentError::Deadline => ExecutorError::DeadlineExceeded, + EnvironmentError::Timeout(timeout) => ExecutorError::Timeout(timeout), + error => DockerExecutorError::Environment(error).into(), + } +} +fn response( + result: EnvironmentResponse, + request: &InvokeRequest, +) -> Result { + if result.logs.len() > 4096 { + return Err(DockerExecutorError::Environment(DockerError::Protocol.into()).into()); + } + let (payload, function_error) = match result.result { + RuntimeResult::Success(payload) => (payload, None), + RuntimeResult::FunctionError(payload) => (payload, Some("Unhandled".into())), + RuntimeResult::InitError(payload) if payload.as_ref().len() <= 32768 => { + return Err(environment_error(EnvironmentError::Initialization(payload))); + } + _ => return Err(environment_error(DockerError::Protocol.into())), + }; + Ok(InvokeResponse { + status: 200, + payload: payload.into(), + function_error, + log_tail: request.capture_logs.then(|| STANDARD.encode(result.logs)), + executed_version: request.revision.qualifier.clone(), + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/executor/tests.rs b/crates/rustack-lambda-core/src/docker/executor/tests.rs new file mode 100644 index 00000000..57a5c91a --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/executor/tests.rs @@ -0,0 +1,410 @@ +use std::{ + error::Error as _, + future::Future, + task::{Context, Waker}, +}; + +use bytes::Bytes; +use tokio::sync::{oneshot, watch}; +use uuid::Uuid; + +use super::*; +use crate::{ + docker::{ + DockerWire, EnvironmentConfig, EnvironmentManager, + environment::test_peer, + plan::tests::{image_with, policy, revision}, + tests::Daemon, + }, + error::LambdaServiceError, + work::WorkManager, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +#[derive(Debug)] +struct Readiness(AtomicBool); +impl DockerReadiness for Readiness { + fn check(&self) -> Result<(), DockerExecutorError> { + if self.0.load(Ordering::Acquire) { + Ok(()) + } else { + Err(DockerExecutorError::Unavailable) + } + } +} +struct Fixture { + daemon: Daemon, + wire: DockerWire, + environments: EnvironmentManager, + directory: DirectoryLease, + ready: Arc, +} +impl Fixture { + async fn new() -> Self { + let daemon = Daemon::new(); + let wire = daemon.wire().await; + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let environments = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + Self { + daemon, + wire, + environments, + directory, + ready: Arc::new(Readiness(AtomicBool::new(true))), + } + } + fn config(&self) -> DockerExecutorConfig { + DockerExecutorConfig::builder() + .directory(self.directory.clone()) + .relay(image_with("1000", vec![])) + .policy(policy()) + .environments(self.environments.port()) + .readiness(self.ready.clone()) + .settings( + DockerExecutorSettings::new(PoolConfig::default(), Duration::from_secs(1)).unwrap(), + ) + .build() + } + async fn close(mut self) { + self.environments.shutdown(deadline()).await.unwrap(); + self.wire.shutdown().await.unwrap(); + self.daemon.close().await; + } +} +fn request() -> InvokeRequest { + InvokeRequest::new( + Arc::new(revision()), + "arn:aws:lambda:us-east-1:000000000000:function:policy-fixture".into(), + Bytes::from_static(b"\x00\xffraw-event"), + ) +} + +#[test] +fn test_should_bound_executor_cleanup_settings() { + for invalid in [ + Duration::ZERO, + Duration::from_millis(1500), + Duration::from_secs(61), + ] { + assert!(matches!( + DockerExecutorSettings::new(PoolConfig::default(), invalid), + Err(DockerExecutorError::Configuration) + )); + } +} + +#[rstest::rstest] +#[case::name("arn:aws:lambda:us-east-1:000000000000:function:other")] +#[case::region("arn:aws:lambda:us-west-2:000000000000:function:policy-fixture")] +#[case::alias_latest("arn:aws:lambda:us-east-1:000000000000:function:policy-fixture:prod")] +#[case::version_latest("arn:aws:lambda:us-east-1:000000000000:function:policy-fixture:2")] +#[case::malformed("private-invalid-metadata")] +#[tokio::test] +async fn test_should_reject_invalid_request_metadata_before_daemon_allocation(#[case] arn: &str) { + let fixture = Fixture::new().await; + let executor = DockerExecutor::new(fixture.config()).unwrap(); + let mut request = request(); + request.function_arn = arn.into(); + assert!(executor.available(&request).is_err()); + assert_eq!(fixture.environments.port().status().allocated, 0); + executor.shutdown().await.unwrap(); + fixture.close().await; +} + +#[tokio::test] +async fn test_should_validate_scope_readiness_and_payload_without_allocating() { + let fixture = Fixture::new().await; + let mut wrong = fixture.config(); + wrong.directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + assert!(matches!( + DockerExecutor::new(wrong), + Err(DockerExecutorError::Configuration) + )); + let mut revoked = fixture.config(); + revoked.relay = PinnedImage::new( + revoked.relay.resolved().clone(), + revoked.relay.unpacked_size(), + watch::channel(()).0, + Arc::new(AtomicBool::new(false)), + watch::channel(false).1, + ); + assert!(matches!( + DockerExecutor::new(revoked), + Err(DockerExecutorError::Configuration) + )); + let executor = DockerExecutor::new(fixture.config()).unwrap(); + executor.available(&request()).unwrap(); + fixture.ready.0.store(false, Ordering::Release); + assert!(matches!( + executor.image_available(), + Err(ExecutorError::Docker(DockerExecutorError::Unavailable)) + )); + fixture.ready.0.store(true, Ordering::Release); + let mut invalid = request(); + invalid.payload = Bytes::from(vec![0; 6 * 1024 * 1024 + 1]); + assert!(executor.available(&invalid).is_err()); + let mut invalid = request(); + Arc::make_mut(&mut invalid.revision).backend = crate::executor::ExecutorBackend::Native; + assert!(executor.available(&invalid).is_err()); + let mut published = request(); + Arc::make_mut(&mut published.revision).qualifier = "1".into(); + assert!(executor.available(&published).is_err()); + published.function_arn.push_str(":prod"); + executor.available(&published).unwrap(); + published.function_arn = published.function_arn.replace(":prod", ":1"); + executor.available(&published).unwrap(); + assert_eq!(fixture.environments.port().status().allocated, 0); + executor.shutdown().await.unwrap(); + assert!(executor.image_available().is_err()); + fixture.close().await; +} + +#[rstest::rstest] +#[case::success(false, false)] +#[case::error(false, true)] +#[case::tail(true, false)] +#[case::tail_error(true, true)] +#[test] +fn test_should_preserve_binary_outcomes_and_encode_requested_window_once( + #[case] tail: bool, + #[case] failed: bool, +) { + let mut request = request(); + request.capture_logs = tail; + Arc::make_mut(&mut request.revision).qualifier = "17".into(); + request.function_arn.push_str(":prod"); + let raw = Bytes::from_static(b"\x00\xffprivate-handler-data"); + let payload = Payload::try_from(raw.clone()).unwrap(); + let logs = Bytes::from_static(b"\xff\x00private-log-data"); + let result = if failed { + RuntimeResult::FunctionError(payload) + } else { + RuntimeResult::Success(payload) + }; + let response = response( + EnvironmentResponse { + result, + logs: logs.clone(), + }, + &request, + ) + .unwrap(); + assert_eq!(response.payload, raw); + assert_eq!(response.status, 200); + assert_eq!( + response.function_error.as_deref(), + if failed { Some("Unhandled") } else { None } + ); + assert_eq!(response.log_tail, tail.then(|| STANDARD.encode(&logs))); + assert_eq!(response.executed_version, "17"); + assert!(!format!("{response:?}").contains("private")); +} +#[test] +fn test_should_preserve_empty_tail_and_reject_oversized_logs_or_init_success() { + let mut request = request(); + request.capture_logs = true; + let value = || RuntimeResult::Success(Payload::try_from(Bytes::new()).unwrap()); + assert_eq!( + response( + EnvironmentResponse { + result: value(), + logs: Bytes::new() + }, + &request + ) + .unwrap() + .log_tail + .as_deref(), + Some("") + ); + assert!( + response( + EnvironmentResponse { + result: value(), + logs: Bytes::from(vec![0; 4097]) + }, + &request + ) + .is_err() + ); + for size in [32768, 32769] { + let result = + RuntimeResult::InitError(Payload::try_from(Bytes::from(vec![0; size])).unwrap()); + let error = response( + EnvironmentResponse { + result, + logs: Bytes::new(), + }, + &request, + ) + .unwrap_err(); + if size == 32768 { + assert!(matches!( + error, + ExecutorError::Docker(DockerExecutorError::Environment( + EnvironmentError::Initialization(_) + )) + )); + } else { + assert!(matches!( + error, + ExecutorError::Docker(DockerExecutorError::Environment(EnvironmentError::Docker( + DockerError::Protocol + ))) + )); + } + } +} + +#[test] +fn test_should_preserve_typed_environment_failures_and_static_service_mapping() { + let source = DockerExecutorError::from(ProtocolError::Io(std::io::Error::other( + "private-transport-data", + ))); + assert!(source.source().is_some()); + assert!(!format!("{source:?} {source}").contains("private-transport-data")); + assert!(matches!( + environment_error(EnvironmentError::Timeout(Duration::from_secs(3))), + ExecutorError::Timeout(_) + )); + assert!(matches!( + environment_error(EnvironmentError::Deadline), + ExecutorError::DeadlineExceeded + )); + assert!(matches!( + environment_error(EnvironmentError::Cancelled), + ExecutorError::Cancelled + )); + assert!(matches!( + LambdaServiceError::from(environment_error(EnvironmentError::Busy)), + LambdaServiceError::TooManyRequests + )); + let nested = environment_error(EnvironmentError::Docker(DockerError::Busy)); + assert!(matches!( + &nested, + ExecutorError::Docker(DockerExecutorError::Environment(EnvironmentError::Docker( + DockerError::Busy + ))) + )); + assert!(matches!( + LambdaServiceError::from(nested), + LambdaServiceError::Internal { .. } + )); + assert!(matches!( + LambdaServiceError::from(ExecutorError::from(DockerExecutorError::Unavailable)), + LambdaServiceError::DockerNotAvailable + )); + for error in [ + EnvironmentError::Initialization( + Payload::try_from(Bytes::from_static(b"private-error")).unwrap(), + ), + EnvironmentError::Exited { + exit_code: 137, + oom: true, + }, + EnvironmentError::Docker(DockerError::Protocol), + ] { + let error = environment_error(error); + assert!(error.source().is_some()); + assert!(!format!("{error:?} {error}").contains("private-error")); + } +} + +#[rstest::rstest] +#[case::released(false)] +#[case::cleanup_expired(true)] +#[tokio::test] +async fn test_should_hold_work_supervisor_concurrency_through_docker_cancellation_settlement( + #[case] expired: bool, +) { + let fixture = Fixture::new().await; + let config = fixture.config(); + let request = request(); + let plan = EnvironmentPlan::new( + config.directory.clone(), + config.relay.clone(), + request.revision.clone(), + config.policy.clone(), + ) + .unwrap(); + let (environment, peer) = test_peer::pair(); + let (entered, observed) = oneshot::channel(); + let (stopping, stopped) = oneshot::channel(); + let (release, gate) = oneshot::channel(); + let peer = tokio::spawn(peer.serve_pending(entered, gate, stopping)); + let pool = super::super::pool::testing_start( + fixture.environments.port(), + PoolConfig::default(), + vec![(plan.warm_key().into(), environment)], + ); + let port = pool.port(); + let executor = Arc::new(DockerExecutor::with_pool( + config, + pool, + port, + Handle::current(), + )); + let work = WorkManager::new(); + let handle = work + .submit(executor.clone(), request.clone(), Some(1), false, None) + .await + .unwrap(); + let id = handle.id(); + let actual = observed.await.unwrap(); + assert_eq!(actual.parts().0.to_string(), id.to_string()); + assert_eq!(actual.parts().2.as_ref(), request.function_arn.as_str()); + assert_eq!(actual.parts().4.as_ref(), request.payload.as_ref()); + let mut waiting = Box::pin(handle.cancel_and_wait(deadline())); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + stopped.await.unwrap(); + let rejected = work + .submit(executor.clone(), request, Some(1), false, None) + .await; + assert!( + matches!(rejected, Err(LambdaServiceError::TooManyRequests)), + "{rejected:?}" + ); + assert!( + waiting + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + let mut shutdown = Box::pin(executor.shutdown()); + assert!( + shutdown + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending() + ); + drop(shutdown); + assert!(executor.image_available().is_err()); + if expired { + assert!(matches!( + executor.shutdown().await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(matches!( + waiting.await, + Err(ExecutorError::CleanupFailed(_)) + )); + assert!(work.quiesce(Duration::from_secs(1)).await.is_err()); + release.send(()).unwrap(); + } else { + release.send(()).unwrap(); + waiting.await.unwrap(); + work.quiesce(Duration::from_secs(10)).await.unwrap(); + } + peer.await.unwrap(); + executor.shutdown().await.unwrap(); + fixture.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index eeaccdf3..57dec94b 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -5,6 +5,7 @@ mod connection; mod environment; +mod executor; mod framing; mod image; mod lease; @@ -33,6 +34,10 @@ pub use environment::{ EnvironmentManager, EnvironmentPhase, EnvironmentPort, EnvironmentResponse, EnvironmentSettings, EnvironmentStatus, }; +pub use executor::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, +}; pub use framing::{DockerOutputReader, MAX_DOCKER_RECORD, OutputChannel, OutputChunk}; use http::{Method, StatusCode}; use hyper::upgrade::Upgraded; diff --git a/crates/rustack-lambda-core/src/docker/pool/actor.rs b/crates/rustack-lambda-core/src/docker/pool/actor.rs index f7753816..048c6428 100644 --- a/crates/rustack-lambda-core/src/docker/pool/actor.rs +++ b/crates/rustack-lambda-core/src/docker/pool/actor.rs @@ -420,7 +420,7 @@ async fn invoke( } #[cfg(test)] -pub(super) fn testing_start( +pub(in crate::docker) fn testing_start( environments: EnvironmentPort, config: PoolConfig, warm: Vec<(String, EnvironmentHandle)>, diff --git a/crates/rustack-lambda-core/src/docker/pool/mod.rs b/crates/rustack-lambda-core/src/docker/pool/mod.rs index 3fd926fc..9b9adc42 100644 --- a/crates/rustack-lambda-core/src/docker/pool/mod.rs +++ b/crates/rustack-lambda-core/src/docker/pool/mod.rs @@ -1,7 +1,6 @@ //! Bounded warm dispatch; external resource ownership stays with `EnvironmentManager`. mod actor; - use std::{ fmt, sync::{ @@ -10,6 +9,8 @@ use std::{ }, }; +#[cfg(test)] +pub(super) use actor::testing_start; use rustack_lambda_runtime::session::Invocation; use tokio::{ runtime::Handle, diff --git a/crates/rustack-lambda-core/src/executor/error.rs b/crates/rustack-lambda-core/src/executor/error.rs index 0e593dff..a9599b66 100644 --- a/crates/rustack-lambda-core/src/executor/error.rs +++ b/crates/rustack-lambda-core/src/executor/error.rs @@ -1,10 +1,16 @@ //! Errors surfaced by an [`Executor`](super::Executor). +#[cfg(unix)] +use crate::docker::{DockerExecutorError, EnvironmentError}; use crate::error::LambdaServiceError; /// Failure modes for an execution backend. #[derive(Debug, Clone, thiserror::Error)] pub enum ExecutorError { + /// Typed image-runtime, request or readiness failure with redacted sources. + #[cfg(unix)] + #[error("{0}")] + Docker(#[from] DockerExecutorError), /// Caller or owner requested cancellation; settlement is tracked separately. #[error("invocation cancelled")] Cancelled, @@ -48,6 +54,14 @@ pub enum ExecutorError { impl From for LambdaServiceError { fn from(e: ExecutorError) -> Self { match e { + #[cfg(unix)] + ExecutorError::Docker(error) => match &error { + DockerExecutorError::Unavailable => Self::DockerNotAvailable, + DockerExecutorError::Environment(EnvironmentError::Busy) => Self::TooManyRequests, + _ => Self::Internal { + message: error.to_string(), + }, + }, ExecutorError::Cancelled | ExecutorError::DeadlineExceeded => Self::ResourceNotReady { message: e.to_string(), }, diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index 2adadb35..c2d0c0ef 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -7,7 +7,8 @@ //! - [`NativeExecutor`] runs trusted host-matching bootstraps without isolation. //! - [`SquibExecutor`] runs arm64 Zip functions through a microVM guest agent. //! -//! Docker execution is not supported; it never falls back to native or successful echo. +//! On Unix, `DockerExecutor` composes the retained image runtime using a trusted +//! readiness port. Ordinary CLI Docker selection requires the startup coordinator. //! //! Native instances expose a host Runtime API socket; Squib delegates the //! bootstrap-side protocol to its guest agent. @@ -33,6 +34,11 @@ pub use noop::NoopExecutor; pub use squib::{SquibExecutor, SquibExecutorConfig}; pub use types::{ExecutionRevision, ExecutorBackend, InvokeRequest, InvokeResponse}; +#[cfg(unix)] +pub use crate::docker::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, +}; use crate::invocation::InvocationContext; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index dcf1e299..62918375 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -185,7 +185,8 @@ pub struct InvokeResponse { pub payload: Bytes, /// `Some("Unhandled")` when the bootstrap posted to `/error`. pub function_error: Option, - /// Base64-encoded last 4KB of stderr, if `capture_logs` was set. + /// Base64-encoded final bounded log bytes when requested. Docker combines + /// observed stdout/stderr in its invocation window; Native uses stderr. pub log_tail: Option, /// What the executor actually invoked (echoes back the qualifier). pub executed_version: String, diff --git a/crates/rustack-lambda-core/src/handler.rs b/crates/rustack-lambda-core/src/handler.rs index 64600ab8..77af70f0 100644 --- a/crates/rustack-lambda-core/src/handler.rs +++ b/crates/rustack-lambda-core/src/handler.rs @@ -208,11 +208,21 @@ async fn dispatch( let function_name = require_path_param(path_params, "FunctionName")?; // Parse InvocationType from header. - let invocation_type = headers - .get("x-amz-invocation-type") - .and_then(|v| v.to_str().ok()) - .and_then(InvocationType::from_str_value) - .unwrap_or(InvocationType::RequestResponse); + let invocation_type = match single_invoke_header(headers, "x-amz-invocation-type")? { + None => InvocationType::RequestResponse, + Some(value) => value + .to_str() + .ok() + .and_then(InvocationType::from_str_value) + .ok_or_else(|| LambdaError::invalid_parameter("Invalid InvocationType"))?, + }; + let capture_logs = match single_invoke_header(headers, "x-amz-log-type")? + .map(http::HeaderValue::as_bytes) + { + None | Some(b"None") => false, + Some(b"Tail") => true, + Some(_) => return Err(LambdaError::invalid_parameter("Invalid LogType")), + }; let kind = match invocation_type { InvocationType::RequestResponse => InvokeKind::RequestResponse, InvocationType::Event => InvokeKind::Event, @@ -220,7 +230,7 @@ async fn dispatch( }; let outcome = provider - .invoke(function_name, qualifier, body, kind) + .invoke_with_logs(function_name, qualifier, body, kind, capture_logs) .await .map_err(LambdaError::from)?; @@ -712,6 +722,20 @@ fn require_path_param<'a>(params: &'a PathParams, name: &str) -> Result<&'a str, .ok_or_else(|| LambdaError::invalid_parameter(format!("Missing path parameter: {name}"))) } +fn single_invoke_header<'a>( + headers: &'a http::HeaderMap, + name: &'static str, +) -> Result, LambdaError> { + let mut values = headers.get_all(name).iter(); + let value = values.next(); + if values.next().is_some() { + return Err(LambdaError::invalid_parameter( + "Duplicate invocation header", + )); + } + Ok(value) +} + /// Wrap a serializable value into a JSON `LambdaResponseBody` response. fn wrap_json_response( status: u16, @@ -737,7 +761,64 @@ fn wrap_empty_response(status: u16) -> Result #[cfg(test)] mod tests { + use http::{HeaderMap, HeaderValue, Method}; + use rustack_lambda_http::router::resolve_operation; + use rustack_lambda_model::error::LambdaErrorCode; + use super::*; + use crate::config::LambdaConfig; + + #[rstest::rstest] + #[case::log_case("x-amz-log-type", b"tail".as_slice())] + #[case::log_unknown("x-amz-log-type", b"private-invalid-log-type".as_slice())] + #[case::log_nonascii("x-amz-log-type", b"\xff".as_slice())] + #[case::invocation_case("x-amz-invocation-type", b"event".as_slice())] + #[case::invocation_unknown("x-amz-invocation-type", b"private-invalid-invocation-type".as_slice())] + #[case::invocation_nonascii("x-amz-invocation-type", b"\xff".as_slice())] + #[tokio::test] + async fn test_should_reject_invalid_invoke_headers_before_resource_lookup( + #[case] key: &'static str, + #[case] value: &[u8], + ) { + let provider = RustackLambda::new(LambdaConfig::default()); + let (operation, params, _) = + resolve_operation(&Method::POST, "/2015-03-31/functions/missing/invocations").unwrap(); + let mut headers = HeaderMap::new(); + headers.insert(key, HeaderValue::from_bytes(value).unwrap()); + let error = dispatch(&provider, operation, ¶ms, "", &headers, &Bytes::new()) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::InvalidParameterValueException); + assert!(!format!("{error:?}").contains("private-invalid")); + provider.shutdown().await; + } + + #[rstest::rstest] + #[case::tail_none("x-amz-log-type", "Tail", "None")] + #[case::none_tail("x-amz-log-type", "None", "Tail")] + #[case::tail_invalid("x-amz-log-type", "Tail", "private-invalid-value")] + #[case::sync_event("x-amz-invocation-type", "RequestResponse", "Event")] + #[case::event_sync("x-amz-invocation-type", "Event", "RequestResponse")] + #[case::sync_invalid("x-amz-invocation-type", "RequestResponse", "private-invalid-value")] + #[tokio::test] + async fn test_should_reject_duplicate_invoke_enum_headers_before_resource_lookup( + #[case] key: &'static str, + #[case] first: &'static str, + #[case] second: &'static str, + ) { + let provider = RustackLambda::new(LambdaConfig::default()); + let (operation, params, _) = + resolve_operation(&Method::POST, "/2015-03-31/functions/missing/invocations").unwrap(); + let mut headers = HeaderMap::new(); + headers.append(key, HeaderValue::from_static(first)); + headers.append(key, HeaderValue::from_static(second)); + let error = dispatch(&provider, operation, ¶ms, "", &headers, &Bytes::new()) + .await + .unwrap_err(); + assert_eq!(error.code, LambdaErrorCode::InvalidParameterValueException); + assert!(!format!("{error:?}").contains("private-invalid")); + provider.shutdown().await; + } #[test] fn test_should_parse_empty_query() { diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 48d3a17a..a5ba10ea 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -1112,6 +1112,32 @@ impl RustackLambda { qualifier: Option<&str>, payload: &[u8], invocation_type: InvokeKind, + ) -> Result { + self.invoke_with_logs(function_ref, qualifier, payload, invocation_type, false) + .await + } + + /// Invoke through shared admission with an optional synchronous log tail. + /// + /// Event and `DryRun` calls never capture response logs. The HTTP adapter passes + /// only its already-validated `LogType` choice; internal adapters default to None. + /// + /// # Errors + /// Returns the same validation, admission and backend errors as `invoke`. + /// + /// ```no_run + /// # async fn call(provider: &rustack_lambda_core::provider::RustackLambda) { + /// use rustack_lambda_core::provider::InvokeKind; + /// let result = provider.invoke_with_logs("worker", None, b"{}", InvokeKind::RequestResponse, true).await; + /// # } + /// ``` + pub async fn invoke_with_logs( + &self, + function_ref: &str, + qualifier: Option<&str>, + payload: &[u8], + invocation_type: InvokeKind, + capture_logs: bool, ) -> Result { validate_invoke_payload( payload, @@ -1133,6 +1159,7 @@ impl RustackLambda { payload, invocation_type == InvokeKind::Event, None, + capture_logs && invocation_type == InvokeKind::RequestResponse, ) .await?; if invocation_type == InvokeKind::Event { @@ -1172,8 +1199,15 @@ impl RustackLambda { ) -> Result { validate_invoke_payload(payload, MAX_SYNC_PAYLOAD)?; let (record, qualifier) = self.resolve_invocation(function_ref, qualifier)?; - self.admit_invocation(&record, qualifier.as_deref(), payload, false, deadline) - .await + self.admit_invocation( + &record, + qualifier.as_deref(), + payload, + false, + deadline, + false, + ) + .await } fn resolve_invocation( @@ -1214,16 +1248,18 @@ impl RustackLambda { payload: &[u8], asynchronous: bool, deadline: Option, + capture_logs: bool, ) -> Result { let version = resolve_version(record, qualifier)?; let capture = self.build_invoke_request(record, version, qualifier, payload); - let request = if let Some(deadline) = deadline { + let mut request = if let Some(deadline) = deadline { timeout_at(deadline, capture).await.map_err(|_| { LambdaServiceError::from(crate::executor::ExecutorError::DeadlineExceeded) })?? } else { capture.await? }; + request.capture_logs = capture_logs; self.work .submit( Arc::clone(&self.executor), diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs index 4ebbab8c..2e5d2932 100644 --- a/crates/rustack-lambda-core/src/work.rs +++ b/crates/rustack-lambda-core/src/work.rs @@ -426,7 +426,7 @@ async fn execute( ) -> Result { let acquired = async { match permits { - Some(permits) => Ok(permits), + Some(permits) => Ok::<_, ExecutorError>(permits), None => Ok(( function.acquire(limit).await, global diff --git a/crates/rustack-lambda-core/tests/common/docker.rs b/crates/rustack-lambda-core/tests/common/docker.rs new file mode 100644 index 00000000..15cf2853 --- /dev/null +++ b/crates/rustack-lambda-core/tests/common/docker.rs @@ -0,0 +1,131 @@ +//! Shared explicit-daemon fixture. Readiness/probes are not implied by setup. +use std::{env, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::{ + artifact::{Architecture, PinnedImage}, + docker::{ + DirectoryLease, DockerWire, EnvironmentConfig, EnvironmentManager, EnvironmentSettings, + ImageLeaseConfig, ImageLeaseManager, ImagePinScope, OwnerFilter, + }, +}; +use tempfile::TempDir; +use tokio::time::Instant; +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(60) +} +pub(crate) struct Fixture { + pub(crate) owner: EnvironmentManager, + pub(crate) images: ImageLeaseManager, + pub(crate) wire: DockerWire, + pub(crate) lease: DirectoryLease, + pub(crate) relay: Option, + pub(crate) function: Option, + pub(crate) directory: Option, +} +impl Fixture { + pub(crate) async fn new() -> Result { + let socket = + env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket required")?; + let wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; + let directory = tempfile::tempdir()?; + let lease = DirectoryLease::acquire(directory.path().into(), deadline()).await?; + let images = ImageLeaseManager::start( + wire.port(), + ImagePinScope::new(lease.clone()), + ImageLeaseConfig::default(), + )?; + let arch = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown architecture"), + }; + let config = EnvironmentConfig::try_from( + EnvironmentSettings::builder() + .environments(1) + .memory_bytes(256 * 1024 * 1024) + .io(Duration::from_secs(60)) + .start(Duration::from_secs(60)) + .build(), + )?; + let owner = EnvironmentManager::start(wire.port(), lease.clone(), config)?; + let mut fixture = Self { + owner, + images, + wire, + lease, + relay: None, + function: None, + directory: Some(directory), + }; + if let Err(error) = fixture.prepare(arch).await { + let cleanup = fixture.cleanup().await; + return Err(error.context(format!( + "setup cleanup: {}", + if cleanup.is_ok() { + "confirmed" + } else { + "unresolved" + } + ))); + } + Ok(fixture) + } + async fn prepare(&mut self, arch: Architecture) -> Result<()> { + let relay_reference = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let relay = self + .images + .port() + .prepare(&relay_reference, arch, None, deadline()) + .await?; + self.relay = Some(relay); + let suffix = if arch == Architecture::Arm64 { + "arm64" + } else { + "amd64" + }; + let function = self + .images + .port() + .prepare( + &format!("rustack-p0-handler:{suffix}").parse()?, + arch, + None, + deadline(), + ) + .await?; + self.function = Some(function); + Ok(()) + } + + pub(crate) async fn cleanup(&mut self) -> Result<()> { + let result = async { + self.owner.shutdown(deadline()).await?; + ensure!( + self.owner.port().status().allocated == 0, + "allocation reservation remains" + ); + let containers = self + .wire + .port() + .list_containers( + OwnerFilter::new(self.lease.owner_id(), Some(self.lease.run_id()), None)?, + deadline(), + ) + .await?; + ensure!(containers.is_empty(), "owned containers remain"); + self.relay.take(); + self.function.take(); + self.images.shutdown(deadline()).await?; + self.wire.shutdown().await?; + Ok(()) + } + .await; + if result.is_err() { + if let Some(directory) = self.directory.take() { + let _ = directory.keep(); + } + } + result + } +} diff --git a/crates/rustack-lambda-core/tests/common/mod.rs b/crates/rustack-lambda-core/tests/common/mod.rs new file mode 100644 index 00000000..c4fdd744 --- /dev/null +++ b/crates/rustack-lambda-core/tests/common/mod.rs @@ -0,0 +1 @@ +pub(crate) mod docker; diff --git a/crates/rustack-lambda-core/tests/docker_environment.rs b/crates/rustack-lambda-core/tests/docker_environment.rs index eecc18b0..82e80286 100644 --- a/crates/rustack-lambda-core/tests/docker_environment.rs +++ b/crates/rustack-lambda-core/tests/docker_environment.rs @@ -1,17 +1,15 @@ //! Real container lifecycle qualification of the retained environment owner. #![cfg(unix)] #![forbid(unsafe_code)] -use std::{collections::HashMap, env, sync::Arc, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use anyhow::{Context, Result, ensure}; use bytes::Bytes; use rustack_lambda_core::{ - artifact::{Architecture, DeploymentArtifact, PinnedImage}, + artifact::DeploymentArtifact, docker::{ - ContainerPolicy, DirectoryLease, DockerWire, EnvironmentConfig, EnvironmentError, - EnvironmentHandle, EnvironmentManager, EnvironmentPlan, EnvironmentPool, - EnvironmentSettings, FunctionLogs, ImageLeaseConfig, ImageLeaseManager, ImagePinScope, - OwnerFilter, PoolConfig, RelayBudgets, + ContainerPolicy, EnvironmentError, EnvironmentHandle, EnvironmentPlan, EnvironmentPool, + FunctionLogs, PoolConfig, RelayBudgets, }, executor::{ExecutionRevision, ExecutorBackend}, identity::FunctionResourceId, @@ -22,97 +20,15 @@ use rustack_lambda_runtime::{ session::{Invocation, RuntimeResult}, }; use serde_json::{Value, json}; -use tempfile::TempDir; use tokio::time::{Instant, timeout}; use uuid::Uuid; fn deadline() -> Instant { Instant::now() + Duration::from_secs(60) } -struct Fixture { - owner: EnvironmentManager, - images: ImageLeaseManager, - wire: DockerWire, - lease: DirectoryLease, - relay: Option, - function: Option, - directory: Option, -} +mod common; +use common::docker::Fixture; impl Fixture { - async fn new() -> Result { - let socket = - env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket required")?; - let wire = DockerWire::connect(format!("unix://{socket}").parse()?, deadline()).await?; - let directory = tempfile::tempdir()?; - let lease = DirectoryLease::acquire(directory.path().into(), deadline()).await?; - let images = ImageLeaseManager::start( - wire.port(), - ImagePinScope::new(lease.clone()), - ImageLeaseConfig::default(), - )?; - let arch = match env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { - "arm64" => Architecture::Arm64, - "amd64" => Architecture::X86_64, - _ => anyhow::bail!("unknown architecture"), - }; - let config = EnvironmentConfig::try_from( - EnvironmentSettings::builder() - .environments(1) - .memory_bytes(256 * 1024 * 1024) - .io(Duration::from_secs(60)) - .start(Duration::from_secs(60)) - .build(), - )?; - let owner = EnvironmentManager::start(wire.port(), lease.clone(), config)?; - let mut fixture = Self { - owner, - images, - wire, - lease, - relay: None, - function: None, - directory: Some(directory), - }; - if let Err(error) = fixture.prepare(arch).await { - let cleanup = fixture.cleanup().await; - return Err(error.context(format!( - "setup cleanup: {}", - if cleanup.is_ok() { - "confirmed" - } else { - "unresolved" - } - ))); - } - Ok(fixture) - } - async fn prepare(&mut self, arch: Architecture) -> Result<()> { - let relay_reference = env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; - let relay = self - .images - .port() - .prepare(&relay_reference, arch, None, deadline()) - .await?; - self.relay = Some(relay); - let suffix = if arch == Architecture::Arm64 { - "arm64" - } else { - "amd64" - }; - let function = self - .images - .port() - .prepare( - &format!("rustack-p0-handler:{suffix}").parse()?, - arch, - None, - deadline(), - ) - .await?; - self.function = Some(function); - Ok(()) - } - - fn revision(&self) -> Result> { + pub(crate) fn revision(&self) -> Result> { let revision = ExecutionRevision::builder() .resource_id(FunctionResourceId::new()) .revision_id(Uuid::new_v4().to_string()) @@ -143,10 +59,13 @@ impl Fixture { .build(); Ok(Arc::new(revision)) } - fn plan(&self) -> Result { + pub(crate) fn plan(&self) -> Result { self.plan_revision(self.revision()?) } - fn plan_revision(&self, revision: Arc) -> Result { + pub(crate) fn plan_revision( + &self, + revision: Arc, + ) -> Result { Ok(EnvironmentPlan::new( self.lease.clone(), self.relay.as_ref().context("relay image")?.clone(), @@ -164,37 +83,8 @@ impl Fixture { )?, )?) } - async fn cleanup(&mut self) -> Result<()> { - let result = async { - self.owner.shutdown(deadline()).await?; - ensure!( - self.owner.port().status().allocated == 0, - "allocation reservation remains" - ); - let containers = self - .wire - .port() - .list_containers( - OwnerFilter::new(self.lease.owner_id(), Some(self.lease.run_id()), None)?, - deadline(), - ) - .await?; - ensure!(containers.is_empty(), "owned containers remain"); - self.relay.take(); - self.function.take(); - self.images.shutdown(deadline()).await?; - self.wire.shutdown().await?; - Ok(()) - } - .await; - if result.is_err() { - if let Some(directory) = self.directory.take() { - let _ = directory.keep(); - } - } - result - } } + fn invocation(event: &Value) -> Result { Ok(Invocation::new( AttemptId::default(), diff --git a/crates/rustack-lambda-core/tests/docker_executor.rs b/crates/rustack-lambda-core/tests/docker_executor.rs new file mode 100644 index 00000000..c2bf4e23 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_executor.rs @@ -0,0 +1,182 @@ +//! Real SDK/HTTP/provider execution; controlled fixture readiness is not startup evidence. +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use hyper::server::conn::http1; +use hyper_util::rt::TokioIo; +use rustack_lambda_core::{ + config::LambdaConfig, + deployment::ImageDeploymentConfig, + docker::{ContainerPolicy, PoolConfig, RelayBudgets}, + executor::{ + DockerExecutor, DockerExecutorConfig, DockerExecutorError, DockerExecutorSettings, + DockerReadiness, Executor, ExecutorBackend, + }, + handler::RustackLambdaHandler, + provider::RustackLambda, + storage::FunctionStore, +}; +use rustack_lambda_http::{LambdaHttpConfig, LambdaHttpService}; +use tokio::{net::TcpListener, process::Command, sync::watch, task::JoinSet, time::timeout}; + +mod common; +use common::docker::Fixture; + +#[derive(Debug)] +struct CompositionOnlyReadiness; +impl DockerReadiness for CompositionOnlyReadiness { + fn check(&self) -> Result<(), DockerExecutorError> { + Ok(()) + } +} + +#[tokio::test] +#[ignore = "requires explicit Docker images/socket and pinned boto3 fixture"] +async fn test_should_deploy_publish_alias_and_invoke_real_images_through_boto3_http() -> Result<()> +{ + let mut fixture = Fixture::new().await?; + let result = run(&fixture).await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} +async fn run(fixture: &Fixture) -> Result<()> { + let relay = fixture.relay.as_ref().context("relay image")?.clone(); + let executor = Arc::new(DockerExecutor::new( + DockerExecutorConfig::builder() + .directory(fixture.lease.clone()) + .relay(relay) + .policy(ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?) + .environments(fixture.owner.port()) + .readiness(Arc::new(CompositionOnlyReadiness)) + .settings(DockerExecutorSettings::new( + PoolConfig::default(), + Duration::from_secs(60), + )?) + .build(), + )?); + let result = serve(fixture, executor.clone()).await; + let retired = executor.shutdown().await; + result.and(retired.map_err(Into::into)) +} +async fn serve(fixture: &Fixture, executor: Arc) -> Result<()> { + let store = FunctionStore::new( + fixture + .directory + .as_ref() + .context("data directory")? + .path() + .join("code"), + ); + let config = LambdaConfig { + executor: ExecutorBackend::Docker, + ..Default::default() + }; + let provider = Arc::new( + RustackLambda::with_executor(store, config, executor.clone()) + .with_image_preparation(fixture.images.port(), ImageDeploymentConfig::default())?, + ); + let result = run_server(fixture, provider.clone()).await; + let quiesced = provider.quiesce(Duration::from_secs(60)).await; + let retired = executor.shutdown().await; + provider.shutdown().await; + result + .and(quiesced.map_err(Into::into)) + .and(retired.map_err(Into::into)) +} +async fn run_server(fixture: &Fixture, provider: Arc) -> Result<()> { + let image = fixture + .function + .as_ref() + .context("function image")? + .resolved() + .metadata(); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let service = LambdaHttpService::new( + Arc::new(RustackLambdaHandler::new(provider)), + LambdaHttpConfig::default(), + ); + let (stop, mut stopped) = watch::channel(false); + let mut server = tokio::spawn(async move { + let mut jobs = JoinSet::new(); + let result = async { loop { + tokio::select! {biased; + _=stopped.wait_for(|stop|*stop)=>return Ok::<(),anyhow::Error>(()), + joined=jobs.join_next(),if !jobs.is_empty()=> {joined.unwrap()??;}, + stream=listener.accept(),if jobs.len()<8=> { + let (stream,_)=stream?;let service=service.clone(); + jobs.spawn(async move {http1::Builder::new().serve_connection(TokioIo::new(stream),service).await}); + } + } + } }.await; + jobs.abort_all(); + let mut workers_ok = true; + while let Some(joined) = jobs.join_next().await { + if let Err(error) = joined { + workers_ok &= error.is_cancelled(); + } + } + result?; + ensure!(workers_ok, "HTTP worker failed"); + Ok::<(), anyhow::Error>(()) + }); + let result = run_sdk( + &endpoint, + image.requested_reference.as_str(), + image.architecture.as_str(), + ) + .await; + stop.send_replace(true); + let joined = timeout(Duration::from_secs(60), &mut server).await; + let joined = match joined { + Ok(result) => result.context("HTTP supervisor join")?, + Err(error) => { + server.abort(); + let _ = server.await; + Err(error.into()) + } + }; + result.and(joined) +} +async fn run_sdk(endpoint: &str, image: &str, architecture: &str) -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut child = Command::new("uv"); + child + .args([ + "run", + "--no-project", + "--python", + "3.13", + "--with-requirements", + ]) + .arg(root.join("tests/workflows/requirements.txt")) + .arg("python") + .arg(root.join("tests/workflows/docker_lambda_sdk.py")) + .args([endpoint, image, architecture]) + .kill_on_drop(true); + let mut process = child.spawn().context("start pinned SDK fixture")?; + let result = timeout(Duration::from_secs(300), process.wait()).await; + let status = match result { + Ok(result) => result?, + Err(error) => { + let _ = process.kill().await; + let _ = process.wait().await; + return Err(error.into()); + } + }; + ensure!(status.success(), "SDK fixture failed: {status}"); + Ok(()) +} diff --git a/specs/index.md b/specs/index.md index 49fed672..faa627b2 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until Executor/provider response wiring, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-00-prd.md b/specs/rustack-workflows-00-prd.md index 57eea870..44f23de8 100644 --- a/specs/rustack-workflows-00-prd.md +++ b/specs/rustack-workflows-00-prd.md @@ -1,6 +1,6 @@ # Rustack Workflows: Step Functions and Real Lambda Execution -Status: implementation contract; not implemented · Date: 2026-10-25 · Delivery: M0–M4 +Status: implementation in progress; full M0–M4 incomplete · Date: 2026-10-25 · Delivery: M0–M4 ## 1. Problem and vision diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 242b57e8..b05088c9 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -660,6 +660,41 @@ Lambda work owner PoolInvocation Pool actor registry Env Tests must hold retirement behind a channel barrier and prove cancellation does not return early, including invocation expiry and dropped cleanup waiters. Cover queued no-allocation rejection, consumed-success bookkeeping without retiring warm reuse, root/worker abort recovery and closed-receipt uncertainty. These ownership tests do not establish actual Docker/provider acceptance. +### 4.6 Docker Executor and provider composition + +`executor::DockerExecutor` implements the existing object-safe Executor over EnvironmentPool. `DockerExecutorConfig` is a typed-builder input containing the retained DirectoryLease, live pinned relay, ContainerPolicy, matching EnvironmentPort, trusted readiness port and validated DockerExecutorSettings. Settings contain PoolConfig and an independent cleanup budget (whole seconds1..60, default10). Construction validates scope, lease availability, settings and an entered Tokio runtime before creating the bounded pool; it performs no daemon mutation. Native/Squib/Auto remain independent. The binary's ordinary Docker selection remains rejected until task1.4 startup acceptance is implemented. + +The synchronous object-safe `DockerReadiness::check()` port returns a typed DockerExecutorError. It is supplied by the **trusted in-process startup coordinator**, which owns reconciliation/probe evidence for the exact directory/relay/policy it injects; checks are bounded, nonblocking and contain no daemon IO. This is the existing trusted Rust dependency-injection boundary (which already permits custom Executors), not a user-request field or a helper that proves readiness from booleans/socket connectivity. Tests may supply a controlled readiness fixture and must label that narrower evidence. Production must supply the actual historical-reconciliation, pull/network/resource probe consumer before CLI enablement. Revocation rejects new admission; explicit runtime shutdown separately retires accepted work. + +`image_available` requires the readiness port, a live relay, accepting/healthy environment service and pool, and an executor that has not begun shutdown. `available` additionally constructs/validates a pure EnvironmentPlan and validates bounded raw payload and FunctionArn, the captured function name and region, and the requested qualifier versus resolved version. Bare/$LATEST ARNs require captured $LATEST; numeric qualifiers must equal the resolved version; aliases require a published numeric revision. Preserve alias text for Runtime API metadata. No admission probe pulls/resolves/inspects an image, starts code or contacts a public endpoint. + +Invoke rechecks availability and InvocationContext, creates one Invocation with the context's UUID/deadline, local trace and original Bytes, then calls PoolPort::start once. Select the cancel-safe wait against context cancellation. On every accepted call, await the separate pool settlement with the independent cleanup budget before returning a response/error to WorkSupervisor. A consumed successful response only waits for bookkeeping; it cannot cancel an environment reused by another call. Unresolved/uncertain settlement becomes existing ExecutorError::CleanupFailed, so WorkSupervisor cannot report clean quiescence. The retained pool/environment owners keep physical cleanup after waiter expiry or Executor Drop. Never replay a request following uncertain transport or function failure. + +HTTP Invoke strictly accepts InvocationType RequestResponse/Event/DryRun and LogType None/Tail (omission defaults to RequestResponse/None); repeated scalar headers and invalid/non-ASCII values return a static InvalidParameter error before admission. Preserve existing provider.invoke as the default no-tail convenience entry point and add invoke_with_logs to carry the validated capture request through immutable admission. Only RequestResponse requests return tails; Event/DryRun do not capture response logs. Internal start_invocation retains its no-tail default. + +Successful raw `/response` bytes produce InvokeResponse HTTP200. A raw `/error` produces HTTP200 with FunctionError=Unhandled and unchanged payload. ExecutedVersion is the captured resolved qualifier, while metadata receives the requested alias ARN. LogType=Tail base64-encodes the already bounded invocation window exactly once, including Some("") for an empty requested tail; None produces no log header. Reject an oversized internal tail rather than accidentally encoding diagnostic history. RuntimeResult::InitError is defensively classified as bounded initialization failure, never successful output. Environment Timeout/Deadline/Cancelled retain their existing ExecutorError categories; pool/environment admission Busy maps to Lambda TooManyRequests; nested Docker wire Busy remains a typed infrastructure error (HTTP500), because it can arise after handler side effects and does not prove pre-execution quota rejection; initialization/early-exit/OOM/protocol/transport errors remain distinct typed sources under DockerExecutorError::Environment. Invalid request/clock/session metadata have typed source variants and static redacted diagnostics. Infrastructure errors never become fabricated handler output. + +Shutdown first closes executor admission synchronously, then transfers its pool root into a retained CleanupTask before awaiting. The owned cleanup operation retries timed-out pool joins without losing its root; public shutdown waiters use the independent budget and can resume. Physical pool settlement precedes environment-manager shutdown, image-owner revocation and wire shutdown in the startup coordinator. Do not hold a mutex over IO; a short owner-slot mutex only transfers the root once. Shutdown and Drop share the same path and keep the cached supervision outcome. + +```text +┌─ Lambda HTTP/provider ──────────────────────────────────────────────────┐ +│ immutable revision + requested ARN → WorkSupervisor (retains permits) │ +└────────────────────────────────┬────────────────────────────────────────┘ + ▼ Executor::invoke +┌─ DockerExecutor ────────────────────────────────────────────────────────┐ +│ pure plan/metadata validation ◀── trusted startup readiness check │ +│ raw result + one base64 tail │ cancellation → wait settlement │ +│ retained CleanupTask owns pool shutdown after waiter/Executor loss │ +└────────────────────────────────┬────────────────────────────────────────┘ + ▼ one accepted PoolInvocation +┌─ retained pool/environment actors ──────────────────────────────────────┐ +│ bounded FIFO / warm checkout → actual relay + handler → raw outcome │ +│ explicit result consumption │ resource/IO retirement → cleanup receipt │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +Verification includes pure rejection before allocation, dynamic readiness/pin revocation, alias/resolved-version metadata, binary response/FunctionError/tail preservation, typed failures, context cancellation with delayed settlement, dropped shutdown waiters and actual Image create→Active→publish/alias→Invoke through provider/HTTP. A real Docker fixture readiness adapter proves execution composition only; final production startup/probe/platform acceptance remains separately required. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 9dda3987..2b0cc880 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -104,6 +104,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi The B05 regression composes the actual session/HTTP/relay/client with an environment worker after BootstrapReady and an abandoned poll. For an active call, observe an offered but unconsumed event with a recording waker, abandon that offer, then POST InitError. Test Idle and Busy at32KiB and32KiB+1, require the bounded initialization class versus Protocol, redacted diagnostics and joined control/log/API retirement. Scripted container absence remains distinct from real Docker code execution. +Docker Executor composition (§4.6/D36) additionally requires pure admission rejection, trusted readiness revocation, raw/binary success and FunctionError preservation, one Tail base64 encoding with empty/None distinction, resolved-version versus requested-alias metadata, typed initialization/OOM/transport outcomes and independent cancellation/shutdown settlement. The actual image/provider/HTTP fixture must perform deployment, version/alias and computed-result/error calls. Controlled fixture readiness is not production startup/probe evidence. `make test-lambda-executor` runs the Executor/header boundary and supervisor tests; `make test-lambda-executor-docker` builds the pinned fixtures and executes boto3→HTTP→provider→actual image deployment/version/alias/results/logs/error/timeout/replacement. Scalar header duplicates must reject before lookup. Only admission quota Busy maps429; nested wire Busy remains typed infrastructure500. Exact invoked alias ARN must reach the RIC independently from resolved-version response headers. + The pool settlement consumer tests exercise `start`/cancel-safe `wait`/independent `cancel_and_wait`. Hold retirement behind an explicit barrier; require pending settlement after invocation expiry, after a dropped cleanup waiter, and during root-abort recovery. Queued rejection must settle without allocation, consumed success must preserve warm reuse, and closed or supervision-failed receipts must never report clean. `make test-lambda-log-windows` verifies diagnostics remain environment-wide while each invocation window starts empty: binary order/overflow, silent next call, partial records across Begin/Finish, matching EOF finish, capture failure, stale/cross-owner capabilities and direct capability Drop. Mutation receipts require direct tests where a ready reply is abandoned or remains live/unpolled through deadline: the quiet socket must close before any later diagnostic request, and the local handle must forbid another mutation. Also cover on-time receipts processed after deadline, no-effect expired/abandoned commands, priority-stop rejection and flooded-output fairness. `make test-lambda-log-windows-docker` runs the seven-case owner/pool suite: the window case alternates Tail/logging, None/logging, Tail/silent and Tail/logging through actual warm counters1..4. Require computed payloads, expected observed stdout/stderr markers and empty silent/None tails; never use the diagnostic tail as the response. The independent-stream timing limitation in runtime §3.9 remains explicit. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 481df094..f4d5fa30 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Missing: real Docker Executor/provider response composition; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Docker Executor/provider/HTTP composition is implemented and qualified through controlled-readiness real SDK fixtures. Missing: production runtime startup construction; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -152,6 +152,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 invocation logs | Implemented/reviewed; Executor/provider response composition remains | §3.9/D34 separate bounded per-attempt windows from diagnostic rings, with opaque revocable capabilities, actor-retained consumption receipts/deadlines, exact ownership, pinned partial reads and Tail/None integration. W85 self-review correction and final independent recheck pass.36 log cases (18 added) pass; core571 tests/134 doctests; workspace2,273 tests/407 gated; strict boundary857 unit tests/201 doctests (1,058 total). Standard build/test/fmt/Clippy/rustdoc, strict production, audit/deny pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes owner/pool6 + wire2 + shared runtime5 cases (13 per daemon), including real requested/uncaptured/silent log windows with warm counters1..4. Post-run owned resource queries are empty. Observed-stream timing limits remain explicit; B05 and final Executor/provider/base64 response wiring remain mandatory before M0. | | 1.3 post-bootstrap initialization | Implemented/reviewed; full M0 remains open | Shared bounded InitError classification preserves initialization failure after an abandoned BootstrapReady poll, including an offered-but-unconsumed first event. Four actual session/HTTP/relay/client/worker composition regressions prove Idle/Busy32KiB boundaries, no active reply before a scripted known-container absence barrier, redaction and joined IO/API retirement. Independent read-only review has no findings. Environment target25 cases, workspace2,277 tests/407 separately gated, strict boundary1,062 unit+doc passes/13 Docker-gated; build/test/fmt/Clippy/rustdoc and strict production checks pass. Evidence: `/tmp/rustack-workflows-b05-{environments,build,tests,fmt,clippy,docs,boundaries}.log`. B05 closed; this is not production Executor/startup or actual Docker acceptance. | | 1.3 supervised pool invocation | Implemented/reviewed; Executor consumer remains required | §4.5/D35 expose synchronous start, cancel-safe response wait and separate resumable cancel_and_wait with an independent cleanup deadline. Per-call settlement senders stay beside Retirement records above JoinSet through worker/root recovery. Nine new regressions,27 total pool cases, prove delayed/expired cleanup, waiter loss, no-allocation rejection, worker/root failure, warm success and missing-producer uncertainty. W86 observer rejection cleanup is fixed; final independent review has no remaining finding. Workspace2,286 tests/408 separately gated; strict boundary870 units+204 docs (1,074 total)/14 Docker-gated; standard build/test/fmt/Clippy/rustdoc and strict production checks pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes7 owner/pool real-image cases, including per-call awaited retirement before global shutdown, container404 and replacement counter1 while admission stays open. Post-run owned containers/internal aliases are absent. This does not close Executor/provider/startup or M0 acceptance. | -| 1.4, P2–P5 | Pending | Remain required; no production Docker executor/provider or workflow-engine acceptance is claimed yet. | +| 1.3 Docker Executor/provider | Implemented/reviewed under trusted runtime injection; production startup remains | §4.6/D36 consume one PoolInvocation with independent cleanup settlement, immutable plan/metadata validation, raw outcomes, resolved version, requested alias ARN and one base64 tail encoding. W87–W91 are fixed/rechecked.15 Executor +16 handler cases pass, including held reserved concurrency, expired cleanup/failed quiescence and resumed physical retirement. Workspace2,313 tests/409 separately gated; core611 units/141 docs; strict boundary897 units+208 docs (1,105 total)/15 Docker-gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU daemon passes actual boto3→HTTP→provider→Docker deploy/publish/alias/Invoke/error/timeout/replacement plus seven existing owner/pool cases (eight per daemon). Controlled fixture readiness is not startup/probe/native-x86 evidence; post-run owned containers/internal aliases are absent. | +| 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. Next compose the real executor consuming the retained deployment manager, typed settlement, ImageLeaseManager and qualified wire/client/relay/HTTP/session contracts. Complete historical-effect reconciliation, allowed pulls and remaining network/probe consumers, followed by task1.4 startup/configuration and acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Next implement the production startup coordinator: historical uncertain-effect reconciliation, allowed pulls and network/resource probe consumers, then task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 34606dce..bc07e24b 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -411,6 +411,28 @@ The final owner/pool suite explicitly passes7 actual-image cases with zero ignor Production Executor/provider composition must consume this cancellation settlement before releasing its work permits. Startup reconciliation, allowed pulls, network/resource probes, M0 SDK acceptance and P2–P5 remain required. The receipt alone does not enable Docker or complete M0. +## P1 task1.3 Docker Executor/provider composition review + +The §4.6/D36 Executor now composes actual image execution through the retained pool: pure captured-plan/metadata validation, trusted readiness injection, original request/response bytes, context UUID/deadline propagation, resolved-version versus requested-alias metadata, synchronous log capture and independent cancellation settlement. A retained CleanupTask owns pool shutdown after dropped/expired waiters or Executor Drop. Controlled fixture readiness is not production reconciliation/probe evidence; ordinary CLI Docker selection remains unavailable. + +Independent read-only review and final rechecks have no remaining in-slice finding. The following corrections are closed in this source: + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W87 | P2; `crates/rustack-lambda-core/src/handler.rs`, Invoke scalar headers | HeaderMap::get silently selected the first repeated enum value. Require zero/one value through get_all before strict enum parsing | Six duplicate order/invalid-second cases plus six malformed/non-ASCII enum cases reject before lookup/admission with static errors | +| W88 | P2; `crates/rustack-lambda-core/src/executor/error.rs`, Busy classification | Initial spec prose conflated admission quota with lower-level wire saturation, which may occur after handler side effects. Refine the uncommitted §4.6/D36 contract: pool/environment admission Busy maps429; nested wire Busy preserves a typed infrastructure500 without replay | Direct and nested category/source regressions pin the distinction; no post-effect error is claimed to prove pre-execution rejection | +| W89 | P3; actual SDK fixture runtime metadata | Version headers did not prove the requested alias reached the RIC. Add context.invoked_function_arn to the optional real handler policy report | Exact :prod and :$LATEST assertions pass through boto3/HTTP/provider/Executor/relay/RIC alongside resolved ExecutedVersion1 | +| W90 | P3; `crates/rustack-lambda-core/src/executor/mod.rs`, conditional documentation | An unconditional intra-doc link targeted a cfg(unix) export. Use plain backticks | Final read-only source confirmation closes portability warning without widening backend support | +| W91 | P2; actual SDK negative evidence, provider/handler log request wiring | The first real SDK run found LogResult absent because HTTP LogType was never captured into InvokeRequest. Add provider.invoke_with_logs, preserve invoke's no-tail default and carry the synchronous flag through admission | Initial SDK run failed on both daemons; corrected final runs require actual stdout/stderr tail, empty silent tail, absent None header and unchanged FunctionError body | + +`make test-lambda-executor` passes31 cases:15 Executor tests plus16 handler cases (27 total additions over the prior baseline). The WorkSupervisor regression proves reserved concurrency remains occupied while Docker retirement is held behind an explicit barrier. Its expired case returns CleanupFailed from both public shutdown and invocation cleanup, rejects clean quiescence, then releases and joins physical ownership and resumes cached executor shutdown successfully. Binary outcomes/tails, static redacted typed errors, invalid scope/backend/ARN/payload/readiness/pins, warm metadata and malformed/duplicate headers are covered. No dependency/version or new advisory exception was introduced. + +Final-source workspace build/test/fmt/Clippy/rustdoc pass with2,313 tests, zero failures and409 separately resource-gated cases. Lambda core has611 unit tests/141 doctests. The six-library strict boundary target passes897 unit tests plus208 doctests (1,105 total),15 separately gated Docker cases and all strict production checks. Audit scans474 dependencies without vulnerabilities; deny advisories/bans/licenses/sources pass with existing unmatched/duplicate-policy notices. Evidence: `/tmp/rustack-workflows-executor-verified-{build,tests,fmt,clippy,docs,boundaries}.log`, focused log `/tmp/rustack-workflows-executor-focused-settlement.log`, and `executor-{audit,deny}.log` under the same prefix. + +Each retained Engine29.5.2 daemon explicitly passes one real boto3/HTTP/provider/Executor image scenario and seven owner/pool cases, zero ignored (eight selected cases per arm64 VZ and amd64/QEMU daemon). The SDK scenario deploys→Active→publishes1→aliases prod, computes42 with warm counters1/2/3, preserves raw ValueError/Unhandled, retires/replaces after FunctionError, preserves published environment across latest updates, proves DryRun does not execute, and reports a captured1s handler timeout followed by replacement counter1. Exact runtime alias ARN and resolved version remain distinct. A shared explicit-daemon fixture preserves the previous owner/pool tests. Post-run scoped queries find no owner/test-labeled containers or internal image aliases; user images remain cached. Logs: `/tmp/rustack-workflows-executor-verified-{arm64,amd64}.log`. QEMU is not native-x86 platform acceptance. + +M0 remains open: the actual production startup coordinator must implement historical uncertain-effect reconciliation, allowed pulls, network/resource probes, configuration/capability publication and complete platform acceptance. P2–P5/ASL remain required by the full objective. A trusted test readiness adapter must never be presented as those missing proofs. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 20d50b15..81bc343a 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -318,3 +318,13 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: upper-layer admission can await a positive reusable/retired proof without owning or aborting lower-layer resources. Failed supervision remains uncertain even after physical recovery; queued no-allocation rejection can settle cleanly. - **Pinned by**: [runtime §4.5](./rustack-workflows-20-lambda-runtime-design.md#45-supervised-pool-invocation-receipt), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. This receipt does not enable production Docker or complete Executor/provider integration. + + +## D36 — Compose Docker execution through the retained pool and trusted readiness port + +- **Context**: the Lambda provider already captures immutable revisions and supervises invocation permits; the Docker environment/pool owners now expose reusable/retired settlement proof. +- **Alternatives considered**: HTTP loopback invocation duplicates protocol/admission; a per-call Docker client loses bounded ownership; advertising readiness from a connected socket or a directory lease bypasses required startup qualification; rewriting handler outcomes into synthetic JSON breaks byte fidelity. +- **Decision**: implement the existing Executor using one pool call and independent settlement, with raw outcomes/resolved-version headers and one base64 tail encoding; inject a bounded synchronous readiness port from the trusted startup coordinator. +- **Why**: actual code execution and provider response composition can be verified independently while ordinary CLI selection remains unavailable until all production reconciliation/probes are implemented. The readiness interface shares the existing trusted Rust custom-Executor boundary and is never supplied through AWS requests. Pool/environment quota rejection is distinct from Docker wire saturation that may occur after side effects; only the former maps to429, while both retain no-replay behavior. +- **Pinned by**: [runtime §4.6](./rustack-workflows-20-lambda-runtime-design.md#46-docker-executor-and-provider-composition), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. No new dependency is required; this does not close M0 startup/platform acceptance. diff --git a/tests/workflows/docker_lambda_sdk.py b/tests/workflows/docker_lambda_sdk.py new file mode 100644 index 00000000..8cc77577 --- /dev/null +++ b/tests/workflows/docker_lambda_sdk.py @@ -0,0 +1,116 @@ +"""Real boto3 -> Lambda HTTP/provider -> Docker Executor composition evidence. + +The Rust harness supplies controlled readiness for a fresh explicit-daemon scope. +This does not qualify the unfinished production startup coordinator. +""" + +import base64 +import json +import sys + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +def main(endpoint, image, architecture): + client = boto3.client( + "lambda", endpoint_url=endpoint, region_name="us-east-1", + aws_access_key_id="test", aws_secret_access_key="test", + config=Config(connect_timeout=10, read_timeout=120, retries={"max_attempts": 0}), + ) + name = "executor-image-fixture" + created = False + try: + client.create_function( + FunctionName=name, PackageType="Image", Code={"ImageUri": image}, + Architectures=[architecture], MemorySize=192, Timeout=30, + Role="arn:aws:iam::000000000000:role/lambda-test", + Environment={"Variables": {"TEST_VALUE": "version-one"}}, + ) + created = True + client.get_waiter("function_active_v2").wait( + FunctionName=name, WaiterConfig={"Delay": 1, "MaxAttempts": 120}, + ) + version = client.publish_version(FunctionName=name)["Version"] + assert version == "1" + client.create_alias(FunctionName=name, Name="prod", FunctionVersion=version) + + def invoke(event, qualifier="prod", logs="Tail", failed=False): + response = client.invoke( + FunctionName=name, Qualifier=qualifier, + Payload=json.dumps(event).encode(), LogType=logs, + ) + assert response["StatusCode"] == 200, response + assert response["ExecutedVersion"] == (version if qualifier == "prod" else qualifier) + with response["Payload"] as body: + raw = body.read(6 * 1024 * 1024 + 1) + assert len(raw) <= 6 * 1024 * 1024 + assert response.get("FunctionError") == ("Unhandled" if failed else None) + tail = response.get("LogResult") + if logs == "Tail": + assert tail is not None + tail = base64.b64decode(tail, validate=True) + assert len(tail) <= 4096 + else: + assert tail is None + return json.loads(raw), tail + + first, tail = invoke({"value": 35, "logs": True, "policy": True, "response_bytes": 1024 * 1024}) + assert first["computed"] == 42 and first["counter"] == 1 + assert first["functionVersion"] == version + assert first["functionName"] == name + assert first["invokedArn"] == f"arn:aws:lambda:us-east-1:000000000000:function:{name}:prod" + assert first["configuredValue"] == "version-one" + assert b"WIRE_STDOUT_CAPTURE" in tail and b"WIRE_STDERR_CAPTURE" in tail + second, tail = invoke({"value": 1}) + assert second["counter"] == 2 and tail == b"" + third, tail = invoke({"value": 2, "logs": True, "response_bytes": 1024 * 1024}, logs="None") + assert third["counter"] == 3 and tail is None + failure, _ = invoke({"fail": True}, failed=True) + assert failure["errorType"] == "ValueError" + assert "intentional P0 failure" in failure["errorMessage"] + replacement, _ = invoke({"value": 3}) + assert replacement["counter"] == 1 + + client.update_function_configuration( + FunctionName=name, Environment={"Variables": {"TEST_VALUE": "version-two"}}, + ) + client.get_waiter("function_updated_v2").wait( + FunctionName=name, WaiterConfig={"Delay": 1, "MaxAttempts": 120}, + ) + latest, _ = invoke({"value": 4, "policy": True}, qualifier="$LATEST") + assert latest["configuredValue"] == "version-two" and latest["counter"] == 1 + assert latest["functionVersion"] == "$LATEST" + assert latest["invokedArn"] == f"arn:aws:lambda:us-east-1:000000000000:function:{name}:$LATEST" + dry = client.invoke(FunctionName=name, Qualifier="$LATEST", InvocationType="DryRun") + assert dry["StatusCode"] == 204 + after_dry, _ = invoke({"value": 5}, qualifier="$LATEST") + assert after_dry["counter"] == 2 + old, _ = invoke({"value": 6, "policy": True}) + assert old["configuredValue"] == "version-one" and old["functionVersion"] == version + assert old["invokedArn"] == f"arn:aws:lambda:us-east-1:000000000000:function:{name}:prod" + client.update_function_configuration(FunctionName=name, Timeout=1) + client.get_waiter("function_updated_v2").wait( + FunctionName=name, WaiterConfig={"Delay": 1, "MaxAttempts": 120}, + ) + try: + invoke({"value": 7, "sleep": True}, qualifier="$LATEST") + except ClientError as error: + assert error.response["Error"]["Code"] == "ServiceException" + assert "timed out after 1s" in error.response["Error"]["Message"] + else: + raise AssertionError("handler timeout returned successful output") + after_timeout, _ = invoke({"value": 8}, qualifier="$LATEST") + assert after_timeout["counter"] == 1 + print(json.dumps({"computed": first["computed"], "version": version, + "warmCounters": [first["counter"], second["counter"], third["counter"]], + "functionError": failure["errorType"], "replacementCounter": replacement["counter"], "timeoutReplacementCounter": after_timeout["counter"]})) + finally: + if created: + client.delete_function(FunctionName=name) + client.close() + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/tools/workflows-spikes/handler.py b/tools/workflows-spikes/handler.py index e24b2a1c..9cba847a 100644 --- a/tools/workflows-spikes/handler.py +++ b/tools/workflows-spikes/handler.py @@ -48,6 +48,7 @@ def handler(event, context): response["taskRoot"] = os.environ.get("LAMBDA_TASK_ROOT") response["functionName"] = os.environ.get("AWS_LAMBDA_FUNCTION_NAME") response["functionVersion"] = os.environ.get("AWS_LAMBDA_FUNCTION_VERSION") + response["invokedArn"] = context.invoked_function_arn response["region"] = os.environ.get("AWS_REGION") response["configuredValue"] = os.environ.get("TEST_VALUE") return response From d766ea8e43466a9becfff8d58bb49522e75ff233 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 04:33:05 -0700 Subject: [PATCH 030/106] phase 1: qualify Docker networks and actual kernel resource limits Pin exact local user bridges by full ID and run a private probe mode from the operator-pinned runtime image through retained EnvironmentManager ownership. Validate bounded kernel observations, private cgroup namespaces and current membership, exact quotas and isolation flags; issue scope-bound proof only after retirement and final deadline checks. D37 and runtime section 4.7 specify the contract. Fifty-seven focused cases, standard workspace gates, strict boundaries and audit/deny pass. Both retained daemons pass seventeen actual probe/SDK/pool/wire/runtime cases, with final host probe rechecks and owned-resource cleanup. Independent review findings W92-W94 are closed. Historical effects, pulls, endpoint/coordinator wiring and full P1-P5 acceptance remain required. --- Makefile | 11 + .../src/docker/environment/mod.rs | 3 + crates/rustack-lambda-core/src/docker/mod.rs | 8 + .../rustack-lambda-core/src/docker/network.rs | 146 +++++++ .../src/docker/network/tests.rs | 86 ++++ crates/rustack-lambda-core/src/docker/plan.rs | 19 +- .../src/docker/plan_tests.rs | 4 + .../rustack-lambda-core/src/docker/probe.rs | 378 ++++++++++++++++++ .../src/docker/probe/tests.rs | 160 ++++++++ .../rustack-lambda-core/tests/docker_probe.rs | 112 ++++++ crates/rustack-lambda-runtime/Cargo.toml | 2 +- crates/rustack-lambda-runtime/src/lib.rs | 1 + .../src/probe/client.rs | 157 ++++++++ .../src/probe/collect.rs | 102 +++++ .../rustack-lambda-runtime/src/probe/mod.rs | 140 +++++++ .../rustack-lambda-runtime/src/probe/parse.rs | 214 ++++++++++ .../src/probe/parse/tests.rs | 83 ++++ crates/rustack-lambda-runtime/src/process.rs | 11 + specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 32 +- .../rustack-workflows-60-operations-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 2 + specs/rustack-workflows-90-impl-plan.md | 5 +- specs/rustack-workflows-93-review.md | 18 + specs/rustack-workflows-99-decisions.md | 10 + 25 files changed, 1704 insertions(+), 12 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/network.rs create mode 100644 crates/rustack-lambda-core/src/docker/network/tests.rs create mode 100644 crates/rustack-lambda-core/src/docker/probe.rs create mode 100644 crates/rustack-lambda-core/src/docker/probe/tests.rs create mode 100644 crates/rustack-lambda-core/tests/docker_probe.rs create mode 100644 crates/rustack-lambda-runtime/src/probe/client.rs create mode 100644 crates/rustack-lambda-runtime/src/probe/collect.rs create mode 100644 crates/rustack-lambda-runtime/src/probe/mod.rs create mode 100644 crates/rustack-lambda-runtime/src/probe/parse.rs create mode 100644 crates/rustack-lambda-runtime/src/probe/parse/tests.rs diff --git a/Makefile b/Makefile index a5afd920..d45e8239 100644 --- a/Makefile +++ b/Makefile @@ -909,6 +909,17 @@ test-lambda-executor-docker: lambda-runtime-image lambda-runtime-fixture-image .PHONY: test-lambda-executor test-lambda-executor-docker +# The bridge lane requires an explicit pre-provisioned operator network. +test-lambda-probes: + @cargo test -p rustack-lambda-runtime --lib probe:: + @cargo test -p rustack-lambda-core --lib docker::network::tests + @cargo test -p rustack-lambda-core --lib docker::probe::tests + +test-lambda-probes-docker: lambda-runtime-image lambda-runtime-fixture-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" RUSTACK_RUNTIME_NETWORK="$(WORKFLOWS_NETWORK)" cargo test -p rustack-lambda-core --test docker_probe -- --ignored --test-threads=1 + +.PHONY: test-lambda-probes test-lambda-probes-docker + # Delivery clocks, protocol-v2 correlation and actual Native process retirement. test-lambda-timeouts: @cargo test -p rustack-lambda-runtime --lib diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index 3658c603..117fdca4 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -404,6 +404,9 @@ impl fmt::Debug for EnvironmentPort { } } impl EnvironmentPort { + pub(super) fn same_service(&self, other: &Self) -> bool { + self.sender.same_channel(&other.sender) + } pub(super) fn same_scope(&self, directory: &DirectoryLease) -> bool { self.directory.owner_id() == directory.owner_id() && self.directory.run_id() == directory.run_id() diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 57dec94b..86e7e316 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -11,10 +11,12 @@ mod image; mod lease; mod logs; mod mutation; +mod network; mod ownership; mod pin; mod plan; mod pool; +mod probe; mod types; use std::{ @@ -49,11 +51,16 @@ use models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, ContainerSummary, ContainerWaitResponse, ImageInspect, SystemInfo, SystemVersion, }; +pub use network::QualifiedNetwork; pub use ownership::{DirectoryLease, DirectoryLeaseError}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; pub use pin::{ImagePin, ImagePinScope}; pub use plan::{ContainerPolicy, EnvironmentPlan, NetworkSelection, RelayBudgets}; pub use pool::{EnvironmentPool, PoolConfig, PoolInvocation, PoolPort, PoolSettings, PoolStatus}; +pub use probe::{ + EnvironmentProbe, EnvironmentProbeConfig, EnvironmentProbeError, ProbeFailure, ProbeSettings, + probe_environment, +}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; @@ -76,6 +83,7 @@ const MAX_BODY_BYTES: usize = 1024 * 1024; enum Operation { Version, Info, + Network, Image, TagPin, InspectPin, diff --git a/crates/rustack-lambda-core/src/docker/network.rs b/crates/rustack-lambda-core/src/docker/network.rs new file mode 100644 index 00000000..90185e61 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/network.rs @@ -0,0 +1,146 @@ +//! Read-only network qualification pins a bridge ID before allocation. + +use bytes::Bytes; +use http::{Method, StatusCode}; +use tokio::time::Instant; + +use super::{ + ContainerPolicy, DockerError, DockerKey, DockerPort, NetworkSelection, Operation, + check_deadline, decode, escaped, models::NetworkInspect, +}; + +/// Validated disconnected or full-ID local bridge selection; not startup readiness. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct QualifiedNetwork { + requested: NetworkSelection, + selected: NetworkSelection, +} +impl QualifiedNetwork { + /// Bind the inspected network ID into a matching operator policy. + /// + /// # Errors + /// Rejects a policy for a different originally requested/qualified network. + /// + /// ```no_run + /// # fn bind(network:&rustack_lambda_core::docker::QualifiedNetwork, policy:&rustack_lambda_core::docker::ContainerPolicy) { + /// let fixed=network.apply(policy); + /// # } + /// ``` + pub fn apply(&self, policy: &ContainerPolicy) -> Result { + if policy.network != self.requested && policy.network != self.selected { + return Err(DockerError::Configuration); + } + let mut policy = policy.clone(); + policy.network = self.selected.clone(); + Ok(policy) + } + /// Whether the selected namespace deliberately has no external network. + /// + /// ```no_run + /// # fn check(network:&rustack_lambda_core::docker::QualifiedNetwork) { + /// let disconnected=network.disconnected(); + /// # } + /// ``` + #[must_use] + pub fn disconnected(&self) -> bool { + self.selected.value() == "none" + } +} +impl NetworkSelection { + /// Inspect and pin an exact operator-owned local user bridge, or select none. + /// + /// No create/delete or host/default-network fallback occurs. A qualified + /// bridge can still be deleted by the operator; allocation then fails. + /// + /// # Errors + /// Rejects expired budgets, wire failure, incomplete/builtin/nonlocal network + /// metadata, partial IDs or conflicting returned identity. + /// + /// ```no_run + /// # async fn qualify(port:&rustack_lambda_core::docker::DockerPort)->Result<(),rustack_lambda_core::docker::DockerError>{ + /// let selection:rustack_lambda_core::docker::NetworkSelection="worker-network".parse()?; + /// let network=selection.qualify(port,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await?; + /// # Ok(()) } + /// ``` + pub async fn qualify( + &self, + port: &DockerPort, + deadline: Instant, + ) -> Result { + check_deadline(deadline)?; + let selected = if self.value() == "none" { + self.clone() + } else { + let inspected = port + .inspect_network(&self.value().parse()?, deadline) + .await?; + validate(self, &inspected)? + }; + check_deadline(deadline)?; + Ok(QualifiedNetwork { + requested: self.clone(), + selected, + }) + } +} +impl DockerPort { + /// Inspect bounded network metadata without mutating its lifetime. + /// + /// # Errors + /// Returns wire admission, status, deadline or bounded metadata failure. + /// + /// ```no_run + /// # async fn inspect(port:&rustack_lambda_core::docker::DockerPort,key:&rustack_lambda_core::docker::DockerKey){ + /// let network=port.inspect_network(key,tokio::time::Instant::now()+std::time::Duration::from_secs(5)).await; + /// # } + /// ``` + pub async fn inspect_network( + &self, + key: &DockerKey, + deadline: Instant, + ) -> Result { + decode( + self.call( + Operation::Network, + Method::GET, + format!("/networks/{}", escaped(key.as_str())), + Bytes::new(), + StatusCode::OK, + deadline, + ) + .await?, + deadline, + ) + } +} +fn validate( + requested: &NetworkSelection, + network: &NetworkInspect, +) -> Result { + let name = network.name.as_deref().ok_or(DockerError::Configuration)?; + name.parse::()?; + let id = network.id.as_deref().ok_or(DockerError::Configuration)?; + let default = network + .options + .as_ref() + .and_then(|options| options.get("com.docker.network.bridge.default_bridge")); + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || !matches!(requested.value(),value if value==name||value==id) + || matches!(name, "host" | "bridge" | "none" | "default") + || network.driver.as_deref() != Some("bridge") + || network.scope.as_deref() != Some("local") + || network.ingress != Some(false) + || network.config_only != Some(false) + || default.is_some_and(|value| value != "false") + { + return Err(DockerError::Configuration); + } + id.parse() +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/network/tests.rs b/crates/rustack-lambda-core/src/docker/network/tests.rs new file mode 100644 index 00000000..57c2c80b --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/network/tests.rs @@ -0,0 +1,86 @@ +use serde_json::{Value, json}; +use tokio::time::{Duration, Instant}; + +use super::*; +use crate::docker::{plan::tests::policy, tests::Daemon}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(3) +} +fn fixture() -> Value { + json!({"Name":"probe-network","Id":"a".repeat(64),"Driver":"bridge","Scope":"local","Ingress":false,"ConfigOnly":false,"Options":{}}) +} +#[rstest::rstest] +#[case::driver("Driver",json!("host"))] +#[case::scope("Scope",json!("swarm"))] +#[case::ingress("Ingress",json!(true))] +#[case::config_only("ConfigOnly",json!(true))] +#[case::missing_ingress("Ingress", Value::Null)] +#[case::missing_scope("Scope", Value::Null)] +#[case::wrong_name("Name",json!("another-network"))] +#[case::builtin("Name",json!("bridge"))] +#[case::bad_id("Id",json!("a".repeat(63)))] +#[case::uppercase_id("Id",json!("A".repeat(64)))] +#[case::default_bridge("Options",json!({"com.docker.network.bridge.default_bridge":"true"}))] +#[case::unknown_default("Options",json!({"com.docker.network.bridge.default_bridge":"unknown"}))] +#[test] +fn test_should_reject_unqualified_network_metadata(#[case] field: &str, #[case] value: Value) { + let mut network = fixture(); + network[field] = value; + let network = serde_json::from_value(network).unwrap(); + assert!(validate(&"probe-network".parse().unwrap(), &network).is_err()); +} +#[test] +fn test_should_pin_only_exact_names_or_full_ids_and_bind_matching_policy() { + let network = serde_json::from_value(fixture()).unwrap(); + let requested: NetworkSelection = "probe-network".parse().unwrap(); + let selected = validate(&requested, &network).unwrap(); + assert_eq!(selected.value(), "a".repeat(64)); + assert!(validate(&"aaaa".parse().unwrap(), &network).is_err()); + assert!(validate(&"a".repeat(64).parse().unwrap(), &network).is_ok()); + let qualified = QualifiedNetwork { + requested, + selected, + }; + let mut configured = policy(); + assert!(qualified.apply(&configured).is_err()); + configured.network = "probe-network".parse().unwrap(); + let fixed = qualified.apply(&configured).unwrap(); + assert_eq!(fixed.network.value(), "a".repeat(64)); + assert!(qualified.apply(&fixed).is_ok()); + assert!(!qualified.disconnected()); +} +#[tokio::test] +async fn test_should_issue_only_bounded_network_inspect_and_no_request_for_none() { + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let task = tokio::spawn(async move { + "probe-network" + .parse::() + .unwrap() + .qualify(&port, deadline()) + .await + }); + let request = daemon.next().await; + assert!( + request + .head + .starts_with("GET /v1.53/networks/probe%2Dnetwork ") + ); + request + .reply(200, &serde_json::to_vec(&fixture()).unwrap()) + .await; + assert!(!task.await.unwrap().unwrap().disconnected()); + let none = "none" + .parse::() + .unwrap() + .qualify(&wire.port(), deadline()) + .await + .unwrap(); + assert!(none.disconnected()); + assert!(none.apply(&policy()).is_ok()); + assert!(daemon.requests.try_recv().is_err()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs index 734fec84..a0168caa 100644 --- a/crates/rustack-lambda-core/src/docker/plan.rs +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -18,7 +18,8 @@ use super::{ DirectoryLease, DockerError, DockerKey, DockerPort, models::{ ContainerCreateBody, ContainerCreateResponse, ContainerInspectResponse, HealthConfig, - HostConfig, HostConfigLogConfig, RestartPolicy, RestartPolicyNameEnum, + HostConfig, HostConfigCgroupnsModeEnum, HostConfigLogConfig, RestartPolicy, + RestartPolicyNameEnum, }, mutation::MutationGuard, }; @@ -38,9 +39,14 @@ pub(super) enum ContainerRole { } /// Operator network selection; a bridge still requires daemon startup validation. -#[derive(Clone, Serialize)] +#[derive(Clone, PartialEq, Eq, Serialize)] #[non_exhaustive] pub struct NetworkSelection(String); +impl NetworkSelection { + pub(super) fn value(&self) -> &str { + &self.0 + } +} impl fmt::Debug for NetworkSelection { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("NetworkSelection()") @@ -61,7 +67,7 @@ impl FromStr for NetworkSelection { } /// Integral relay session budgets independent of handler and deployment budgets. -#[derive(Debug, Clone, Copy, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[non_exhaustive] pub struct RelayBudgets { io: Duration, @@ -104,13 +110,13 @@ impl RelayBudgets { } /// Validated operator limits, independent of user function configuration. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[non_exhaustive] pub struct ContainerPolicy { - network: NetworkSelection, + pub(super) network: NetworkSelection, uid: u32, gid: u32, - nano_cpus: i64, + pub(super) nano_cpus: i64, relay: RelayBudgets, } impl ContainerPolicy { @@ -589,6 +595,7 @@ fn limits( ) -> Result { let memory = i64::try_from(memory).map_err(|_| DockerError::Configuration)?; Ok(HostConfig { + cgroupns_mode: Some(HostConfigCgroupnsModeEnum::PRIVATE), network_mode: Some(network), memory: Some(memory), memory_swap: Some(memory), diff --git a/crates/rustack-lambda-core/src/docker/plan_tests.rs b/crates/rustack-lambda-core/src/docker/plan_tests.rs index eadbddc1..a4b8df84 100644 --- a/crates/rustack-lambda-core/src/docker/plan_tests.rs +++ b/crates/rustack-lambda-core/src/docker/plan_tests.rs @@ -139,6 +139,10 @@ fn test_should_capture_fixed_images_identities_labels_and_complete_resource_poli ); assert_eq!(labels["rustack.lambda.role"], role); let limits = body.host_config.as_ref().unwrap(); + assert_eq!( + limits.cgroupns_mode, + Some(HostConfigCgroupnsModeEnum::PRIVATE) + ); assert_eq!(limits.memory, Some(i64::try_from(memory).unwrap())); assert_eq!(limits.memory_swap, limits.memory); assert_eq!(limits.oom_kill_disable, Some(false)); diff --git a/crates/rustack-lambda-core/src/docker/probe.rs b/crates/rustack-lambda-core/src/docker/probe.rs new file mode 100644 index 00000000..2d9e35bb --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe.rs @@ -0,0 +1,378 @@ +//! Actual resource qualification through the retained environment owner. + +use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; + +use bytes::Bytes; +use rustack_core::json::{JsonError, JsonLimits, from_slice}; +use rustack_lambda_model::types::ImageConfig; +use rustack_lambda_runtime::{ + metadata::local_trace_id, + probe::{REPORT_BYTES, ResourceReport}, + protocol::{Architecture as RuntimeArchitecture, AttemptId, Payload}, + session::{Invocation, RuntimeResult}, +}; +use thiserror::Error; +use tokio::time::Instant; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use super::{ + ContainerPolicy, DirectoryLease, EnvironmentError, EnvironmentPlan, EnvironmentPort, + QualifiedNetwork, check_deadline, +}; +use crate::{ + artifact::{Architecture, DeploymentArtifact, PinnedImage}, + executor::{ExecutionRevision, ExecutorBackend}, + identity::FunctionResourceId, +}; + +/// Specific bounded resource qualification mismatch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum ProbeFailure { + /// Wrong report schema, attempt or compiled architecture. + #[error("resource probe identity mismatch")] + Identity, + /// Controller files are not rooted at the current process membership. + #[error("resource probe cgroup namespace mismatch")] + Namespace, + /// Memory hard cap was not enforced. + #[error("resource probe memory mismatch")] + Memory, + /// Extra swap was permitted or unavailable. + #[error("resource probe swap mismatch")] + Swap, + /// PID cap was not enforced. + #[error("resource probe PID mismatch")] + Pids, + /// CPU quota/period was not enforced. + #[error("resource probe CPU mismatch")] + Cpu, + /// The actual process has a root UID/GID. + #[error("resource probe user mismatch")] + User, + /// Effective capabilities are not empty. + #[error("resource probe capability mismatch")] + Capabilities, + /// `no_new_privs` was not enforced. + #[error("resource probe privilege mismatch")] + Privilege, + /// No active seccomp filter. + #[error("resource probe seccomp mismatch")] + Seccomp, + /// The effective root mount is writable. + #[error("resource probe root filesystem mismatch")] + Root, + /// The effective tmp mount differs from the captured policy. + #[error("resource probe temporary filesystem mismatch")] + Temporary, + /// The trusted probe did not return a report. + #[error("resource probe execution failed")] + Outcome, +} +/// Probe failures preserve bounded classes and redacted typed sources. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum EnvironmentProbeError { + /// Invalid settings or mismatched retained runtime scope. + #[error("invalid environment probe configuration")] + Configuration, + /// Bounded strict report decoding failed. + #[error("invalid environment probe report")] + Report(#[from] JsonError), + /// Actual observations did not satisfy policy. + #[error("{0}")] + Rejected(#[from] ProbeFailure), + /// Retained allocation/invocation failed. + #[error("environment probe failed: {0}")] + Environment(#[from] EnvironmentError), + /// Cleanup did not confirm physical/IO settlement; lower owner retains it. + #[error("environment probe cleanup not confirmed")] + Cleanup(#[source] EnvironmentError), +} +impl fmt::Debug for EnvironmentProbeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// Validated positive whole-second startup and independent cleanup budgets. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub struct ProbeSettings { + start: Duration, + cleanup: Duration, +} +impl Default for ProbeSettings { + fn default() -> Self { + Self { + start: Duration::from_secs(60), + cleanup: Duration::from_secs(10), + } + } +} +impl ProbeSettings { + /// Validate both budgets in1..60 whole seconds. + /// + /// # Errors + /// Rejects zero, fractional or excessive durations. + /// + /// ``` + /// let settings=rustack_lambda_core::docker::ProbeSettings::new( + /// std::time::Duration::from_secs(30),std::time::Duration::from_secs(10))?; + /// # Ok::<(),rustack_lambda_core::docker::EnvironmentProbeError>(()) + /// ``` + pub fn new(start: Duration, cleanup: Duration) -> Result { + for duration in [start, cleanup] { + if duration.subsec_nanos() != 0 || !(1..=60).contains(&duration.as_secs()) { + return Err(EnvironmentProbeError::Configuration); + } + } + Ok(Self { start, cleanup }) + } +} +/// Operator-owned private probe inputs; no user image or endpoint is accepted. +#[derive(Debug, Clone, TypedBuilder)] +#[non_exhaustive] +pub struct EnvironmentProbeConfig { + /// Persistent namespace capability. + pub directory: DirectoryLease, + /// Pinned runtime image used for both relay and trusted probe function. + pub relay: PinnedImage, + /// Matching retained environment service. + pub environments: EnvironmentPort, + /// Operator policy, bound to the qualified network before allocation. + pub policy: ContainerPolicy, + /// Independent read-only network qualification. + pub network: QualifiedNetwork, + /// Validated startup/cleanup budgets. + #[builder(default)] + pub settings: ProbeSettings, +} +/// Private proof of actual matching kernel observations after joined retirement. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct EnvironmentProbe { + report: ResourceReport, + directory: DirectoryLease, + relay: PinnedImage, + policy: ContainerPolicy, + environments: EnvironmentPort, +} +impl EnvironmentProbe { + /// Inspect bounded observation data; it cannot reconstruct this proof. + /// + /// ```no_run + /// # fn inspect(probe:&rustack_lambda_core::docker::EnvironmentProbe){ + /// let memory=probe.report().memory_max; + /// # } + /// ``` + #[must_use] + pub fn report(&self) -> &ResourceReport { + &self.report + } + /// Check the exact retained inputs and accepting environment service. + /// + /// ```no_run + /// # fn matches(probe:&rustack_lambda_core::docker::EnvironmentProbe, inputs:&rustack_lambda_core::docker::EnvironmentProbeConfig){ + /// let current=probe.matches(inputs); + /// # } + /// ``` + #[must_use] + pub fn matches(&self, inputs: &EnvironmentProbeConfig) -> bool { + let metadata = self.relay.resolved().metadata(); + let other = inputs.relay.resolved().metadata(); + self.directory.owner_id() == inputs.directory.owner_id() + && self.directory.run_id() == inputs.directory.run_id() + && self.relay.is_live() + && inputs.relay.is_live() + && metadata.image_id == other.image_id + && metadata.inspection_hash == other.inspection_hash + && inputs + .network + .apply(&inputs.policy) + .is_ok_and(|policy| policy == self.policy) + && self.environments.same_service(&inputs.environments) + && self.environments.status().accepting + } +} + +/// Execute the pinned Rust probe once and issue proof only after actual retirement. +/// +/// This does not grant production readiness. The environment manager must outlive +/// a dropped/expired waiter, retaining uncertain container/IO cleanup. +/// +/// # Errors +/// Returns invalid inputs, runtime/wire failure, strict report mismatch or cleanup +/// uncertainty. A failed accepted probe is never replayed. +/// +/// ```no_run +/// # async fn probe(config:rustack_lambda_core::docker::EnvironmentProbeConfig){ +/// let evidence=rustack_lambda_core::docker::probe_environment(config, +/// tokio::time::Instant::now()+std::time::Duration::from_secs(60)).await; +/// # } +/// ``` +pub async fn probe_environment( + config: EnvironmentProbeConfig, + deadline: Instant, +) -> Result { + let deadline = deadline.min(Instant::now() + config.settings.start); + check_deadline(deadline).map_err(EnvironmentError::from)?; + if !config.environments.same_scope(&config.directory) || !config.relay.is_live() { + return Err(EnvironmentProbeError::Configuration); + } + let policy = config + .network + .apply(&config.policy) + .map_err(EnvironmentError::from)?; + let architecture = config.relay.resolved().metadata().architecture; + let revision = ExecutionRevision::builder() + .resource_id(FunctionResourceId::new()) + .revision_id(Uuid::new_v4().to_string()) + .backend(ExecutorBackend::Docker) + .function_name("rustack-runtime-probe".into()) + .qualifier("$LATEST".into()) + .runtime(None) + .handler(None) + .architecture(architecture) + .artifact(DeploymentArtifact::Image(config.relay.clone())) + .image_config(Some(ImageConfig { + entry_point: Some(vec!["/rustack-lambda-runtime".into(), "probe".into()]), + command: Some(vec![]), + working_directory: None, + })) + .environment(HashMap::from([ + ("AWS_REGION".into(), "us-east-1".into()), + ("AWS_DEFAULT_REGION".into(), "us-east-1".into()), + ])) + .timeout(Duration::from_secs(10)) + .memory_mb(128) + .ephemeral_storage_mb(512) + .build(); + let plan = EnvironmentPlan::new( + config.directory.clone(), + config.relay.clone(), + Arc::new(revision), + policy.clone(), + ) + .map_err(EnvironmentError::from)?; + let attempt = AttemptId::default(); + let call = Invocation::new( + attempt, + deadline, + "arn:aws:lambda:us-east-1:000000000000:function:rustack-runtime-probe" + .parse() + .map_err(|_| EnvironmentProbeError::Configuration)?, + local_trace_id().map_err(|_| EnvironmentProbeError::Configuration)?, + Payload::try_from(Bytes::from_static(b"{}")) + .map_err(|_| EnvironmentProbeError::Configuration)?, + ) + .map_err(|_| EnvironmentProbeError::Configuration)?; + let mut environment = config.environments.spawn(plan, deadline)?; + let result: Result = async { + environment.wait_ready(deadline).await?; + let response = environment.invoke(call, false).await?; + let RuntimeResult::Success(payload) = response.result else { + return Err(ProbeFailure::Outcome.into()); + }; + let report = decode_report(payload.as_ref())?; + verify(&report, attempt, architecture, &policy)?; + Ok(report) + } + .await; + environment + .shutdown(Instant::now() + config.settings.cleanup) + .await + .map_err(EnvironmentProbeError::Cleanup)?; + let report = result?; + if Instant::now() >= deadline { + return Err(EnvironmentError::Deadline.into()); + } + if !config.relay.is_live() || !config.environments.status().accepting { + return Err(EnvironmentProbeError::Configuration); + } + Ok(EnvironmentProbe { + report, + directory: config.directory, + relay: config.relay, + policy, + environments: config.environments, + }) +} +fn decode_report(bytes: &[u8]) -> Result { + Ok(from_slice( + bytes, + JsonLimits::new(REPORT_BYTES, 8, 128, 256, 32)?, + )?) +} +fn verify( + report: &ResourceReport, + attempt: AttemptId, + architecture: Architecture, + policy: &ContainerPolicy, +) -> Result<(), ProbeFailure> { + if report.version != 1 + || report.attempt != attempt + || report.architecture + != match architecture { + Architecture::Arm64 => RuntimeArchitecture::Arm64, + Architecture::X86_64 => RuntimeArchitecture::X86_64, + } + { + return Err(ProbeFailure::Identity); + } + if report.memory_max != Some(128 * 1024 * 1024) { + return Err(ProbeFailure::Memory); + } + if !report.cgroup_root { + return Err(ProbeFailure::Namespace); + } + if report.swap_max != Some(0) { + return Err(ProbeFailure::Swap); + } + if report.pids_max != Some(256) { + return Err(ProbeFailure::Pids); + } + let expected = u64::try_from(policy.nano_cpus) + .ok() + .and_then(|cpu| cpu.checked_mul(report.cpu_period)) + .map(|total| total / 1_000_000_000); + if !(1000..=1_000_000).contains(&report.cpu_period) + || !report + .cpu_quota + .is_some_and(|quota| (1000..=16_000_000).contains("a)) + || expected == Some(0) + || report.cpu_quota != expected + || expected.is_none() + { + return Err(ProbeFailure::Cpu); + } + if report.euid == 0 || report.egid == 0 { + return Err(ProbeFailure::User); + } + if report.cap_effective != 0 { + return Err(ProbeFailure::Capabilities); + } + if !report.no_new_privileges { + return Err(ProbeFailure::Privilege); + } + if report.seccomp_mode != 2 { + return Err(ProbeFailure::Seccomp); + } + if !report.root_readonly { + return Err(ProbeFailure::Root); + } + let temporary = &report.temporary; + if temporary.bytes != 512 * 1024 * 1024 + || temporary.mode != 0o1777 + || !temporary.tmpfs + || !temporary.nosuid + || !temporary.nodev + { + return Err(ProbeFailure::Temporary); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/probe/tests.rs b/crates/rustack-lambda-core/src/docker/probe/tests.rs new file mode 100644 index 00000000..d78555f6 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/probe/tests.rs @@ -0,0 +1,160 @@ +use rustack_lambda_runtime::{ + probe::TemporaryReport, protocol::Architecture as RuntimeArchitecture, +}; +use serde_json::{Value, json}; + +use super::*; +use crate::docker::{ + DockerWire, EnvironmentConfig, EnvironmentManager, RelayBudgets, + plan::tests::{image_with, policy}, + tests::Daemon, +}; + +fn report(attempt: AttemptId) -> ResourceReport { + ResourceReport::builder() + .version(1) + .attempt(attempt) + .architecture(RuntimeArchitecture::Arm64) + .cgroup_root(true) + .memory_max(Some(128 * 1024 * 1024)) + .swap_max(Some(0)) + .pids_max(Some(256)) + .cpu_quota(Some(100_000)) + .cpu_period(100_000) + .euid(1000) + .egid(1000) + .cap_effective(0) + .no_new_privileges(true) + .seccomp_mode(2) + .root_readonly(true) + .temporary( + TemporaryReport::builder() + .bytes(512 * 1024 * 1024) + .mode(0o1777) + .tmpfs(true) + .nosuid(true) + .nodev(true) + .build(), + ) + .build() +} +#[rstest::rstest] +#[case::version("version",json!(2),ProbeFailure::Identity)] +#[case::namespace("cgroupRoot",json!(false),ProbeFailure::Namespace)] +#[case::architecture("architecture",json!("x86_64"),ProbeFailure::Identity)] +#[case::memory("memoryMax",json!(134_217_729),ProbeFailure::Memory)] +#[case::unlimited_memory("memoryMax", Value::Null, ProbeFailure::Memory)] +#[case::swap("swapMax",json!(1),ProbeFailure::Swap)] +#[case::pids("pidsMax",json!(257),ProbeFailure::Pids)] +#[case::cpu("cpuQuota", Value::Null, ProbeFailure::Cpu)] +#[case::period("cpuPeriod",json!(0),ProbeFailure::Cpu)] +#[case::overflow("cpuPeriod",json!(u64::MAX),ProbeFailure::Cpu)] +#[case::root_uid("euid",json!(0),ProbeFailure::User)] +#[case::root_gid("egid",json!(0),ProbeFailure::User)] +#[case::capabilities("capEffective",json!(1),ProbeFailure::Capabilities)] +#[case::privileges("noNewPrivileges",json!(false),ProbeFailure::Privilege)] +#[case::seccomp("seccompMode",json!(0),ProbeFailure::Seccomp)] +#[case::rootfs("rootReadonly",json!(false),ProbeFailure::Root)] +#[case::tmp_size("temporary",json!({"bytes":1,"mode":1023,"tmpfs":true,"nosuid":true,"nodev":true}),ProbeFailure::Temporary)] +#[case::tmp_flags("temporary",json!({"bytes":536_870_912,"mode":1023,"tmpfs":true,"nosuid":false,"nodev":true}),ProbeFailure::Temporary)] +#[test] +fn test_should_reject_each_actual_policy_mismatch( + #[case] field: &str, + #[case] value: Value, + #[case] failure: ProbeFailure, +) { + let attempt = AttemptId::default(); + let mut report = serde_json::to_value(report(attempt)).unwrap(); + report[field] = value; + let decoded = decode_report(&serde_json::to_vec(&report).unwrap()).unwrap(); + assert_eq!( + verify(&decoded, attempt, Architecture::Arm64, &policy()), + Err(failure) + ); +} +#[test] +fn test_should_verify_exact_attempt_and_checked_cpu_rounding_and_bound_reports() { + let attempt = AttemptId::default(); + let mut observed = report(attempt); + verify(&observed, attempt, Architecture::Arm64, &policy()).unwrap(); + assert_eq!( + verify( + &observed, + AttemptId::default(), + Architecture::Arm64, + &policy() + ), + Err(ProbeFailure::Identity) + ); + let policy = ContainerPolicy::new( + "none".parse().unwrap(), + 1000, + 1000, + 333_333_333, + RelayBudgets::default(), + ) + .unwrap(); + observed.cpu_quota = Some(33_333); + verify(&observed, attempt, Architecture::Arm64, &policy).unwrap(); + assert!(decode_report(&vec![b' '; REPORT_BYTES + 1]).is_err()); + let mut raw = serde_json::to_value(&observed).unwrap(); + raw["unknown"] = json!(true); + assert!(decode_report(&serde_json::to_vec(&raw).unwrap()).is_err()); + assert!(decode_report(br#"{"version":1,"version":1}"#).is_err()); + assert!(ProbeSettings::new(Duration::ZERO, Duration::from_secs(1)).is_err()); + assert!(ProbeSettings::new(Duration::from_secs(1), Duration::from_millis(1500)).is_err()); +} +#[rstest::rstest] +#[case::rejected(false)] +#[case::abandoned(true)] +#[tokio::test] +async fn test_should_settle_rejected_probe_creation_and_retain_abandoned_waiter_cleanup( + #[case] abandoned: bool, +) { + let mut daemon = Daemon::new(); + let mut wire: DockerWire = daemon.wire().await; + let directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + let mut environments = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let network = "none" + .parse::() + .unwrap() + .qualify(&wire.port(), deadline) + .await + .unwrap(); + let config = EnvironmentProbeConfig::builder() + .directory(directory) + .relay(image_with("1000", vec![])) + .environments(environments.port()) + .policy(policy()) + .network(network) + .build(); + let mut foreign = config.clone(); + foreign.directory = DirectoryLease::testing(Uuid::new_v4(), Uuid::new_v4()); + assert!(matches!( + probe_environment(foreign, deadline).await, + Err(EnvironmentProbeError::Configuration) + )); + let task = tokio::spawn(probe_environment(config, deadline)); + let creation = daemon.next().await; + if abandoned { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(environments.port().status().allocated, 1); + creation.reply(400, b"{}").await; + } else { + creation.reply(400, b"{}").await; + assert!(matches!( + task.await.unwrap(), + Err(EnvironmentProbeError::Environment(_)) + )); + assert_eq!(environments.port().status().allocated, 0); + } + assert!(daemon.requests.try_recv().is_err()); + environments.shutdown(deadline).await.unwrap(); + assert_eq!(environments.port().status().allocated, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/tests/docker_probe.rs b/crates/rustack-lambda-core/tests/docker_probe.rs new file mode 100644 index 00000000..27f00a10 --- /dev/null +++ b/crates/rustack-lambda-core/tests/docker_probe.rs @@ -0,0 +1,112 @@ +//! Actual pinned Rust resource probe in disconnected and operator bridge networks. +#![cfg(unix)] +#![forbid(unsafe_code)] + +use std::{env, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use rustack_lambda_core::docker::{ + ContainerPolicy, EnvironmentProbeConfig, NetworkSelection, ProbeSettings, RelayBudgets, + probe_environment, +}; +use tokio::time::Instant; + +mod common; +use common::docker::Fixture; + +#[rstest::rstest] +#[case::disconnected(false)] +#[case::operator_bridge(true)] +#[tokio::test] +#[ignore = "requires explicit Docker runtime image/socket and operator probe bridge"] +async fn test_should_qualify_actual_kernel_limits_and_join_probe_retirement( + #[case] bridge: bool, +) -> Result<()> { + let mut fixture = Fixture::new().await?; + let result = async { + let selection: NetworkSelection = if bridge { + env::var("RUSTACK_RUNTIME_NETWORK") + .ok() + .filter(|value| !value.is_empty()) + .context("explicit operator probe bridge required")? + .parse()? + } else { + "none".parse()? + }; + let deadline = Instant::now() + Duration::from_secs(60); + let network = selection.qualify(&fixture.wire.port(), deadline).await?; + ensure!(network.disconnected() != bridge, "network mode differs"); + let policy = ContainerPolicy::new( + selection, + 1000, + 1000, + 333_333_333, + RelayBudgets::new( + Duration::from_secs(60), + Duration::from_secs(600), + Duration::from_secs(60), + )?, + )?; + let config = EnvironmentProbeConfig::builder() + .directory(fixture.lease.clone()) + .relay(fixture.relay.as_ref().context("relay pin")?.clone()) + .environments(fixture.owner.port()) + .policy(policy) + .network(network) + .settings(ProbeSettings::new( + Duration::from_secs(60), + Duration::from_secs(60), + )?) + .build(); + let evidence = probe_environment(config.clone(), deadline).await?; + ensure!( + evidence.matches(&config), + "fresh probe does not match retained inputs" + ); + ensure!( + fixture.owner.port().status().allocated == 0, + "proof issued before physical retirement" + ); + let report = evidence.report(); + ensure!( + report.memory_max == Some(128 * 1024 * 1024) + && report.swap_max == Some(0) + && report.pids_max == Some(256), + "kernel limits differ" + ); + ensure!( + report.cpu_quota == Some(33_333) && report.cpu_period == 100_000, + "CPU rounding differs" + ); + ensure!( + report.euid != 0 + && report.egid != 0 + && report.cap_effective == 0 + && report.no_new_privileges + && report.seccomp_mode == 2, + "privilege isolation differs" + ); + ensure!( + report.root_readonly + && report.temporary.bytes == 512 * 1024 * 1024 + && report.temporary.mode == 0o1777, + "filesystem isolation differs" + ); + let mut changed = config.clone(); + changed.policy = ContainerPolicy::new( + "none".parse()?, + 1000, + 1000, + 1_000_000_000, + RelayBudgets::default(), + )?; + ensure!( + !evidence.matches(&changed), + "proof matched a different policy" + ); + Ok(()) + } + .await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} diff --git a/crates/rustack-lambda-runtime/Cargo.toml b/crates/rustack-lambda-runtime/Cargo.toml index d0ec070e..6e2a013f 100644 --- a/crates/rustack-lambda-runtime/Cargo.toml +++ b/crates/rustack-lambda-runtime/Cargo.toml @@ -18,6 +18,7 @@ http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } typed-builder = { workspace = true } tokio = { workspace = true } @@ -30,5 +31,4 @@ uuid = { workspace = true } anyhow = { workspace = true } proptest = { workspace = true } rstest = { workspace = true } -serde_json = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/rustack-lambda-runtime/src/lib.rs b/crates/rustack-lambda-runtime/src/lib.rs index 3a097ecf..5a18232d 100644 --- a/crates/rustack-lambda-runtime/src/lib.rs +++ b/crates/rustack-lambda-runtime/src/lib.rs @@ -10,6 +10,7 @@ pub mod client; pub mod diagnostics; pub mod http; pub mod metadata; +pub mod probe; pub mod protocol; pub mod relay; pub mod session; diff --git a/crates/rustack-lambda-runtime/src/probe/client.rs b/crates/rustack-lambda-runtime/src/probe/client.rs new file mode 100644 index 00000000..52c39391 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/client.rs @@ -0,0 +1,157 @@ +//! One owned HTTP connection per request; never spawn a detached connection task. + +use std::{env, future::pending, num::NonZeroU64, sync::Arc, time::Duration}; + +use bytes::Bytes; +use http::{HeaderMap, Method, Request, StatusCode}; +use http_body_util::{BodyExt, Full, Limited}; +use hyper::client::conn::http1; +use hyper_util::rt::TokioIo; +use tokio::{ + net::TcpStream, + time::{Instant, sleep_until, timeout_at}, +}; + +use super::{ProbeError, REPORT_BYTES, collect}; +use crate::{metadata::deadline_from_unix_ms, protocol::AttemptId}; + +struct Reply { + status: StatusCode, + headers: HeaderMap, + body: Bytes, +} +async fn request( + method: Method, + path: &str, + body: Bytes, + deadline: Instant, +) -> Result { + let stream = timeout_at(deadline, TcpStream::connect("127.0.0.1:9001")) + .await + .map_err(|_| ProbeError::Deadline)??; + let mut builder = http1::Builder::new(); + builder.max_buf_size(16384).max_headers(32); + let (mut sender, connection) = timeout_at(deadline, builder.handshake(TokioIo::new(stream))) + .await + .map_err(|_| ProbeError::Deadline)??; + tokio::pin!(connection); + let operation = async { + let request = Request::builder() + .method(method) + .uri(path) + .header("host", "127.0.0.1:9001") + .header("connection", "close") + .header("content-type", "application/json") + .body(Full::new(body)) + .map_err(|_| ProbeError::Protocol)?; + let response = sender.send_request(request).await?; + let (parts, body) = response.into_parts(); + let body = Limited::new(body, REPORT_BYTES) + .collect() + .await + .map_err(|_| ProbeError::Protocol)? + .to_bytes(); + Ok(Reply { + status: parts.status, + headers: parts.headers, + body, + }) + }; + tokio::pin!(operation); + let mut finished = false; + loop { + tokio::select! {biased; + ()=sleep_until(deadline)=>return Err(ProbeError::Deadline), + result=&mut operation=>return result, + result=&mut connection,if !finished=>{result?;finished=true;}, + } + } +} +fn header<'a>(headers: &'a HeaderMap, key: &str, cap: usize) -> Result<&'a str, ProbeError> { + let mut values = headers.get_all(key).iter(); + let value = values.next().ok_or(ProbeError::Protocol)?; + if values.next().is_some() || value.as_bytes().len() > cap { + return Err(ProbeError::Protocol); + } + value.to_str().map_err(|_| ProbeError::Protocol) +} +pub(super) async fn run() -> Result<(), ProbeError> { + if !cfg!(target_os = "linux") + || env::var("AWS_LAMBDA_RUNTIME_API").as_deref() != Ok("127.0.0.1:9001") + { + return Err(ProbeError::Configuration); + } + let challenge = request( + Method::GET, + "/2018-06-01/runtime/invocation/next", + Bytes::new(), + Instant::now() + Duration::from_secs(60), + ) + .await?; + if challenge.status != StatusCode::OK || challenge.body.as_ref() != b"{}" { + return Err(ProbeError::Protocol); + } + let attempt: AttemptId = header(&challenge.headers, "lambda-runtime-aws-request-id", 36)? + .parse() + .map_err(|_| ProbeError::Protocol)?; + let wall = header(&challenge.headers, "lambda-runtime-deadline-ms", 20)? + .parse::() + .map_err(|_| ProbeError::Protocol)?; + let deadline = deadline_from_unix_ms(wall).map_err(|_| ProbeError::Deadline)?; + let (suffix, body) = match collect::collect(attempt, deadline).await { + Ok(report) => { + let bytes = + serde_json::to_vec(&report).map_err(|error| ProbeError::Report(Arc::new(error)))?; + if bytes.len() > REPORT_BYTES { + return Err(ProbeError::Protocol); + } + ("response", Bytes::from(bytes)) + } + Err(_) => ( + "error", + Bytes::from_static( + br#"{"errorType":"Rustack.ProbeFailed","errorMessage":"resource probe failed"}"#, + ), + ), + }; + let reply = request( + Method::POST, + &format!("/2018-06-01/runtime/invocation/{attempt}/{suffix}"), + body, + deadline.min(Instant::now() + Duration::from_secs(5)), + ) + .await?; + if reply.status != StatusCode::ACCEPTED || !reply.body.is_empty() { + return Err(ProbeError::Protocol); + } + // The host owns this one-shot function's retirement, including on rejection. + pending().await +} + +#[cfg(test)] +mod tests { + use http::HeaderValue; + + use super::*; + + #[test] + fn test_should_require_exact_single_bounded_ascii_probe_metadata() { + let mut values = HeaderMap::new(); + assert!(header(&values, "probe", 3).is_err()); + values.insert("probe", HeaderValue::from_static("123")); + assert_eq!(header(&values, "probe", 3).unwrap(), "123"); + assert!(header(&values, "probe", 2).is_err()); + values.append("probe", HeaderValue::from_static("456")); + assert!(header(&values, "probe", 3).is_err()); + values.clear(); + values.insert("probe", HeaderValue::from_bytes(b"\xff").unwrap()); + assert!(header(&values, "probe", 3).is_err()); + } + #[test] + fn test_should_redact_probe_error_sources() { + use std::error::Error as _; + let error = ProbeError::from(std::io::Error::other("private-kernel-path")); + assert!(error.source().is_some()); + assert!(!format!("{error:?} {error}").contains("private-kernel-path")); + } +} diff --git a/crates/rustack-lambda-runtime/src/probe/collect.rs b/crates/rustack-lambda-runtime/src/probe/collect.rs new file mode 100644 index 00000000..3b76e814 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/collect.rs @@ -0,0 +1,102 @@ +//! Fixed-path bounded async reads; process/container retirement owns cancellation. + +use std::{env::consts::ARCH, io, os::unix::fs::MetadataExt}; + +use tokio::{ + fs::{File, metadata}, + io::AsyncReadExt, + time::{Instant, timeout_at}, +}; + +use super::{ProbeError, ResourceReport, parse}; +use crate::protocol::{Architecture, AttemptId}; + +async fn read(path: &'static str, cap: usize) -> Result { + let mut bytes = Vec::new(); + File::open(path) + .await? + .take( + u64::try_from(cap) + .map_err(|_| ProbeError::Observation)? + .saturating_add(1), + ) + .read_to_end(&mut bytes) + .await?; + if bytes.len() > cap { + return Err(ProbeError::Observation); + } + String::from_utf8(bytes).map_err(|_| ProbeError::Observation) +} +pub(super) async fn collect( + attempt: AttemptId, + deadline: Instant, +) -> Result { + timeout_at(deadline, async { + let memory = read("/sys/fs/cgroup/memory.max", 128).await; + let (memory_max, swap_max, pids_max, cpu_quota, cpu_period, unified) = match memory { + Ok(memory) => { + let (quota, period) = parse::cpu(&read("/sys/fs/cgroup/cpu.max", 128).await?)?; + ( + parse::limit(&memory)?, + parse::limit(&read("/sys/fs/cgroup/memory.swap.max", 128).await?)?, + parse::limit(&read("/sys/fs/cgroup/pids.max", 128).await?)?, + quota, + period, + true, + ) + } + Err(ProbeError::Io(error)) if error.kind() == io::ErrorKind::NotFound => { + let memory = + parse::limit(&read("/sys/fs/cgroup/memory/memory.limit_in_bytes", 128).await?)?; + let total = parse::limit( + &read("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes", 128).await?, + )?; + let swap = match (memory, total) { + (Some(memory), Some(total)) => { + Some(total.checked_sub(memory).ok_or(ProbeError::Observation)?) + } + _ => None, + }; + ( + memory, + swap, + parse::limit(&read("/sys/fs/cgroup/pids/pids.max", 128).await?)?, + parse::limit(&read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", 128).await?)?, + parse::number(&read("/sys/fs/cgroup/cpu/cpu.cfs_period_us", 128).await?)?, + false, + ) + } + Err(error) => return Err(error), + }; + let status = parse::status(&read("/proc/self/status", 65536).await?)?; + let cgroup_root = parse::membership(&read("/proc/self/cgroup", 65536).await?, unified)?; + let mode = metadata("/tmp").await?.mode() & 0o7777; + let (root_readonly, temporary) = + parse::mounts(&read("/proc/self/mountinfo", 65536).await?, mode)?; + let architecture = match ARCH { + "aarch64" => Architecture::Arm64, + "x86_64" => Architecture::X86_64, + _ => return Err(ProbeError::Configuration), + }; + Ok(ResourceReport::builder() + .version(1) + .attempt(attempt) + .architecture(architecture) + .cgroup_root(cgroup_root) + .memory_max(memory_max) + .swap_max(swap_max) + .pids_max(pids_max) + .cpu_quota(cpu_quota) + .cpu_period(cpu_period) + .euid(status.uid) + .egid(status.gid) + .cap_effective(status.capabilities) + .no_new_privileges(status.no_new_privileges) + .seccomp_mode(status.seccomp) + .root_readonly(root_readonly) + .temporary(temporary) + .build()) + }) + .await + .map_err(|_| ProbeError::Deadline)? +} diff --git a/crates/rustack-lambda-runtime/src/probe/mod.rs b/crates/rustack-lambda-runtime/src/probe/mod.rs new file mode 100644 index 00000000..da35b9d6 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/mod.rs @@ -0,0 +1,140 @@ +//! Typed bounded kernel observations; qualification belongs to the host owner. + +#[cfg(unix)] +mod client; +#[cfg(unix)] +mod collect; +#[cfg(any(unix, test))] +mod parse; + +use std::{fmt, io, sync::Arc}; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use typed_builder::TypedBuilder; + +use crate::protocol::{Architecture, AttemptId}; + +/// Maximum encoded resource report/challenge response size. +pub const REPORT_BYTES: usize = 4096; + +/// Static probe errors never display kernel text or filesystem paths. +#[derive(Clone, Error)] +#[non_exhaustive] +pub enum ProbeError { + /// Unsupported platform, arguments or private API configuration. + #[error("invalid runtime probe configuration")] + Configuration, + /// Incomplete, duplicate, malformed or unsupported kernel observations. + #[error("invalid runtime probe observation")] + Observation, + /// The private API protocol/challenge did not match. + #[error("invalid runtime probe protocol")] + Protocol, + /// Absolute collection/request budget expired. + #[error("runtime probe deadline exceeded")] + Deadline, + /// Fixed local kernel/API IO failed. + #[error("runtime probe IO failed")] + Io(#[source] Arc), + /// Private Runtime API HTTP failed. + #[error("runtime probe HTTP failed")] + Http(#[source] Arc), + /// Typed report serialization failed. + #[error("runtime probe report failed")] + Report(#[source] Arc), +} +impl fmt::Debug for ProbeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl From for ProbeError { + fn from(error: io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} +impl From for ProbeError { + fn from(error: hyper::Error) -> Self { + Self::Http(Arc::new(error)) + } +} + +/// Observed container-local temporary filesystem, without mount source text. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct TemporaryReport { + /// Kernel-reported tmpfs size in bytes. + pub bytes: u64, + /// Actual permission/special mode bits. + pub mode: u32, + /// The mount filesystem is tmpfs. + pub tmpfs: bool, + /// The effective mount disallows setuid transitions. + pub nosuid: bool, + /// The effective mount disallows device interpretation. + pub nodev: bool, +} + +/// Finite/unlimited normalized controller and process observations. +/// +/// A deserialized report is untrusted data, never a readiness capability. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[non_exhaustive] +pub struct ResourceReport { + /// Report schema, currently1. + pub version: u8, + /// Consumed Runtime API attempt identity. + pub attempt: AttemptId, + /// Actual compiled Linux architecture. + pub architecture: Architecture, + /// Required controller memberships are rooted in this private namespace. + pub cgroup_root: bool, + /// Observed memory controller value; textual unlimited becomes None. Numeric + /// v1 unlimited sentinels retain their numeric value and cannot qualify. + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_max: Option, + /// Observed extra swap cap; textual unlimited becomes None. + #[serde(skip_serializing_if = "Option::is_none")] + pub swap_max: Option, + /// Process/thread hard cap; None means unlimited. + #[serde(skip_serializing_if = "Option::is_none")] + pub pids_max: Option, + /// CPU time quota in microseconds; None means unlimited. + #[serde(skip_serializing_if = "Option::is_none")] + pub cpu_quota: Option, + /// CPU scheduling period in microseconds. + pub cpu_period: u64, + /// Effective UID observed from proc status. + pub euid: u32, + /// Effective GID observed from proc status. + pub egid: u32, + /// Effective Linux capability bitmask. + pub cap_effective: u64, + /// Actual `no_new_privs` flag. + pub no_new_privileges: bool, + /// Actual seccomp mode (2 means filter mode). + pub seccomp_mode: u8, + /// Root's effective mount flags include read-only. + pub root_readonly: bool, + /// Actual /tmp mount observation. + pub temporary: TemporaryReport, +} + +/// Run one private Linux probe challenge, then wait for host-owned retirement. +/// +/// # Errors +/// Rejects non-Linux execution, any endpoint other than fixed loopback9001, +/// malformed challenges, expired budgets and failed kernel/API observations. +/// +/// ```no_run +/// # #[cfg(unix)] async fn probe() { +/// let outcome = rustack_lambda_runtime::probe::run().await; +/// # } +/// ``` +#[cfg(unix)] +pub async fn run() -> Result<(), ProbeError> { + client::run().await +} diff --git a/crates/rustack-lambda-runtime/src/probe/parse.rs b/crates/rustack-lambda-runtime/src/probe/parse.rs new file mode 100644 index 00000000..747d2424 --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/parse.rs @@ -0,0 +1,214 @@ +//! Checked parsers over fixed, bounded kernel text; no paths are derived from it. + +use super::{ProbeError, TemporaryReport}; + +pub(super) fn number(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() || value.len() > 20 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProbeError::Observation); + } + value.parse().map_err(|_| ProbeError::Observation) +} +pub(super) fn limit(value: &str) -> Result, ProbeError> { + if matches!(value.trim(), "max" | "-1") { + Ok(None) + } else { + number(value).map(Some) + } +} +pub(super) fn cpu(value: &str) -> Result<(Option, u64), ProbeError> { + let mut fields = value.split_ascii_whitespace(); + let quota = limit(fields.next().ok_or(ProbeError::Observation)?)?; + let period = number(fields.next().ok_or(ProbeError::Observation)?)?; + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + Ok((quota, period)) +} +pub(super) struct Status { + pub(super) uid: u32, + pub(super) gid: u32, + pub(super) capabilities: u64, + pub(super) no_new_privileges: bool, + pub(super) seccomp: u8, +} +pub(super) fn status(text: &str) -> Result { + bounded(text)?; + let field = |key: &str| -> Result<&str, ProbeError> { + let mut matching = text + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| *name == key) + .map(|(_, value)| value.trim()); + let value = matching.next().ok_or(ProbeError::Observation)?; + if matching.next().is_some() { + return Err(ProbeError::Observation); + } + Ok(value) + }; + let effective = |text: &str| -> Result { + let mut fields = text.split_ascii_whitespace(); + let mut ids = [0_u32; 4]; + for id in &mut ids { + *id = u32::try_from(number(fields.next().ok_or(ProbeError::Observation)?)?) + .map_err(|_| ProbeError::Observation)?; + } + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + let [_, effective, _, _] = ids; + Ok(effective) + }; + let cap = field("CapEff")?; + if cap.len() != 16 || !cap.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(ProbeError::Observation); + } + let no_new_privileges = match field("NoNewPrivs")? { + "0" => false, + "1" => true, + _ => return Err(ProbeError::Observation), + }; + Ok(Status { + uid: effective(field("Uid")?)?, + gid: effective(field("Gid")?)?, + capabilities: u64::from_str_radix(cap, 16).map_err(|_| ProbeError::Observation)?, + no_new_privileges, + seccomp: u8::try_from(number(field("Seccomp")?)?).map_err(|_| ProbeError::Observation)?, + }) +} +fn bounded(text: &str) -> Result<(), ProbeError> { + if text.len() > 65536 + || text.lines().count() > 512 + || text.lines().any(|line| line.len() > 4096) + { + Err(ProbeError::Observation) + } else { + Ok(()) + } +} +pub(super) fn membership(text: &str, unified: bool) -> Result { + bounded(text)?; + let mut v2 = None; + let mut required = [None; 3]; + for line in text.lines() { + let mut fields = line.splitn(3, ':'); + let id = number(fields.next().ok_or(ProbeError::Observation)?)?; + u32::try_from(id).map_err(|_| ProbeError::Observation)?; + let controllers = fields.next().ok_or(ProbeError::Observation)?; + let path = fields.next().ok_or(ProbeError::Observation)?; + if controllers.len() > 256 || controllers.split(',').count() > 32 || path.is_empty() { + return Err(ProbeError::Observation); + } + if controllers.is_empty() { + if id != 0 { + return Err(ProbeError::Observation); + } + if v2.replace(path == "/").is_some() { + return Err(ProbeError::Observation); + } + } else if !unified { + if id == 0 { + return Err(ProbeError::Observation); + } + for controller in controllers.split(',') { + let slot = match controller { + "memory" => required.get_mut(0), + "pids" => required.get_mut(1), + "cpu" => required.get_mut(2), + _ => None, + }; + if let Some(slot) = slot { + if slot.replace(path == "/").is_some() { + return Err(ProbeError::Observation); + } + } + } + } + } + if unified { + v2.ok_or(ProbeError::Observation) + } else { + if required.iter().any(Option::is_none) { + return Err(ProbeError::Observation); + } + Ok(required.iter().all(|value| *value == Some(true))) + } +} +fn size(value: &str) -> Result { + let (digits, scale) = match value.as_bytes().last() { + Some(b'k') => ( + value.strip_suffix('k').ok_or(ProbeError::Observation)?, + 1024, + ), + Some(b'm') => ( + value.strip_suffix('m').ok_or(ProbeError::Observation)?, + 1024 * 1024, + ), + Some(b'g') => ( + value.strip_suffix('g').ok_or(ProbeError::Observation)?, + 1024 * 1024 * 1024, + ), + _ => (value, 1), + }; + number(digits)? + .checked_mul(scale) + .ok_or(ProbeError::Observation) +} +pub(super) fn mounts(text: &str, mode: u32) -> Result<(bool, TemporaryReport), ProbeError> { + bounded(text)?; + let mut root = None; + let mut temporary = None; + for line in text.lines() { + let mut fields = line.split_ascii_whitespace(); + for _ in 0..4 { + fields.next().ok_or(ProbeError::Observation)?; + } + let target = fields.next().ok_or(ProbeError::Observation)?; + let options = fields.next().ok_or(ProbeError::Observation)?; + if !matches!(target, "/" | "/tmp") { + continue; + } + if !fields.by_ref().any(|field| field == "-") { + return Err(ProbeError::Observation); + } + let filesystem = fields.next().ok_or(ProbeError::Observation)?; + fields.next().ok_or(ProbeError::Observation)?; + let super_options = fields.next().ok_or(ProbeError::Observation)?; + if fields.next().is_some() { + return Err(ProbeError::Observation); + } + if target == "/" { + if root + .replace(options.split(',').any(|v| v == "ro")) + .is_some() + { + return Err(ProbeError::Observation); + } + } else { + let mut sizes = super_options + .split(',') + .filter_map(|v| v.strip_prefix("size=")); + let bytes = size(sizes.next().ok_or(ProbeError::Observation)?)?; + if sizes.next().is_some() { + return Err(ProbeError::Observation); + } + let observed = TemporaryReport::builder() + .bytes(bytes) + .mode(mode) + .tmpfs(filesystem == "tmpfs") + .nosuid(options.split(',').any(|v| v == "nosuid")) + .nodev(options.split(',').any(|v| v == "nodev")) + .build(); + if temporary.replace(observed).is_some() { + return Err(ProbeError::Observation); + } + } + } + Ok(( + root.ok_or(ProbeError::Observation)?, + temporary.ok_or(ProbeError::Observation)?, + )) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-runtime/src/probe/parse/tests.rs b/crates/rustack-lambda-runtime/src/probe/parse/tests.rs new file mode 100644 index 00000000..93969eeb --- /dev/null +++ b/crates/rustack-lambda-runtime/src/probe/parse/tests.rs @@ -0,0 +1,83 @@ +use super::*; + +const STATUS: &str = "Uid:\t1000 1001 1000 1000\nGid:\t1000 1002 1000 \ + 1000\nCapEff:\t0000000000000000\nNoNewPrivs:\t1\nSeccomp:\t2\n"; +const MOUNTS: &str = "1 0 0:1 / / ro,relatime - overlay overlay rw\n2 1 0:2 / /tmp \ + rw,nosuid,nodev,relatime - tmpfs tmpfs rw,size=524288k,inode64\n"; +#[test] +fn test_should_parse_independent_kernel_observation_fixtures() { + let report = status(STATUS).unwrap(); + assert_eq!((report.uid, report.gid), (1001, 1002)); + assert_eq!(report.capabilities, 0); + assert!(report.no_new_privileges); + assert_eq!(report.seccomp, 2); + let (root, tmp) = mounts(MOUNTS, 0o1777).unwrap(); + assert!(root); + assert_eq!((tmp.bytes, tmp.mode), (512 * 1024 * 1024, 0o1777)); + assert!(tmp.tmpfs && tmp.nosuid && tmp.nodev); + assert_eq!(cpu("100000 100000\n").unwrap(), (Some(100_000), 100_000)); + assert_eq!(cpu("max 100000").unwrap(), (None, 100_000)); + assert_eq!(limit("-1").unwrap(), None); + assert_eq!( + limit("9223372036854771712").unwrap(), + Some(9_223_372_036_854_771_712) + ); +} +#[rstest::rstest] +#[case::empty("")] +#[case::overflow("18446744073709551616")] +#[case::negative("-2")] +#[case::fraction("1.0")] +#[case::nonascii("١")] +#[case::multiple("1 2")] +#[test] +fn test_should_reject_invalid_kernel_numbers(#[case] value: &str) { + assert!(number(value).is_err()); +} +#[rstest::rstest] +#[case::missing("Seccomp:\t2\n", "")] +#[case::duplicate("Seccomp:\t2\n", "Seccomp:\t2\nSeccomp:\t2\n")] +#[case::short_uid("1000 1001 1000 1000", "1000 1001 1000")] +#[case::long_uid("1000 1001 1000 1000", "1000 1001 1000 1000 1000")] +#[case::uid_overflow("1000 1001 1000 1000", "1000 4294967296 1000 1000")] +#[case::bad_privilege("NoNewPrivs:\t1", "NoNewPrivs:\t2")] +#[case::bad_cap("0000000000000000", "00000000000000zz")] +#[test] +fn test_should_reject_missing_duplicate_or_invalid_process_status( + #[case] before: &str, + #[case] after: &str, +) { + assert!(status(&STATUS.replace(before, after)).is_err()); +} +#[rstest::rstest] +#[case::missing( + "2 1 0:2 / /tmp rw,nosuid,nodev,relatime - tmpfs tmpfs rw,size=524288k,inode64\n", + "" +)] +#[case::size("size=524288k", "size=18446744073709551615g")] +#[case::duplicate_size("size=524288k", "size=524288k,size=1")] +#[case::missing_separator(" - tmpfs", " tmpfs")] +#[test] +fn test_should_reject_incomplete_or_overflowing_mounts(#[case] before: &str, #[case] after: &str) { + assert!(mounts(&MOUNTS.replace(before, after), 0o1777).is_err()); +} +#[test] +fn test_should_bound_lines_and_reject_duplicate_root_or_temporary_mounts() { + assert!(status(&"x".repeat(65537)).is_err()); + assert!(status(&format!("{STATUS}{}", "x\n".repeat(513))).is_err()); + assert!(mounts(&format!("{MOUNTS}{MOUNTS}"), 0o1777).is_err()); + assert!(cpu("1 2 3").is_err()); +} + +#[test] +fn test_should_require_current_controller_membership_at_namespace_root() { + assert!(membership("0::/\n", true).unwrap()); + assert!(!membership("0::/docker/foreign\n", true).unwrap()); + let v1 = "2:memory:/\n3:pids:/\n4:cpu,cpuacct:/\n5:name=systemd:/unrelated\n"; + assert!(membership(v1, false).unwrap()); + assert!(!membership(&v1.replace("memory:/", "memory:/docker/current"), false).unwrap()); + assert!(membership(&v1.replace("2:memory:/\n", ""), false).is_err()); + assert!(membership(&format!("{v1}9:memory:/\n"), false).is_err()); + assert!(membership("0::/\n0::/\n", true).is_err()); + assert!(membership("1::/\n", true).is_err()); +} diff --git a/crates/rustack-lambda-runtime/src/process.rs b/crates/rustack-lambda-runtime/src/process.rs index 1016a305..5e25e1ba 100644 --- a/crates/rustack-lambda-runtime/src/process.rs +++ b/crates/rustack-lambda-runtime/src/process.rs @@ -39,6 +39,17 @@ fn seconds(key: &str, default: u64, maximum: u64) -> Result Result<(), RelayError> { + let mut arguments = env::args_os(); + arguments.next(); + match arguments.next() { + None => {} + Some(command) if command == "probe" && arguments.next().is_none() => { + return rustack_lambda_runtime::probe::run() + .await + .map_err(|_| RelayError::Configuration); + } + Some(_) => return Err(RelayError::Configuration), + } let environment = env::var("RUSTACK_LAMBDA_ENVIRONMENT_ID").map_err(|_| RelayError::Configuration)?; if environment.len() != 36 { diff --git a/specs/index.md b/specs/index.md index faa627b2..49ba019e 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, historical reconciliation, allowed pulls, network/resource probes and final startup acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, historical reconciliation, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index b05088c9..9aae2e6b 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -563,7 +563,7 @@ Errors are typed Configuration/InUse/InvalidState/Deadline/Cancelled/Worker/Io w Revalidate externally constructible revisions: Docker backend, Image artifact, live matching Linux architectures, absent ZIP runtime/handler, canonical nonnil revision UUID, bounded function name and resolved numeric version/$LATEST, integral timeout1..900s, memory128..10,240MiB and ephemeral512..10,240MiB. Validate captured user environment under the existing4KiB/count/key rules, separately accepting only the provider's bounded AWS_REGION/AWS_DEFAULT_REGION injections; require matching nonempty region values. Compose a deterministic map with exactly one entry per key: last inherited image value/deletion marker, then captured variables, then authoritative runtime metadata. Bare-name inherited deletion markers remain explicit unless a later tier supplies that key. Do not concatenate duplicate names: process lookup of duplicate environment entries is not a portable precedence mechanism. Inject fixed loopback Runtime API, name/resolved version/memory/timeout, on-demand initialization, log group/stream and task root. Never copy process environment or host credentials. The four Rustack-supplied relay Env entries contain only its environment UUID and validated relay budgets; Docker also retains validated relay-image defaults. No captured function variables or process environment are supplied to the relay. Preserve omitted/empty ImageConfig through the already-validated execution plan, with no shell or tag re-resolution. -Both create bodies disable inherited healthchecks and daemon persistent logging, set Tty=false, no automatic removal/restart, read-only rootfs, cap-drop ALL, no-new-privileges and privileged=false, with no host binds/devices or published ports. Function memory and swap-total are equal (no extra unaccounted swap),256 PID cap and the selected CPU cap; relay is64MiB/64PIDs/0.1CPU. Aggregate memory admission uses checked function+relay bytes. The function /tmp tmpfs has its captured ephemeral byte cap, nosuid/nodev and mode1777; relay /tmp is1MiB. Tmpfs memory still counts against each memory cap. Matching memory/swap semantics follow [Docker resource constraints](https://docs.docker.com/engine/containers/resource_constraints/). Require default seccomp/resource enforcement in the separate daemon probe. +Both create bodies explicitly request private cgroup namespaces (never inherit the v1 host default) and disable inherited healthchecks and daemon persistent logging, set Tty=false, no automatic removal/restart, read-only rootfs, cap-drop ALL, no-new-privileges and privileged=false, with no host binds/devices or published ports. Function memory and swap-total are equal (no extra unaccounted swap),256 PID cap and the selected CPU cap; relay is64MiB/64PIDs/0.1CPU. Aggregate memory admission uses checked function+relay bytes. The function /tmp tmpfs has its captured ephemeral byte cap, nosuid/nodev and mode1777; relay /tmp is1MiB. Tmpfs memory still counts against each memory cap. Matching memory/swap semantics follow [Docker resource constraints](https://docs.docker.com/engine/containers/resource_constraints/). Require default seccomp/resource enforcement in the separate daemon probe. Absent/root/numeric-zero image USER is replaced with configured nonzero UID:GID; preserve bounded nonroot numeric/named image USER declarations and reject malformed user/group grammar rather than passing arbitrary text. Named-account resolution remains the image's own /etc/passwd contract, not a host account lookup or proof against operator-supplied hostile image aliases. The relay always uses configured nonzero UID:GID. Permission-incompatible images fail initialization. No host filesystem/user lookup or credential inheritance is allowed. @@ -695,6 +695,36 @@ Shutdown first closes executor admission synchronously, then transfers its pool Verification includes pure rejection before allocation, dynamic readiness/pin revocation, alias/resolved-version metadata, binary response/FunctionError/tail preservation, typed failures, context cancellation with delayed settlement, dropped shutdown waiters and actual Image create→Active→publish/alias→Invoke through provider/HTTP. A real Docker fixture readiness adapter proves execution composition only; final production startup/probe/platform acceptance remains separately required. +### 4.7 Startup network qualification and actual resource probe + +Startup qualification has separate proofs; a connected daemon, parsed policy or successful image lease is insufficient. `NetworkSelection::qualify(port, deadline)` is read-only: none issues a disconnected selection without a daemon call; a named/ID network uses the bounded GET /networks/{key} endpoint. Require an exact selected name or full canonical64-lowercase-hex ID, local scope, bridge driver, non-ingress/non-config-only flags, a non-builtin name and a missing or explicit false default_bridge option. Reject partial IDs and ambiguous/missing metadata. Return a private QualifiedNetwork containing the full ID selection so later name reuse cannot redirect allocation. No network is created/deleted by qualification; operator-selected egress remains explicit, without an SSRF/firewall promise. This follows [Docker bridge semantics](https://docs.docker.com/engine/network/drivers/bridge/). + +The pinned Rust runtime binary adds an exact `probe` command mode (normal relay accepts no arguments). The probe image is the same operator-pinned relay image, started as the function side of an EnvironmentManager allocation with an explicit `/rustack-lambda-runtime probe` entrypoint. Its private minimal Runtime API client connects only to127.0.0.1:9001, consumes exactly one empty-object challenge, and reports a bounded typed JSON ResourceReport with version1, consumed attempt UUID, architecture, actual kernel limits and isolation flags. It never consults host process credentials, runs a shell, writes large files, spawns children or infers success from Docker inspect fields. Unknown arguments fail statically. Probe mode is not a public API listener. + +Both relay/function allocation bodies explicitly request private cgroup namespaces. Verify /proc/self/cgroup membership is rooted at / for unified v2 or every required v1 memory/PID/CPU controller before accepting controller-root files; missing/duplicate membership rejects. Docker defaults differ between [v1 and v2](https://docs.docker.com/engine/containers/runmetrics/), so inherited defaults cannot establish this invariant. Read kernel-owned /proc/self/status and mountinfo with64KiB/512-line/4KiB-line caps, and fixed cgroup files with128-byte caps, within the consumed handler deadline and bounded per-request IO. Normalize cgroup v2 memory.max, memory.swap.max, pids.max and cpu.max; for a v1 layout normalize memory.limit_in_bytes, memory.memsw.limit_in_bytes (subtract memory with checked arithmetic), pids.max and cpu.cfs_{quota,period}_us. Only a missing v2 memory file selects v1; permissions, malformed or incomplete controller evidence fail. Textual unlimited markers (v2 max and v1 -1) become None/omitted report fields and cannot satisfy a finite expected quota. Numeric v1 unlimited sentinels retain their numeric form and fail the exact finite-memory/swap comparisons; do not claim every kernel unlimited encoding becomes None. Parse proc UID/GID/capabilities/no_new_privs/seccomp with duplicate/missing-field rejection. Require an actual read-only root mount and a tmpfs /tmp with nosuid/nodev, captured size and mode1777. Bound/check every numeric conversion; parse kernel unit suffixes with checked multiplication. The [kernel cgroup contract](https://docs.kernel.org/admin-guide/cgroup-v2.html), [v1 memory](https://docs.kernel.org/admin-guide/cgroup-v1/memory.html), [CPU bandwidth](https://docs.kernel.org/scheduler/sched-bwc.html) and [proc contract](https://www.kernel.org/doc/html/latest/filesystems/proc.html) define these observations. + +`probe_environment` receives the retained directory/relay/environment port, qualified allocation policy and bounded startup/cleanup budgets. It constructs a private captured Image revision with128MiB memory,512MiB tmpfs,256 PIDs, configured CPU and a10s consumed-handler budget; only one probe attempt is submitted and never replayed. Await BootstrapReady, actual invocation result and independent environment retirement. A returned EnvironmentProbe is private proof of the matching directory/run, relay and policy inputs and cannot be constructed from serialized metadata. Validate the4KiB report before issuing it: exact version/attempt/architecture and namespace-root membership; finite memory, zero extra swap and PID limits; finite CPU quota matching `floor(nano_cpus * period / 1e9)` with checked arithmetic and period1,000..1,000,000us; nonroot UID/GID, zero effective capabilities, no_new_privs and seccomp mode2; root/tmp flags and exact tmp bytes/mode. Missing/mismatched fields or FunctionError reject, never fabricate qualification. This observes actual kernel enforcement configuration; it does not claim to exhaust memory/PID limits or replace dedicated OOM/fault tests. + +The minimal probe HTTP connection stays owned inside its request future (no detached connection worker), with bounded response/header/body and one absolute deadline. After posting its report and receiving202, the probe waits for host-owned retirement. On collection failure it reports a static bounded invocation error and still waits for retirement; payloads, paths and kernel text do not enter infrastructure diagnostics. The host uses existing retained environment ownership, so a dropped/timed-out probe waiter cancels the environment without losing container/IO cleanup. Issue proof only after shutdown confirms settlement and a final startup/caller-deadline and retained-input availability recheck; expired callers never receive late successful proof. Cleanup keeps its independent budget and uncertainty remains a distinct error. + +```text +┌─ startup coordinator (admission closed) ─────────────────────────────────┐ +│ persistent directory │ historical effects │ daemon/image qualification │ +│ │ (independent mandatory gates) │ +│ operator network → inspect → fixed network ID │ +│ pinned relay/policy → private probe revision │ +└─────────────────────────────┬───────────────────────────────────────────┘ + ▼ retained EnvironmentManager allocation +┌─ relay container ───────────┐ ┌─ same-image probe function ─────────┐ +│ loopback Runtime API │◀────▶│ one challenge → kernel observations │ +│ ordered control transport │ │ bounded report → wait for retirement│ +└─────────────────────────────┘ └──────────────────────────────────────┘ + ▼ host validates + joins normal retirement + private EnvironmentProbe (does not open readiness) +``` + +The coordinator must combine these proofs with historical uncertain-effect reconciliation, allowed pulls, endpoint reachability and final configuration/capability acceptance. Neither QualifiedNetwork nor EnvironmentProbe alone implements DockerReadiness or enables CLI Docker. Tests include strict report/parser mismatches, exact/no-effect network requests, actual none and operator bridge probes, failed/unavailable resource observations and retained cancellation/cleanup. A fake report is never counted as actual resource enforcement evidence. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index ffbe93d3..f1e1e17d 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -89,6 +89,14 @@ The permanent lease.lock inode is never truncated, replaced or deleted during re This capability alone is not a ready backend. Startup must separately reconcile prior runs and unresolved historical effects under the same persistent owner before accepting image execution. Do not treat one empty list/404 as proof that a prior uncertain daemon mutation cannot appear later. Shutdown keeps the lease until environment/deployment/image cleanup owners are joined; unresolved cleanup prevents successful quiescence/snapshot publication. +### 1.4 Network and kernel probe receipts + +Startup must use [runtime §4.7](./rustack-workflows-20-lambda-runtime-design.md#47-startup-network-qualification-and-actual-resource-probe) to inspect an exact operator bridge and bind the full network ID before allocation. none is deliberately disconnected. Relay/function creation explicitly requests private cgroup namespaces; the pinned Rust probe checks current controller membership before reading normalized kernel limits. Host validation and joined retirement precede issuance of a private, exact-input resource proof; a final deadline/live-input check prevents late or revoked success. A serialized report alone cannot establish readiness. + +`make test-lambda-probes` runs bounded parser/qualification/facade regressions. `make test-lambda-probes-docker WORKFLOWS_DOCKER_SOCKET= WORKFLOWS_ARCH=arm64 WORKFLOWS_NETWORK=` requires an explicit pre-provisioned operator bridge and the pinned fixture images; it runs both none and bridge cases without silently skipping either. Tests do not create/delete the operator bridge. Test provisioners retain their own bridge identity/ownership and remove only their known bridge after all attached probe environments settle. + +Current actual execution evidence is on cgroup-v2 arm64 VZ and amd64/QEMU daemons. v1 normalization/private namespace behavior has parser and allocation-body coverage; an actual v1 daemon lane is still unmeasured. Network/resource receipts are implemented components; historical uncertain effects, registry pulls, endpoint reachability and the production startup coordinator remain mandatory before CLI Docker enablement. + ## 2. Deployment topology and readiness | Topology | Control path | Handler SDK path | Required test | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 2b0cc880..24e0df0e 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -104,6 +104,8 @@ Dependency security closure (§7.2/D27) additionally requires current cargo audi The B05 regression composes the actual session/HTTP/relay/client with an environment worker after BootstrapReady and an abandoned poll. For an active call, observe an offered but unconsumed event with a recording waker, abandon that offer, then POST InitError. Test Idle and Busy at32KiB and32KiB+1, require the bounded initialization class versus Protocol, redacted diagnostics and joined control/log/API retirement. Scripted container absence remains distinct from real Docker code execution. +`make test-lambda-probes` and explicit `make test-lambda-probes-docker` (with WORKFLOWS_NETWORK set to an operator-provisioned bridge) cover startup network/resource qualification (§4.7/D37). They must reject builtin/nonlocal/partial-ID networks and pin accepted bridge IDs. Probe report decoding is bounded and strict; independently vary attempt/version/architecture, finite memory/swap/PIDs/CPU, UID/GID/capabilities/seccomp and root/tmp observations. Actual pinned Rust probe execution in none and an explicit user bridge must report kernel limits, then leave zero owned environment reservations/resources before proof issuance. Historical reconciliation and production readiness remain separate gates. + Docker Executor composition (§4.6/D36) additionally requires pure admission rejection, trusted readiness revocation, raw/binary success and FunctionError preservation, one Tail base64 encoding with empty/None distinction, resolved-version versus requested-alias metadata, typed initialization/OOM/transport outcomes and independent cancellation/shutdown settlement. The actual image/provider/HTTP fixture must perform deployment, version/alias and computed-result/error calls. Controlled fixture readiness is not production startup/probe evidence. `make test-lambda-executor` runs the Executor/header boundary and supervisor tests; `make test-lambda-executor-docker` builds the pinned fixtures and executes boto3→HTTP→provider→actual image deployment/version/alias/results/logs/error/timeout/replacement. Scalar header duplicates must reject before lookup. Only admission quota Busy maps429; nested wire Busy remains typed infrastructure500. Exact invoked alias ARN must reach the RIC independently from resolved-version response headers. The pool settlement consumer tests exercise `start`/cancel-safe `wait`/independent `cancel_and_wait`. Hold retirement behind an explicit barrier; require pending settlement after invocation expiry, after a dropped cleanup waiter, and during root-abort recovery. Queued rejection must settle without allocation, consumed success must preserve warm reuse, and closed or supervision-failed receipts must never report clean. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index f4d5fa30..73f9f6b2 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Docker Executor/provider/HTTP composition is implemented and qualified through controlled-readiness real SDK fixtures. Missing: production runtime startup construction; historical-effect reconciliation, allowed pulls, network/resource probes and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Docker Executor/provider/HTTP composition is implemented and qualified through controlled-readiness real SDK fixtures. Startup network/full-ID qualification and actual pinned Rust kernel resource proofs are implemented. Missing: production runtime startup construction; historical-effect reconciliation, allowed pulls, endpoint reachability and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -153,6 +153,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 post-bootstrap initialization | Implemented/reviewed; full M0 remains open | Shared bounded InitError classification preserves initialization failure after an abandoned BootstrapReady poll, including an offered-but-unconsumed first event. Four actual session/HTTP/relay/client/worker composition regressions prove Idle/Busy32KiB boundaries, no active reply before a scripted known-container absence barrier, redaction and joined IO/API retirement. Independent read-only review has no findings. Environment target25 cases, workspace2,277 tests/407 separately gated, strict boundary1,062 unit+doc passes/13 Docker-gated; build/test/fmt/Clippy/rustdoc and strict production checks pass. Evidence: `/tmp/rustack-workflows-b05-{environments,build,tests,fmt,clippy,docs,boundaries}.log`. B05 closed; this is not production Executor/startup or actual Docker acceptance. | | 1.3 supervised pool invocation | Implemented/reviewed; Executor consumer remains required | §4.5/D35 expose synchronous start, cancel-safe response wait and separate resumable cancel_and_wait with an independent cleanup deadline. Per-call settlement senders stay beside Retirement records above JoinSet through worker/root recovery. Nine new regressions,27 total pool cases, prove delayed/expired cleanup, waiter loss, no-allocation rejection, worker/root failure, warm success and missing-producer uncertainty. W86 observer rejection cleanup is fixed; final independent review has no remaining finding. Workspace2,286 tests/408 separately gated; strict boundary870 units+204 docs (1,074 total)/14 Docker-gated; standard build/test/fmt/Clippy/rustdoc and strict production checks pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes7 owner/pool real-image cases, including per-call awaited retirement before global shutdown, container404 and replacement counter1 while admission stays open. Post-run owned containers/internal aliases are absent. This does not close Executor/provider/startup or M0 acceptance. | | 1.3 Docker Executor/provider | Implemented/reviewed under trusted runtime injection; production startup remains | §4.6/D36 consume one PoolInvocation with independent cleanup settlement, immutable plan/metadata validation, raw outcomes, resolved version, requested alias ARN and one base64 tail encoding. W87–W91 are fixed/rechecked.15 Executor +16 handler cases pass, including held reserved concurrency, expired cleanup/failed quiescence and resumed physical retirement. Workspace2,313 tests/409 separately gated; core611 units/141 docs; strict boundary897 units+208 docs (1,105 total)/15 Docker-gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU daemon passes actual boto3→HTTP→provider→Docker deploy/publish/alias/Invoke/error/timeout/replacement plus seven existing owner/pool cases (eight per daemon). Controlled fixture readiness is not startup/probe/native-x86 evidence; post-run owned containers/internal aliases are absent. | +| 1.3/1.4 startup probes | Implemented/reviewed; no readiness grant | §4.7/D37 validate exact user bridge/none selections, bind full IDs, and execute the pinned Rust probe under retained EnvironmentManager ownership. Private cgroup namespaces and process root-membership checks cover v1/v2 parser semantics; exact kernel limits/flags, scoped proof matching, caller abandonment, independent cleanup and final deadline/live-input checks are enforced. W92–W94 closed;57 focused cases pass. Workspace2,370/411 gated, boundary954 units+217 docs (1,171 total)/17 gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU cgroup-v2 daemon passes17 distinct actual probe/SDK/pool/wire/runtime cases; scoped resources and the two dedicated probe bridges are removed. Actual v1/native-x86, historical effects, pulls, endpoint/config/capability coordinator acceptance remain open. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Next implement the production startup coordinator: historical uncertain-effect reconciliation, allowed pulls and network/resource probe consumers, then task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Next implement historical uncertain-effect reconciliation and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index bc07e24b..ee66bd7f 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -433,6 +433,24 @@ Each retained Engine29.5.2 daemon explicitly passes one real boto3/HTTP/provider M0 remains open: the actual production startup coordinator must implement historical uncertain-effect reconciliation, allowed pulls, network/resource probes, configuration/capability publication and complete platform acceptance. P2–P5/ASL remain required by the full objective. A trusted test readiness adapter must never be presented as those missing proofs. +## P1 startup network/resource qualification review + +§4.7/D37 now provides read-only exact network inspection/full-ID policy binding and an actual pinned Rust kernel probe. Probe mode consumes one fixed-loopback Runtime API challenge, reports a bounded typed schema and waits for the host's existing retained environment retirement. Host proof remains private, exact-input/service-bound, nonserializable and independent from production DockerReadiness. Final read-only review has no remaining in-slice finding. + +| ID | Priority / source | Finding and resolution | Evidence | +|---|---|---|---| +| W92 | P2; `crates/rustack-lambda-core/src/docker/plan.rs:589`, `crates/rustack-lambda-runtime/src/probe/parse.rs:89` | Controller-root reads cannot qualify a v1 container under Docker's default host cgroup namespace. Explicitly emit private namespaces for relay/function and require current-process unified or required v1 controller membership at / | Both create-body assertions and positive/foreign/missing/duplicate v1/v2 membership fixtures pass; host rejects false namespace membership. Actual final daemons use cgroup v2; no actual v1 lane is claimed | +| W93 | P3; runtime §4.7 and `crates/rustack-lambda-runtime/src/probe/parse.rs:12` | The initial prose overstated None normalization. Document textual max/-1 as omitted None while preserving numeric v1 unlimited sentinels for exact finite-mismatch rejection | Independent huge-sentinel fixture preserves9223372036854771712 numerically; finite memory/swap expectations reject unavailable/unlimited enforcement | +| W94 | P2; self-review, `crates/rustack-lambda-core/src/docker/probe.rs:287` | A timely report could otherwise produce a late success after independent cleanup overran startup/caller time. Recheck deadline and live retained inputs after joined cleanup, before proof construction | Independent final source recheck confirms ordering; existing bounded environment timeouts and facade-abandonment ownership tests remain green; final actual probe rechecks pass | + +`make test-lambda-probes` passes57 new cases:22 runtime parser/client boundary cases,14 network qualification cases and21 host report/facade cases. Independent kernel text fixtures cover v1/v2 normalization/membership, UID/GID/capabilities/privilege/seccomp, mount options/size/mode, numeric overflow and bounded lines. Strict report cases independently alter identity/namespace/memory/swap/PIDs/CPU/users/flags/filesystems, and reject unknown/duplicate/oversized JSON. The facade cancellation regression holds an issued create response, abandons the public probe waiter and requires the allocation reservation to remain held until definitive400 and joined manager settlement. Scripted evidence is not counted as kernel execution. + +Final-source workspace build/test/fmt/Clippy/rustdoc and strict production boundary checks pass. Workspace has2,370 passed, zero failed and411 separately resource-gated tests. Lambda core has646 unit tests/149 doctests; shared runtime has141 unit tests/52 doctests. The six-library boundary target passes954 unit tests+217 doctests (1,171 total), with17 separately gated real-Docker cases. Audit scans474 dependencies without vulnerabilities; deny passes all four categories with existing unmatched/duplicate-policy notices. The already-resolved serde_json workspace dependency moves from runtime dev-only to production for typed report encoding; no new library/version or advisory exception is added. Logs: `/tmp/rustack-workflows-probe-verified-{build,tests,fmt,clippy,docs,boundaries}.log`, focused `/tmp/rustack-workflows-probe-focused-final.log` and security `/tmp/rustack-workflows-probe-{audit,deny}.log`. + +Each retained Engine29.5.2 daemon passes17 explicitly selected actual cases with zero ignored: two pinned Rust probe cases (none/operator bridge), one boto3/HTTP/provider/Executor scenario, seven owner/pool cases, two wire cases and five shared runtime cases. Final host deadline rechecks rerun the two probe cases without counting them as additional distinct cases. Both arm64 VZ and amd64/QEMU observe cgroup v2 with128MiB finite memory, zero extra swap,256 PIDs, actual33333/100000us CPU quota, nonroot IDs, zero effective capabilities, no_new_privs, seccomp2 and read-only root/512MiB mode1777 tmpfs. Proof is returned only after allocation count reaches zero. Reports cannot match changed policies. Final exact-ID/label checks remove the two bridges provisioned for this run; scoped container/internal-alias queries are empty. User images/caches remain. Actual logs: `/tmp/rustack-workflows-probe-final-{arm64,amd64}.log` (full sweep), `probe-verified-{arm64,amd64}.log` under the same prefix (host recheck). QEMU is not native-x86 acceptance; v1 behavior is fixture/policy evidence, not a measured v1 daemon claim. + +Production startup still requires historical uncertain-effect reconciliation, allowed pulls, function-visible endpoint reachability, final configuration/capability publication and complete platform qualification. Neither proof opens readiness by itself. M0 and P2–P5/ASL remain required by the full objective. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 81bc343a..41845823 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -328,3 +328,13 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: actual code execution and provider response composition can be verified independently while ordinary CLI selection remains unavailable until all production reconciliation/probes are implemented. The readiness interface shares the existing trusted Rust custom-Executor boundary and is never supplied through AWS requests. Pool/environment quota rejection is distinct from Docker wire saturation that may occur after side effects; only the former maps to429, while both retain no-replay behavior. - **Pinned by**: [runtime §4.6](./rustack-workflows-20-lambda-runtime-design.md#46-docker-executor-and-provider-composition), [operations](./rustack-workflows-60-operations-design.md), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. No new dependency is required; this does not close M0 startup/platform acceptance. + + +## D37 — Qualify actual kernel resources through the retained environment path + +- **Context**: production readiness must verify the daemon's real container limits without trusting declared create fields or operator user images to attest themselves. +- **Alternatives considered**: info/inspect-only checks do not observe process isolation; a new one-off container owner duplicates uncertain-create/cleanup semantics; shell probes require image tooling and leak text; a network name can be retargeted after inspection. +- **Decision**: pin qualified bridge selections by full ID and execute a bounded Rust probe mode from the operator-pinned relay image as a private Lambda revision under EnvironmentManager; issue private report proof only after exact kernel validation and joined retirement. +- **Why**: probes consume the same policy/runtime path they qualify and retain existing cleanup ownership. Resource and network proofs remain independent from historical mutation reconciliation and cannot open readiness by themselves. +- **Pinned by**: [runtime §4.7](./rustack-workflows-20-lambda-runtime-design.md#47-startup-network-qualification-and-actual-resource-probe), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Promote the already-resolved runtime serde_json dev dependency to a normal workspace dependency for bounded typed report serialization; no new library/version is introduced. From 1a715ae203ed6de4c931fe400adfc8e7a3668e93 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 04:50:03 -0700 Subject: [PATCH 031/106] phase 1: persist container intents and reconcile historical allocations Implement runtime section 4.8 and D38: bounded descriptor-relative atomic container snapshots before create, durable verified IDs before deletion, and resumable historical function-before-relay cleanup. Unknown absence retains ownership; failed durable erasure retains live allocation capacity. Independent review fixes W95-W96 and reports no remaining in-slice findings. Workspace 2394 tests, all standard and strict boundary gates, audit and deny pass. Both retained daemons pass 11 actual recovery/pool/SDK/wire cases with empty scoped resources afterward. Historical image aliases, pulls, startup coordination/platform acceptance and P2-P5 remain required; production Docker stays disabled. --- Makefile | 11 + .../src/docker/environment/cleanup.rs | 41 +- .../src/docker/environment/mod.rs | 8 + .../src/docker/environment/tests.rs | 42 ++ .../src/docker/environment/worker.rs | 11 +- crates/rustack-lambda-core/src/docker/mod.rs | 2 + .../src/docker/ownership.rs | 27 +- .../src/docker/ownership/journal.rs | 385 ++++++++++++++++++ .../src/docker/ownership/journal/tests.rs | 327 +++++++++++++++ crates/rustack-lambda-core/src/docker/plan.rs | 6 + .../src/docker/recovery.rs | 134 ++++++ .../src/docker/recovery/tests.rs | 346 ++++++++++++++++ specs/index.md | 4 +- ...tack-workflows-20-lambda-runtime-design.md | 28 +- .../rustack-workflows-60-operations-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 3 +- specs/rustack-workflows-93-review.md | 13 + specs/rustack-workflows-99-decisions.md | 9 + 19 files changed, 1392 insertions(+), 19 deletions(-) create mode 100644 crates/rustack-lambda-core/src/docker/ownership/journal.rs create mode 100644 crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs create mode 100644 crates/rustack-lambda-core/src/docker/recovery.rs create mode 100644 crates/rustack-lambda-core/src/docker/recovery/tests.rs diff --git a/Makefile b/Makefile index d45e8239..2ebc7459 100644 --- a/Makefile +++ b/Makefile @@ -920,6 +920,17 @@ test-lambda-probes-docker: lambda-runtime-image lambda-runtime-fixture-image .PHONY: test-lambda-probes test-lambda-probes-docker +# Durable container identities, uncertain absence and actual restart recovery. +test-lambda-recovery: + @cargo test -p rustack-lambda-core --lib docker::ownership::journal::tests + @cargo test -p rustack-lambda-core --lib docker::recovery::tests + @cargo test -p rustack-lambda-core --lib docker::environment::tests::test_should_publish_intent_before_create_and_retain_reservation_until_durable_erase + +test-lambda-recovery-docker: lambda-runtime-image + @RUSTACK_RUNTIME_DOCKER_SOCKET="$(WORKFLOWS_DOCKER_SOCKET)" RUSTACK_RUNTIME_ARCH="$(WORKFLOWS_ARCH)" RUSTACK_RUNTIME_IMAGE="$(LAMBDA_RUNTIME_IMAGE)" cargo test -p rustack-lambda-core --lib docker::recovery::tests::test_should_reconcile_real_previous_run_containers_from_durable_intents -- --ignored --test-threads=1 + +.PHONY: test-lambda-recovery test-lambda-recovery-docker + # Delivery clocks, protocol-v2 correlation and actual Native process retirement. test-lambda-timeouts: @cargo test -p rustack-lambda-runtime --lib diff --git a/crates/rustack-lambda-core/src/docker/environment/cleanup.rs b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs index a8131a77..d9c7b6e4 100644 --- a/crates/rustack-lambda-core/src/docker/environment/cleanup.rs +++ b/crates/rustack-lambda-core/src/docker/environment/cleanup.rs @@ -20,7 +20,7 @@ use super::{ use crate::{ docker::{ DockerError, DockerKey, DockerPort, EnvironmentPlan, models::ContainerInspectResponse, - mutation::MutationReceipt, plan::ContainerRole, + mutation::MutationReceipt, ownership::ContainerRecord, plan::ContainerRole, }, invocation::CleanupTask, }; @@ -68,15 +68,30 @@ pub(super) struct Cleaner { pub(super) signals: Signals, } impl Cleaner { + async fn settled(&self, role: ContainerRole) -> bool { + let Ok(record) = ContainerRecord::from_plan(&self.plan, role) else { + return false; + }; + if self + .plan + .directory + .settle_container(record, self.config.deadline()) + .await + .is_err() + { + return false; + } + self.progress.phase(role).send_replace(Phase::Clean); + true + } pub(super) async fn container(&self, role: ContainerRole) -> bool { let phase = self.progress.phase(role).borrow().clone(); let id = match phase { - Phase::Unissued | Phase::Clean => return true, + Phase::Unissued | Phase::Clean => return self.settled(role).await, Phase::Known(id) => id, Phase::Uncertain(receipt) => { if receipt.definitely_unissued() { - self.progress.phase(role).send_replace(Phase::Clean); - return true; + return self.settled(role).await; } let Ok(inspected) = self @@ -89,6 +104,18 @@ impl Cleaner { let Ok(id) = self.plan.owned(role, &inspected) else { return false; }; + let Ok(record) = ContainerRecord::from_plan(&self.plan, role) else { + return false; + }; + if self + .plan + .directory + .know_container(record, id.clone(), self.config.deadline()) + .await + .is_err() + { + return false; + } self.progress .phase(role) .send_replace(Phase::Known(id.clone())); @@ -104,8 +131,7 @@ impl Cleaner { .await { Err(DockerError::Status(StatusCode::NOT_FOUND)) => { - self.progress.phase(role).send_replace(Phase::Clean); - return true; + return self.settled(role).await; } Ok(inspected) => inspected, Err(_) => return false, @@ -145,8 +171,7 @@ impl Cleaner { .await, Err(DockerError::Status(StatusCode::NOT_FOUND)) ) { - self.progress.phase(role).send_replace(Phase::Clean); - true + self.settled(role).await } else { false } diff --git a/crates/rustack-lambda-core/src/docker/environment/mod.rs b/crates/rustack-lambda-core/src/docker/environment/mod.rs index 117fdca4..2aa1375c 100644 --- a/crates/rustack-lambda-core/src/docker/environment/mod.rs +++ b/crates/rustack-lambda-core/src/docker/environment/mod.rs @@ -114,6 +114,9 @@ pub enum EnvironmentError { /// A worker/root/cleanup supervisor failed. #[error("environment supervision failed")] Worker, + /// Persistent container ownership could not be read or durably settled. + #[error("persistent container ownership failed")] + Ownership(#[source] Arc), /// Function reported a bounded initialization error. #[error("function initialization failed")] Initialization(Payload), @@ -132,6 +135,11 @@ pub enum EnvironmentError { #[error(transparent)] Relay(#[from] RelayError), } +impl From for EnvironmentError { + fn from(error: super::DirectoryLeaseError) -> Self { + Self::Ownership(Arc::new(error)) + } +} /// Current ownership/lifecycle state, independent of external API readiness. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/rustack-lambda-core/src/docker/environment/tests.rs b/crates/rustack-lambda-core/src/docker/environment/tests.rs index 30eb6e31..1c163b6a 100644 --- a/crates/rustack-lambda-core/src/docker/environment/tests.rs +++ b/crates/rustack-lambda-core/src/docker/environment/tests.rs @@ -656,3 +656,45 @@ async fn test_should_retire_ready_reply_only_when_its_waiter_is_abandoned(#[case "only abandoned replies retire the environment" ); } + +#[tokio::test] +async fn test_should_publish_intent_before_create_and_retain_reservation_until_durable_erase() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let mut manager = + EnvironmentManager::start(wire.port(), directory.clone(), EnvironmentConfig::default()) + .unwrap(); + let mut handle = manager.port().spawn(plan(&directory), deadline()).unwrap(); + let create = daemon.next().await; + assert!(create.head.starts_with("POST ")); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 1 + ); + let staging = root + .path() + .join(".rustack-lambda-docker/containers.pending"); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, &staging).unwrap(); + create.reply(400, b"{}").await; + quarantined(&handle).await; + assert_eq!(manager.port().status().allocated, 1); + tokio::fs::remove_file(staging).await.unwrap(); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 1 + ); + handle.shutdown(deadline()).await.unwrap(); + assert_eq!(manager.port().status().allocated, 0); + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + assert_eq!(tokio::fs::read(outside).await.unwrap(), b"keep"); + manager.shutdown(deadline()).await.unwrap(); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/crates/rustack-lambda-core/src/docker/environment/worker.rs b/crates/rustack-lambda-core/src/docker/environment/worker.rs index f2f5b7c8..df0fc593 100644 --- a/crates/rustack-lambda-core/src/docker/environment/worker.rs +++ b/crates/rustack-lambda-core/src/docker/environment/worker.rs @@ -26,7 +26,7 @@ use crate::{ artifact::Architecture, docker::{ DockerError, DockerKey, DockerPort, EnvironmentPlan, connection::stopped as wait_for_stop, - mutation::MutationGuard, plan::ContainerRole, + mutation::MutationGuard, ownership::ContainerRecord, plan::ContainerRole, }, invocation::CleanupTask, }; @@ -141,6 +141,11 @@ impl Worker { self.progress .phase(role) .send_replace(Phase::Uncertain(mutation.receipt())); + let record = ContainerRecord::from_plan(&self.plan, role)?; + self.plan + .directory + .prepare_container(record.clone(), self.config.deadline()) + .await?; let created = self .plan .create_tracked(role, &self.docker, self.config.deadline(), mutation) @@ -161,6 +166,10 @@ impl Worker { if id.as_str() != created.id { return Err(DockerError::Protocol.into()); } + self.plan + .directory + .know_container(record, id.clone(), self.config.deadline()) + .await?; self.progress .phase(role) .send_replace(Phase::Known(id.clone())); diff --git a/crates/rustack-lambda-core/src/docker/mod.rs b/crates/rustack-lambda-core/src/docker/mod.rs index 86e7e316..663bc953 100644 --- a/crates/rustack-lambda-core/src/docker/mod.rs +++ b/crates/rustack-lambda-core/src/docker/mod.rs @@ -17,6 +17,7 @@ mod pin; mod plan; mod pool; mod probe; +mod recovery; mod types; use std::{ @@ -61,6 +62,7 @@ pub use probe::{ EnvironmentProbe, EnvironmentProbeConfig, EnvironmentProbeError, ProbeFailure, ProbeSettings, probe_environment, }; +pub use recovery::{ContainerRecovery, reconcile_containers}; use rustack_core::json::{JsonLimits, from_slice}; use rustack_lambda_runtime::diagnostics::runtime_dispatch; use serde::{Serialize, de::DeserializeOwned}; diff --git a/crates/rustack-lambda-core/src/docker/ownership.rs b/crates/rustack-lambda-core/src/docker/ownership.rs index 71ac927f..7e03a305 100644 --- a/crates/rustack-lambda-core/src/docker/ownership.rs +++ b/crates/rustack-lambda-core/src/docker/ownership.rs @@ -4,6 +4,7 @@ //! on bounded blocking workers; lease clones perform no IO and retain the lock. #![allow(clippy::disallowed_types, clippy::disallowed_methods)] +mod journal; use std::{ ffi::OsString, fmt, @@ -18,6 +19,7 @@ use std::{ time::Duration, }; +pub(super) use journal::ContainerRecord; use nix::{ errno::Errno, fcntl::{OFlag, open, openat, renameat}, @@ -93,15 +95,19 @@ pub struct DirectoryLease(Arc); struct Lease { owner: Uuid, run: Uuid, - _guard: LeaseGuard, + transaction: Arc, + guard: LeaseGuard, } enum LeaseGuard { Held { - _directory: File, + directory: File, _lock: File, }, #[cfg(test)] - Testing, + Testing { + directory: File, + _root: tempfile::TempDir, + }, } impl fmt::Debug for DirectoryLease { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -168,7 +174,15 @@ impl DirectoryLease { Self(Arc::new(Lease { owner, run, - _guard: LeaseGuard::Testing, + transaction: Arc::new(Semaphore::new(1)), + guard: { + let root = tempfile::tempdir().unwrap(); + let directory = File::open(root.path()).unwrap(); + LeaseGuard::Testing { + directory, + _root: root, + } + }, })) } } @@ -303,8 +317,9 @@ fn acquire( Ok(DirectoryLease(Arc::new(Lease { owner, run: Uuid::new_v4(), - _guard: LeaseGuard::Held { - _directory: namespace, + transaction: Arc::new(Semaphore::new(1)), + guard: LeaseGuard::Held { + directory: namespace, _lock: lock, }, }))) diff --git a/crates/rustack-lambda-core/src/docker/ownership/journal.rs b/crates/rustack-lambda-core/src/docker/ownership/journal.rs new file mode 100644 index 00000000..92fa44c9 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/journal.rs @@ -0,0 +1,385 @@ +//! Bounded, atomically replaced container identities; no deployment secrets. + +use std::{ + fmt, + fs::File, + io::{self, Read, Write}, + time::Duration, +}; + +use nix::fcntl::{OFlag, renameat}; +use sha2::{Digest, Sha256}; +use tokio::time::{Instant, timeout_at}; +use uuid::Uuid; + +use super::{Control, DirectoryLease, DirectoryLeaseError, LeaseGuard, WORKERS, managed_file, run}; +use crate::docker::{ + DockerError, DockerKey, EnvironmentPlan, models::ContainerInspectResponse, plan::ContainerRole, +}; + +const PUBLISHED: &str = "containers.v1"; +const STAGING: &str = "containers.pending"; +const HEADER: &[u8; 8] = b"RSDKCTR1"; +const RECORD_BYTES: usize = 114; +const MAX_RECORDS: usize = 64; +const MAX_BYTES: usize = 25 + MAX_RECORDS * RECORD_BYTES + 32; + +#[derive(Clone, PartialEq, Eq)] +pub(in crate::docker) struct ContainerRecord { + pub(in crate::docker) run: Uuid, + pub(in crate::docker) environment: Uuid, + resource: Uuid, + function: bool, + image: [u8; 32], + pub(in crate::docker) known: Option<[u8; 32]>, +} +impl fmt::Debug for ContainerRecord { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ContainerRecord()") + } +} +impl ContainerRecord { + pub(in crate::docker) fn from_plan( + plan: &EnvironmentPlan, + role: ContainerRole, + ) -> Result { + let image = plan + .image_id(role) + .strip_prefix("sha256:") + .ok_or(DockerError::Configuration)?; + let mut digest = [0; 32]; + hex::decode_to_slice(image, &mut digest).map_err(|_| DockerError::Configuration)?; + Ok(Self { + run: plan.directory.run_id(), + environment: plan + .environment_id() + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + resource: plan + .revision() + .resource_id + .to_string() + .parse() + .map_err(|_| DockerError::Configuration)?, + function: matches!(role, ContainerRole::Function), + image: digest, + known: None, + }) + } + pub(in crate::docker) fn function(&self) -> bool { + self.function + } + fn role(&self) -> &'static str { + if self.function { "function" } else { "relay" } + } + pub(in crate::docker) fn key(&self) -> Result { + format!("rustack-{}-{}", self.role(), self.environment).parse() + } + pub(in crate::docker) fn known_key(&self) -> Result, DockerError> { + self.known.map(|id| hex::encode(id).parse()).transpose() + } + pub(in crate::docker) fn owned( + &self, + owner: Uuid, + inspected: &ContainerInspectResponse, + ) -> Result { + let config = inspected.config.as_ref().ok_or(DockerError::Protocol)?; + let labels = config.labels.as_ref().ok_or(DockerError::Quarantined)?; + let id = inspected.id.as_deref().ok_or(DockerError::Protocol)?; + let mut decoded = [0; 32]; + if id.len() != 64 + || !id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DockerError::Quarantined); + } + hex::decode_to_slice(id, &mut decoded).map_err(|_| DockerError::Quarantined)?; + if self.known.is_some_and(|expected| expected != decoded) + || inspected.name.as_deref() != Some(format!("/{}", self.key()?.as_str()).as_str()) + || config.image.as_deref() + != Some(format!("sha256:{}", hex::encode(self.image)).as_str()) + || ![ + ("rustack.lambda.ownerId", owner.to_string()), + ("rustack.lambda.runId", self.run.to_string()), + ("rustack.lambda.environmentId", self.environment.to_string()), + ( + "rustack.lambda.functionResourceId", + self.resource.to_string(), + ), + ("rustack.lambda.role", self.role().to_owned()), + ] + .iter() + .all(|(key, value)| labels.get(*key) == Some(value)) + { + return Err(DockerError::Quarantined); + } + id.parse() + } + fn same_key(&self, other: &Self) -> bool { + self.environment == other.environment && self.function == other.function + } + fn identity(&self, other: &Self) -> bool { + self.run == other.run + && self.environment == other.environment + && self.resource == other.resource + && self.function == other.function + && self.image == other.image + } +} +impl DirectoryLease { + fn namespace(&self) -> &File { + match &self.0.guard { + LeaseGuard::Held { directory, .. } => directory, + #[cfg(test)] + LeaseGuard::Testing { directory, .. } => directory, + } + } + async fn journal( + &self, + deadline: Instant, + work: impl FnOnce(&mut Vec) -> Result + Send + 'static, + ) -> Result { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or(DirectoryLeaseError::Deadline)?; + if remaining > Duration::from_secs(60) { + return Err(DirectoryLeaseError::Configuration); + } + let permit = timeout_at(deadline, self.0.transaction.clone().acquire_owned()) + .await + .map_err(|_| DirectoryLeaseError::Deadline)? + .map_err(|_| DirectoryLeaseError::Worker)?; + let lease = self.clone(); + run(&WORKERS, deadline, move |control| { + let _transaction = permit; + let mut records = read(lease.namespace(), lease.owner_id())?; + control.check()?; + let original = records.clone(); + let result = work(&mut records)?; + if records != original { + write(lease.namespace(), lease.owner_id(), &records, control)?; + } + Ok(result) + }) + .await + } + pub(in crate::docker) async fn container_records( + &self, + deadline: Instant, + ) -> Result, DirectoryLeaseError> { + self.journal(deadline, |records| Ok(records.clone())).await + } + pub(in crate::docker) async fn prepare_container( + &self, + record: ContainerRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + if record.run != self.run_id() { + return Err(DirectoryLeaseError::InvalidState); + } + self.journal(deadline, move |records| { + if let Some(previous) = records.iter().find(|old| old.same_key(&record)) { + if !previous.identity(&record) { + return Err(DirectoryLeaseError::InvalidState); + } + } else { + if records.len() >= MAX_RECORDS { + return Err(DirectoryLeaseError::InvalidState); + } + records.push(record); + } + Ok(()) + }) + .await + } + pub(in crate::docker) async fn know_container( + &self, + record: ContainerRecord, + id: DockerKey, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + let mut decoded = [0; 32]; + if id.as_str().len() != 64 + || !id + .as_str() + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DirectoryLeaseError::InvalidState); + } + hex::decode_to_slice(id.as_str(), &mut decoded) + .map_err(|_| DirectoryLeaseError::InvalidState)?; + self.journal(deadline, move |records| { + let previous = records + .iter_mut() + .find(|old| old.same_key(&record)) + .ok_or(DirectoryLeaseError::InvalidState)?; + if !previous.identity(&record) || previous.known.is_some_and(|old| old != decoded) { + return Err(DirectoryLeaseError::InvalidState); + } + previous.known = Some(decoded); + Ok(()) + }) + .await + } + pub(in crate::docker) async fn settle_container( + &self, + record: ContainerRecord, + deadline: Instant, + ) -> Result<(), DirectoryLeaseError> { + self.journal(deadline, move |records| { + if let Some(position) = records.iter().position(|old| old.same_key(&record)) { + if !records + .get(position) + .is_some_and(|old| old.identity(&record)) + { + return Err(DirectoryLeaseError::InvalidState); + } + records.remove(position); + } + Ok(()) + }) + .await + } +} + +fn read(directory: &File, owner: Uuid) -> Result, DirectoryLeaseError> { + // A torn staging snapshot is never authoritative, but its inode must be safe. + optional(directory, STAGING)?; + let Some(mut file) = optional(directory, PUBLISHED)? else { + directory.sync_all()?; + return Ok(Vec::new()); + }; + let mut bytes = Vec::with_capacity(MAX_BYTES); + Read::by_ref(&mut file) + .take((MAX_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + let records = decode(&bytes, owner)?; + file.sync_all()?; + directory.sync_all()?; + Ok(records) +} +fn optional(directory: &File, name: &str) -> Result, DirectoryLeaseError> { + match managed_file(directory, name, OFlag::O_RDONLY, MAX_BYTES as u64) { + Ok(file) => Ok(Some(file)), + Err(DirectoryLeaseError::Io(error)) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} +fn encode(owner: Uuid, records: &[ContainerRecord]) -> Result, DirectoryLeaseError> { + if records.len() > MAX_RECORDS || owner.is_nil() { + return Err(DirectoryLeaseError::InvalidState); + } + let mut bytes = Vec::with_capacity(MAX_BYTES); + bytes.extend_from_slice(HEADER); + bytes.extend_from_slice(owner.as_bytes()); + bytes.push(u8::try_from(records.len()).map_err(|_| DirectoryLeaseError::InvalidState)?); + for record in records { + bytes.extend_from_slice(record.run.as_bytes()); + bytes.extend_from_slice(record.environment.as_bytes()); + bytes.extend_from_slice(record.resource.as_bytes()); + bytes.push(u8::from(!record.function)); + bytes.extend_from_slice(&record.image); + bytes.push(u8::from(record.known.is_some())); + bytes.extend_from_slice(&record.known.unwrap_or([0; 32])); + } + let checksum = Sha256::digest(&bytes); + bytes.extend_from_slice(&checksum); + decode(&bytes, owner)?; + Ok(bytes) +} +fn decode(bytes: &[u8], owner: Uuid) -> Result, DirectoryLeaseError> { + let invalid = || DirectoryLeaseError::InvalidState; + let count = usize::from(*bytes.get(24).ok_or_else(invalid)?); + let body_len = 25 + count * RECORD_BYTES; + if count > MAX_RECORDS + || bytes.len() != body_len + 32 + || bytes.get(..8) != Some(HEADER.as_slice()) + || bytes.get(8..24) != Some(owner.as_bytes().as_slice()) + || bytes.get(body_len..) + != Some(Sha256::digest(bytes.get(..body_len).ok_or_else(invalid)?).as_slice()) + { + return Err(invalid()); + } + let mut records: Vec = Vec::with_capacity(count); + for row in bytes + .get(25..body_len) + .ok_or_else(invalid)? + .as_chunks::() + .0 + { + let uuid = |start| -> Result { + let value = Uuid::from_slice(row.get(start..start + 16).ok_or_else(invalid)?) + .map_err(|_| invalid())?; + if value.is_nil() { + Err(invalid()) + } else { + Ok(value) + } + }; + let function = match row.get(48) { + Some(0) => true, + Some(1) => false, + _ => return Err(invalid()), + }; + let id: [u8; 32] = row + .get(82..114) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?; + let known = match row.get(81) { + Some(0) if id == [0; 32] => None, + Some(1) => Some(id), + _ => return Err(invalid()), + }; + let record = ContainerRecord { + run: uuid(0)?, + environment: uuid(16)?, + resource: uuid(32)?, + function, + image: row + .get(49..81) + .ok_or_else(invalid)? + .try_into() + .map_err(|_| invalid())?, + known, + }; + if records.iter().any(|old| { + old.same_key(&record) + || (old.environment == record.environment + && (old.run != record.run || old.resource != record.resource)) + }) { + return Err(invalid()); + } + records.push(record); + } + Ok(records) +} +fn write( + directory: &File, + owner: Uuid, + records: &[ContainerRecord], + control: &Control, +) -> Result<(), DirectoryLeaseError> { + let bytes = encode(owner, records)?; + let mut file = managed_file( + directory, + STAGING, + OFlag::O_RDWR | OFlag::O_CREAT, + MAX_BYTES as u64, + )?; + control.check()?; + file.set_len(0)?; + file.write_all(&bytes)?; + file.sync_all()?; + control.check()?; + renameat(directory, STAGING, directory, PUBLISHED)?; + directory.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs b/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs new file mode 100644 index 00000000..9e0da70c --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/ownership/journal/tests.rs @@ -0,0 +1,327 @@ +use std::{ + os::unix::fs::{MetadataExt, PermissionsExt, symlink}, + path::PathBuf, +}; + +use super::*; +use crate::docker::ownership::NAMESPACE; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +fn record(run: Uuid) -> ContainerRecord { + ContainerRecord { + run, + environment: Uuid::new_v4(), + resource: Uuid::new_v4(), + function: true, + image: [2; 32], + known: None, + } +} +async fn lease(root: &tempfile::TempDir) -> DirectoryLease { + DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap() +} +fn path(root: &tempfile::TempDir, file: &str) -> PathBuf { + root.path().join(NAMESPACE).join(file) +} + +#[tokio::test] +async fn test_should_persist_unknown_known_and_settled_across_reacquisition() { + let root = tempfile::tempdir().unwrap(); + let first = lease(&root).await; + let entry = record(first.run_id()); + first + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + let bytes = tokio::fs::read(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!(bytes.len(), 171); + assert_eq!( + decode(&bytes, first.owner_id()).unwrap(), + vec![entry.clone()] + ); + drop(first); + let second = lease(&root).await; + assert_ne!(second.run_id(), entry.run); + assert_eq!( + second.container_records(deadline()).await.unwrap(), + vec![entry.clone()] + ); + second + .know_container(entry.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + drop(second); + let third = lease(&root).await; + assert_eq!( + third.container_records(deadline()).await.unwrap()[0].known, + Some([0xaa; 32]) + ); + third + .settle_container(entry.clone(), deadline()) + .await + .unwrap(); + third.settle_container(entry, deadline()).await.unwrap(); + drop(third); + assert_eq!( + lease(&root) + .await + .container_records(deadline()) + .await + .unwrap(), + [] + ); +} +#[tokio::test] +async fn test_should_serialize_concurrent_snapshots_without_losing_records() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let mut jobs = tokio::task::JoinSet::new(); + for _ in 0..32 { + let lease = lease.clone(); + jobs.spawn(async move { + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + }); + } + while let Some(result) = jobs.join_next().await { + result.unwrap().unwrap(); + } + assert_eq!(lease.container_records(deadline()).await.unwrap().len(), 32); + let metadata = tokio::fs::metadata(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!(metadata.permissions().mode() & 0o777, 0o600); + assert_eq!(metadata.nlink(), 1); +} +#[tokio::test] +async fn test_should_bound_records_and_reject_cross_run_or_changed_identity() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let first = record(lease.run_id()); + lease + .prepare_container(first.clone(), deadline()) + .await + .unwrap(); + let mut changed = first.clone(); + changed.image = [9; 32]; + assert!( + lease + .prepare_container(changed.clone(), deadline()) + .await + .is_err() + ); + assert!(lease.settle_container(changed, deadline()).await.is_err()); + assert!( + lease + .prepare_container(record(Uuid::new_v4()), deadline()) + .await + .is_err() + ); + lease + .know_container(first.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + assert!( + lease + .know_container(first, "b".repeat(64).parse().unwrap(), deadline()) + .await + .is_err() + ); + for _ in 1..64 { + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + .unwrap(); + } + assert_eq!(lease.container_records(deadline()).await.unwrap().len(), 64); + assert!( + lease + .prepare_container(record(lease.run_id()), deadline()) + .await + .is_err() + ); +} +#[test] +fn test_should_reject_truncated_corrupted_foreign_duplicate_and_invalid_encoding() { + let owner = Uuid::new_v4(); + let entry = record(Uuid::new_v4()); + let bytes = encode(owner, std::slice::from_ref(&entry)).unwrap(); + for length in 0..bytes.len() { + assert!(decode(&bytes[..length], owner).is_err()); + } + for position in 0..bytes.len() { + let mut bad = bytes.clone(); + bad[position] ^= 1; + assert!(decode(&bad, owner).is_err()); + } + assert!(decode(&bytes, Uuid::new_v4()).is_err()); + assert!(encode(owner, &[entry.clone(), entry.clone()]).is_err()); + let mut bad = entry.clone(); + bad.run = Uuid::nil(); + assert!(encode(owner, &[bad]).is_err()); + let mut relay = entry.clone(); + relay.function = false; + relay.resource = Uuid::new_v4(); + assert!(encode(owner, &[entry, relay]).is_err()); + assert!(encode(owner, &vec![record(Uuid::new_v4()); 65]).is_err()); +} +#[rstest::rstest] +#[case(PUBLISHED)] +#[case(STAGING)] +#[tokio::test] +async fn test_should_reject_unsafe_snapshot_inodes(#[case] file: &str) { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let target = path(&root, file); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, &target).unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::hard_link(&outside, &target).await.unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::remove_file(&target).await.unwrap(); + tokio::fs::write(&target, b"").await.unwrap(); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)) + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + tokio::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o600)) + .await + .unwrap(); + tokio::fs::write(&target, vec![0; MAX_BYTES + 1]) + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); + assert_eq!(tokio::fs::read(outside).await.unwrap(), b"keep"); +} +#[tokio::test] +async fn test_should_ignore_only_bounded_safe_interrupted_staging() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + tokio::fs::write(path(&root, STAGING), b"partial") + .await + .unwrap(); + tokio::fs::set_permissions(path(&root, STAGING), std::fs::Permissions::from_mode(0o600)) + .await + .unwrap(); + assert_eq!( + lease.container_records(deadline()).await.unwrap(), + vec![entry] + ); + tokio::fs::write(path(&root, PUBLISHED), b"partial") + .await + .unwrap(); + assert!(lease.container_records(deadline()).await.is_err()); +} +#[tokio::test] +async fn test_should_fail_before_publication_with_expired_budget() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + assert!(matches!( + lease + .prepare_container(record(lease.run_id()), Instant::now()) + .await, + Err(DirectoryLeaseError::Deadline) + )); + assert!(!tokio::fs::try_exists(path(&root, PUBLISHED)).await.unwrap()); + assert!( + lease + .container_records(Instant::now() + Duration::from_secs(61)) + .await + .is_err() + ); +} + +#[tokio::test] +async fn test_should_keep_transaction_and_lease_owned_after_waiter_abandonment() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let (entered, observed) = tokio::sync::oneshot::channel(); + let (release, barrier) = std::sync::mpsc::channel(); + let entry = record(lease.run_id()); + let worker_lease = lease.clone(); + let task = tokio::spawn(async move { + worker_lease + .journal(deadline(), move |records| { + records.push(entry); + entered.send(()).unwrap(); + barrier.recv().unwrap(); + Ok(()) + }) + .await + }); + observed.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let check = lease.clone(); + let mut read = tokio::spawn(async move { check.container_records(deadline()).await }); + assert!( + tokio::time::timeout(Duration::from_millis(30), &mut read) + .await + .is_err() + ); + drop(lease); + assert!(matches!( + DirectoryLease::acquire(root.path().into(), deadline()).await, + Err(DirectoryLeaseError::InUse) + )); + release.send(()).unwrap(); + // The cancelled worker must finish/dispose its transaction before this read. + assert_eq!(read.await.unwrap().unwrap(), []); + let _next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_should_keep_authoritative_record_when_erase_cannot_publish() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let entry = record(lease.run_id()); + lease + .prepare_container(entry.clone(), deadline()) + .await + .unwrap(); + let outside = root.path().join("outside"); + tokio::fs::write(&outside, b"keep").await.unwrap(); + symlink(&outside, path(&root, STAGING)).unwrap(); + assert!( + lease + .settle_container(entry.clone(), deadline()) + .await + .is_err() + ); + let bytes = tokio::fs::read(path(&root, PUBLISHED)).await.unwrap(); + assert_eq!( + decode(&bytes, lease.owner_id()).unwrap(), + vec![entry.clone()] + ); + tokio::fs::remove_file(path(&root, STAGING)).await.unwrap(); + lease.settle_container(entry, deadline()).await.unwrap(); + assert_eq!(lease.container_records(deadline()).await.unwrap(), []); +} + +#[tokio::test] +async fn test_should_reject_overlong_budget_before_waiting_for_transaction() { + let root = tempfile::tempdir().unwrap(); + let lease = lease(&root).await; + let _held = lease.0.transaction.acquire().await.unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(50), + lease.container_records(Instant::now() + Duration::from_secs(61)), + ) + .await; + assert!(matches!( + result, + Ok(Err(DirectoryLeaseError::Configuration)) + )); +} diff --git a/crates/rustack-lambda-core/src/docker/plan.rs b/crates/rustack-lambda-core/src/docker/plan.rs index a0168caa..3d7e092c 100644 --- a/crates/rustack-lambda-core/src/docker/plan.rs +++ b/crates/rustack-lambda-core/src/docker/plan.rs @@ -185,6 +185,12 @@ impl fmt::Debug for EnvironmentPlan { } } impl EnvironmentPlan { + pub(super) fn image_id(&self, role: ContainerRole) -> &str { + match role { + ContainerRole::Relay => self.relay.resolved().metadata().image_id.as_str(), + ContainerRole::Function => self.bodies.function.image.as_deref().unwrap_or_default(), + } + } pub(super) fn warm_key(&self) -> &str { &self.warm_key } diff --git a/crates/rustack-lambda-core/src/docker/recovery.rs b/crates/rustack-lambda-core/src/docker/recovery.rs new file mode 100644 index 00000000..14722426 --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery.rs @@ -0,0 +1,134 @@ +//! Resumable historical container cleanup; unknown absence is never settlement. + +use http::StatusCode; +use tokio::time::Instant; + +use super::{ + DirectoryLease, DockerError, DockerPort, EnvironmentError, check_deadline, + ownership::ContainerRecord, +}; + +/// Container-only recovery accounting, not a complete backend readiness proof. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ContainerRecovery { + /// Records durably erased during this pass. + pub settled: usize, + /// Records still retained, including current-run records never touched here. + pub remaining: usize, +} + +/// Reconcile previous-run recorded containers using exact identity and ID proofs. +/// +/// Unknown404 responses retain the record; unresolved functions retain their +/// relay. Every destructive request follows durable publication of the full ID. +/// Cancellation or errors preserve records for a subsequent pass. This function +/// never creates containers and never touches current-run records. +/// +/// # Errors +/// Returns invalid/expired budgets, unsafe/corrupt/unavailable ownership storage, +/// daemon transport/protocol failures or foreign resource identity. No error +/// grants readiness, and previous successfully settled records remain settled. +/// +/// ```no_run +/// # async fn recover(directory: &rustack_lambda_core::docker::DirectoryLease, docker: &rustack_lambda_core::docker::DockerPort) -> Result<(), rustack_lambda_core::docker::EnvironmentError> { +/// let report = rustack_lambda_core::docker::reconcile_containers(directory, docker, +/// tokio::time::Instant::now() + std::time::Duration::from_secs(30)).await?; +/// assert_eq!(report.remaining, 0); +/// // Image aliases and the other startup proofs still require qualification. +/// # Ok(()) } +/// ``` +pub async fn reconcile_containers( + directory: &DirectoryLease, + docker: &DockerPort, + deadline: Instant, +) -> Result { + check_deadline(deadline)?; + let mut records = directory.container_records(deadline).await?; + records.sort_by_key(|record| (!record.function(), record.environment)); + let mut report = ContainerRecovery { + settled: 0, + remaining: records.len(), + }; + let mut blocked = Vec::new(); + for record in records { + check_deadline(deadline)?; + if record.run == directory.run_id() { + continue; + } + if !record.function() && blocked.contains(&record.environment) { + continue; + } + if Box::pin(recover(directory, docker, record.clone(), deadline)).await? { + report.settled += 1; + report.remaining -= 1; + } else if record.function() { + blocked.push(record.environment); + } + } + check_deadline(deadline)?; + Ok(report) +} + +async fn recover( + directory: &DirectoryLease, + docker: &DockerPort, + mut record: ContainerRecord, + deadline: Instant, +) -> Result { + let key = record.known_key()?.unwrap_or(record.key()?); + let inspected = match docker.inspect_container(&key, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) if record.known.is_some() => { + directory.settle_container(record, deadline).await?; + return Ok(true); + } + Err(DockerError::Status(StatusCode::NOT_FOUND)) => return Ok(false), + result => result?, + }; + let id = record.owned(directory.owner_id(), &inspected)?; + directory + .know_container(record.clone(), id.clone(), deadline) + .await?; + // Re-read the durable identity so subsequent mutation never relies on an + // unpersisted local success or a name which can be reused. + record = directory + .container_records(deadline) + .await? + .into_iter() + .find(|saved| { + saved.environment == record.environment && saved.function() == record.function() + }) + .ok_or(DockerError::Quarantined)?; + let current = match docker.inspect_container(&id, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_container(record, deadline).await?; + return Ok(true); + } + result => result?, + }; + record.owned(directory.owner_id(), ¤t)?; + match current.state.and_then(|state| state.running) { + Some(true) => { + let _ = docker.kill_container(&id, deadline).await; + let exit = docker.wait_container(&id, deadline).await?; + if exit.error.is_some() { + return Err(DockerError::Quarantined.into()); + } + } + Some(false) => {} + None => return Err(DockerError::Protocol.into()), + } + check_deadline(deadline)?; + let _ = docker.remove_container(&id, deadline).await; + match docker.inspect_container(&id, deadline).await { + Err(DockerError::Status(StatusCode::NOT_FOUND)) => { + directory.settle_container(record, deadline).await?; + Ok(true) + } + Ok(_) => Ok(false), + Err(error) => Err(error.into()), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-lambda-core/src/docker/recovery/tests.rs b/crates/rustack-lambda-core/src/docker/recovery/tests.rs new file mode 100644 index 00000000..593d7ffd --- /dev/null +++ b/crates/rustack-lambda-core/src/docker/recovery/tests.rs @@ -0,0 +1,346 @@ +use std::{sync::Arc, time::Duration}; + +use serde_json::{Value, json}; +use tokio::time::timeout; + +use super::*; +use crate::docker::{ + EnvironmentPlan, + plan::{ + ContainerRole, + tests::{image_with, policy, revision}, + }, + tests::Daemon, +}; + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(10) +} +async fn fixture() -> ( + tempfile::TempDir, + DirectoryLease, + Vec, + Vec, +) { + let root = tempfile::tempdir().unwrap(); + let directory = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + let mut records = Vec::new(); + let mut inspected = Vec::new(); + for role in [ContainerRole::Function, ContainerRole::Relay] { + let record = ContainerRecord::from_plan(&plan, role).unwrap(); + directory + .prepare_container(record.clone(), deadline()) + .await + .unwrap(); + let role = if record.function() { + "function" + } else { + "relay" + }; + let id = if record.function() { "a" } else { "b" }.repeat(64); + inspected.push(json!({"Id": id,"Name":format!("/{}",record.key().unwrap().as_str()),"Config":{"Image":plan.image_id(if record.function() {ContainerRole::Function} else {ContainerRole::Relay}),"Labels":{ + "rustack.lambda.ownerId":directory.owner_id().to_string(),"rustack.lambda.runId":directory.run_id().to_string(),"rustack.lambda.environmentId":plan.environment_id().to_string(),"rustack.lambda.functionResourceId":plan.revision().resource_id.to_string(),"rustack.lambda.role":role}},"State":{"Running":false}})); + records.push(record); + } + drop(plan); + drop(directory); + let next = DirectoryLease::acquire(root.path().into(), deadline()) + .await + .unwrap(); + (root, next, records, inspected) +} +async fn respond(daemon: &mut Daemon, value: &Value) { + daemon + .next() + .await + .reply(200, &serde_json::to_vec(value).unwrap()) + .await; +} +#[tokio::test] +async fn test_should_keep_unknown_absence_and_block_relay_then_recover_late_commit() { + let (_root, directory, records, inspected) = fixture().await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let recover = reconcile_containers(&directory, &port, deadline()); + let serve = async { + let request = daemon.next().await; + assert!(request.head.contains("function")); + request.reply(404, b"{}").await; + }; + let (result, ()) = tokio::join!(recover, serve); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 0, + remaining: 2 + } + ); + assert_eq!( + directory.container_records(deadline()).await.unwrap(), + records + ); + let port = wire.port(); + let recover = reconcile_containers(&directory, &port, deadline()); + let serve = async { + for value in &inspected { + respond(&mut daemon, value).await; + // The full ID must already be durable before any destructive call. + let request = daemon.next().await; + let saved = directory.container_records(deadline()).await.unwrap(); + assert!(saved.iter().any(|record| record.known.is_some())); + assert!(request.head.contains(value["Id"].as_str().unwrap())); + request + .reply(200, &serde_json::to_vec(value).unwrap()) + .await; + let remove = daemon.next().await; + assert!(remove.head.starts_with("DELETE ")); + remove.reply(204, b"").await; + daemon.next().await.reply(404, b"{}").await; + } + }; + let (result, ()) = tokio::join!(recover, serve); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 2, + remaining: 0 + } + ); + assert_eq!(directory.container_records(deadline()).await.unwrap(), []); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_settle_only_durable_known_absence() { + let (_root, directory, records, _) = fixture().await; + for record in &records { + directory + .know_container(record.clone(), "a".repeat(64).parse().unwrap(), deadline()) + .await + .unwrap(); + } + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!(reconcile_containers(&directory, &port, deadline()), async { + for _ in 0..2 { + let request = daemon.next().await; + assert!(request.head.contains(&"a".repeat(64))); + request.reply(404, b"{}").await; + } + }); + assert_eq!(result.unwrap().remaining, 0); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[rstest::rstest] +#[case("/Name")] +#[case("/Id")] +#[case("/Config/Image")] +#[case("/Config/Labels/rustack.lambda.ownerId")] +#[case("/Config/Labels/rustack.lambda.runId")] +#[case("/Config/Labels/rustack.lambda.environmentId")] +#[case("/Config/Labels/rustack.lambda.functionResourceId")] +#[case("/Config/Labels/rustack.lambda.role")] +#[tokio::test] +async fn test_should_never_mutate_foreign_or_malformed_resources(#[case] field: &str) { + let (_root, directory, _, mut inspected) = fixture().await; + *inspected[0].pointer_mut(field).unwrap() = json!("foreign"); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let port = wire.port(); + let (result, ()) = tokio::join!( + reconcile_containers(&directory, &port, deadline()), + respond(&mut daemon, &inspected[0]) + ); + assert!(result.is_err()); + assert_eq!( + directory.container_records(deadline()).await.unwrap().len(), + 2 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} +#[tokio::test] +async fn test_should_never_reconcile_current_run_records() { + let (_root, directory, records, _) = fixture().await; + for record in records { + directory + .settle_container(record, deadline()) + .await + .unwrap(); + } + let plan = EnvironmentPlan::new( + directory.clone(), + image_with("1000", vec![]), + Arc::new(revision()), + policy(), + ) + .unwrap(); + directory + .prepare_container( + ContainerRecord::from_plan(&plan, ContainerRole::Relay).unwrap(), + deadline(), + ) + .await + .unwrap(); + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + assert_eq!( + reconcile_containers(&directory, &wire.port(), deadline()) + .await + .unwrap() + .remaining, + 1 + ); + assert!( + timeout(Duration::from_millis(20), daemon.requests.recv()) + .await + .is_err() + ); + wire.shutdown().await.unwrap(); + daemon.close().await; +} + +#[tokio::test] +#[ignore = "requires explicit Docker socket and matching runtime image"] +async fn test_should_reconcile_real_previous_run_containers_from_durable_intents() +-> anyhow::Result<()> { + use anyhow::{Context, ensure}; + + use crate::{ + artifact::{Architecture, DeploymentArtifact}, + docker::{DockerWire, ImageLeaseConfig, ImageLeaseManager, ImagePinScope}, + }; + let socket = + std::env::var("RUSTACK_RUNTIME_DOCKER_SOCKET").context("explicit Docker socket")?; + let architecture = match std::env::var("RUSTACK_RUNTIME_ARCH")?.as_str() { + "arm64" => Architecture::Arm64, + "amd64" => Architecture::X86_64, + _ => anyhow::bail!("unknown fixture architecture"), + }; + let image = std::env::var("RUSTACK_RUNTIME_IMAGE")?.parse()?; + let limit = || Instant::now() + Duration::from_secs(60); + let mut wire = DockerWire::connect(format!("unix://{socket}").parse()?, limit()).await?; + let root = tempfile::tempdir()?; + let directory = DirectoryLease::acquire(root.path().into(), limit()).await?; + let previous_run = directory.run_id(); + let mut images = ImageLeaseManager::start( + wire.port(), + ImagePinScope::new(directory.clone()), + ImageLeaseConfig::default(), + )?; + let setup = async { + let image = images + .port() + .prepare(&image, architecture, None, limit()) + .await?; + let mut revision = revision(); + revision.architecture = architecture; + revision.artifact = DeploymentArtifact::Image(image.clone()); + let plan = EnvironmentPlan::new(directory.clone(), image, Arc::new(revision), policy())?; + for role in [ContainerRole::Relay, ContainerRole::Function] { + directory + .prepare_container(ContainerRecord::from_plan(&plan, role)?, limit()) + .await?; + let created = match role { + ContainerRole::Relay => plan.create_relay(&wire.port(), limit()).await?, + ContainerRole::Function => plan.create_function(&wire.port(), limit()).await?, + }; + if matches!(role, ContainerRole::Relay) { + wire.port() + .start_container(&created.id.parse()?, limit()) + .await?; + } + } + anyhow::Ok(()) + } + .await; + let image_cleanup = images.shutdown(limit()).await; + drop(images); + drop(directory); + let recovery = async { + let next = DirectoryLease::acquire(root.path().into(), limit()).await?; + ensure!(next.run_id() != previous_run, "run identity did not change"); + let report = reconcile_containers(&next, &wire.port(), limit()).await?; + ensure!( + next.container_records(limit()).await?.is_empty(), + "durable records remain" + ); + anyhow::Ok(report) + } + .await; + if !recovery.as_ref().is_ok_and(|report| report.remaining == 0) + || image_cleanup.is_err() + || setup.is_err() + { + // Any post-intent failure retains the evidence, including failed lease + // reacquisition while an image recovery owner still holds exclusion. + let _ = root.keep(); + } + let wire_cleanup = wire.shutdown().await; + setup?; + image_cleanup?; + let report = recovery?; + ensure!( + report.settled == 2 && report.remaining == 0, + "historical containers were not both settled" + ); + wire_cleanup?; + Ok(()) +} + +#[tokio::test] +async fn test_should_resume_known_identity_after_recovery_waiter_is_aborted() { + let (_root, directory, _, inspected) = fixture().await; + let mut daemon = Daemon::new(); + let mut wire = daemon.wire().await; + let lease = directory.clone(); + let port = wire.port(); + let task = tokio::spawn(async move { reconcile_containers(&lease, &port, deadline()).await }); + respond(&mut daemon, &inspected[0]).await; + let held = daemon.next().await; + assert!(held.head.contains(&"a".repeat(64))); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + held.reply(200, &serde_json::to_vec(&inspected[0]).unwrap()) + .await; + let saved = directory.container_records(deadline()).await.unwrap(); + assert!( + saved + .iter() + .any(|record| record.function() && record.known.is_some()) + ); + let port = wire.port(); + let (result, ()) = tokio::join!(reconcile_containers(&directory, &port, deadline()), async { + let known = daemon.next().await; + assert!(known.head.contains(&"a".repeat(64))); + known.reply(404, b"{}").await; + daemon.next().await.reply(404, b"{}").await; + }); + assert_eq!( + result.unwrap(), + ContainerRecovery { + settled: 1, + remaining: 1 + } + ); + assert!(!directory.container_records(deadline()).await.unwrap()[0].function()); + wire.shutdown().await.unwrap(); + daemon.close().await; +} diff --git a/specs/index.md b/specs/index.md index 49ba019e..7618dc86 100644 --- a/specs/index.md +++ b/specs/index.md @@ -4,9 +4,9 @@ ## Step Functions and Lambda Runtime — new specification set -Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. +Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, historical reconciliation, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-20-lambda-runtime-design.md b/specs/rustack-workflows-20-lambda-runtime-design.md index 9aae2e6b..04beac91 100644 --- a/specs/rustack-workflows-20-lambda-runtime-design.md +++ b/specs/rustack-workflows-20-lambda-runtime-design.md @@ -531,7 +531,7 @@ Labels include persistent data-directory ownerId, runId, environmentId and funct Validate the operator path before IO: absolute, nonempty,≤4,096 encoded OS bytes, no NUL or parent components, and at most128 path components. Normalize the accepted component sequence before IO, removing redundant separators/current-directory components so a trailing slash or /. cannot bypass final-component NOFOLLOW. Existing ancestor aliases such as macOS /var→/private/var remain usable, but the selected root itself must not be a symlink. Validate both logical and canonical existing ancestor chains before creating anything: directory/symlink owners must be the effective UID or root, and directories must not permit group/other writes unless protected by a trusted-owner sticky bit. Operator ACLs and namespace/ancestor entries must not grant untrusted rename/unlink access or be replaced while leases exist. Create missing components with restrictive permissions relative to the pinned trusted ancestor, validating each opened component with NOFOLLOW. Every successful acquisition syncs the resolved root-to-filesystem-root ancestor chain, even on retries through already-existing paths; stop the sync barrier when crossing a device boundary, after validating the remaining trusted path. A failed/expired prior creation therefore cannot hide an unsynced parent on retry. Pin the root descriptor, create/open `.rustack-lambda-docker` relative to it, and require an actual directory owned by the effective UID with no group/other permissions. Use the already-resolved nix0.31.3 safe `openat`/`mkdirat`/`renameat` wrappers with only fs/user features; no project unsafe or raw-FD conversion. NOFOLLOW/DIRECTORY/CLOEXEC and NONBLOCK where appropriate prevent symlink/FIFO/device redirection. The managed namespace is exclusively Rustack-controlled; operators must not replace its files while leases exist. This does not promise isolation from an adversary with the same UID or daemon access. -The namespace has three fixed children: `lease.lock` (permanent, empty regular file), `owner.v1` (authoritative immutable identity), and `owner.pending` (bounded initialization staging). Managed regular files must have effective-UID ownership, link count1 and no group/other permissions; reject unexpected file kinds or oversized/malformed contents. Lock `lease.lock` without truncating or unlinking it. A competing independent descriptor/process receives InUse. Never implement exclusivity with a PID file, an in-process registry or a replaceable lock inode. +Identity acquisition uses three fixed namespace children (the container journal in §4.8 adds its own two fixed files): `lease.lock` (permanent, empty regular file), `owner.v1` (authoritative immutable identity), and `owner.pending` (bounded initialization staging). Managed regular files must have effective-UID ownership, link count1 and no group/other permissions; reject unexpected file kinds or oversized/malformed contents. Lock `lease.lock` without truncating or unlinking it. A competing independent descriptor/process receives InUse. Never implement exclusivity with a PID file, an in-process registry or a replaceable lock inode. `owner.v1` contains exactly56 bytes: eight-byte `RSDKOWN1` magic, the16 raw bytes of a nonnil UUID, and a32-byte SHA-256 checksum over the first24 bytes. The checksum detects accidental byte corruption; it is not authentication against a same-UID writer. Existing records are read through bounded descriptor IO and rejected on malformed size/magic/UUID/checksum; never regenerate a damaged published identity. When absent under the lock, validate/reuse the fixed pending regular file (≤56 bytes), write a new complete identity, sync it, atomically rename relative to the directory descriptor, and sync both the namespace and data-root directories before issuing the lease. Repeat these syncs even when the owner record already exists, so a prior failed publication sync cannot be bypassed on retry. A first-initialization crash before publication may leave only a bounded pending file; an unsuccessful acquisition never authorizes resource creation. Published identity survives ordinary restart, while run UUID changes on every new successful acquisition. The OS releases the lock after the last owning descriptor closes or the process exits. @@ -725,6 +725,32 @@ The minimal probe HTTP connection stays owned inside its request future (no deta The coordinator must combine these proofs with historical uncertain-effect reconciliation, allowed pulls, endpoint reachability and final configuration/capability acceptance. Neither QualifiedNetwork nor EnvironmentProbe alone implements DockerReadiness or enables CLI Docker. Tests include strict report/parser mismatches, exact/no-effect network requests, actual none and operator bridge probes, failed/unavailable resource observations and retained cancellation/cleanup. A fake report is never counted as actual resource enforcement evidence. +### 4.8 Durable container intents and restart reconciliation + +Before each environment create can reach Docker, persist its preallocated identity under the retained DirectoryLease. A bounded descriptor-relative snapshot (`containers.v1`, staging `containers.pending`) contains at most64 container records. It stores no invocation payload, environment variables, credentials or paths. Each record contains run/environment/function-resource UUIDs, relay/function role, immutable image SHA-256 and an optional full container ID. Names and authoritative labels are derived from these fields. All UUIDs are nonnil; duplicate environment/role keys and inconsistent identities are invalid. Fixed binary encoding is `RSDKCTR1`, owner UUID, one-byte count,114-byte records, then SHA-256 over preceding bytes. A record is three16-byte UUIDs, one-byte role(function0/relay1),32-byte image digest, one-byte ID-presence flag and32-byte ID (zero when absent). The maximum is7,353 bytes. Corrupt published state is rejected, never reset. + +Serialize snapshot transactions with one bounded async permit per lease, then use the existing four-worker blocking budget. Workers retain the lease and transaction permit through cancellation/completion; no detached filesystem write can overtake cleanup. Apply existing private regular-file/no-follow/link/owner checks to both files. Write/truncate only staging, sync staging, rename relative to the pinned namespace, then sync the namespace. Validate and sync authoritative state on reads, including retries after failed publication sync. An interrupted bounded staging file does not override a published snapshot. All operations share an absolute positive deadline of at most60s and typed, redacted ownership errors. + +The environment worker registers an uncertain in-memory receipt before awaiting intent publication. It sends no create if publication fails. Positive exact inspection is durably recorded as a known full ID before removal can occur. Cleanup erases a record only after a definitely-unissued receipt, pinned source-defined no-effect response, or confirmed absence of that known ID. A failed erase remains quarantined and retains allocation ownership. Thus process death before issue can conservatively leave an unknown intent, but cannot leave an issued create without an intent. + +```text +Environment owner Directory transaction worker Docker + │ register uncertain │ │ + │ publish intent ──────────────────▶│ staging sync → rename → sync │ + │ durable acknowledgement ◀────────│ │ + │ create ────────────────────────────────────────────────────────▶│ + │ exact inspect ◀─────────────────────────────────────────────────│ + │ publish known ID ────────────────▶│ durable replacement │ + │ kill/wait/remove by ID ─────────────────────────────────────────▶│ + │ confirmed known-ID absence ◀────────────────────────────────────│ + │ erase intent ───────────────────▶│ durable replacement │ + │ release reservation only after joined IO and durable erase │ +``` + +`reconcile_containers(directory, docker, deadline)` is a resumable startup operation, not a readiness grant. Read the durable snapshot; skip current-run records and process historical functions before their relays. An unresolved function blocks its matching relay. For an unknown record, require positive exact name, owner/run/environment/resource/role labels, immutable Config.Image and full64-character lowercase container ID before persisting the ID. A404/empty list cannot discharge an unknown record. Known-ID404 is terminal; otherwise revalidate identity before kill/wait/remove and require subsequent known-ID404 before durable erase. Foreign/malformed resources are never mutated. Cancellation, failure or deadline leaves the record for retry. Return remaining/settled counts; remaining records prevent startup admission. Do not reset, prune, or remove foreign resources. This covers recorded environment containers only: historical image aliases, other mutation classes, production coordinator and complete platform acceptance remain separate required gates. + +Tests cover restart identity and ordering, unknown404 followed by late commit, known absence, foreign labels/image/name/ID, failed/cancelled publication, concurrent transactions, bounded/corrupt/unsafe files, repeated cleanup, and a failed erase retaining ownership. Actual Docker qualification must use real persisted records, reacquire the directory under a new run and verify exact owned-resource cleanup. Existing lifecycle/probe/SDK suites must still pass. + ## 5. Isolation and resource limits - Initial Docker transport is a local Unix socket only. Disable implicit TCP/SSH fallback. Explicit docker configuration requires a startup daemon/version/relay-image/resource-limit probe; failure aborts startup. Default remains disabled. Auto does not acquire a new Docker fallback; Image on native/squib/auto fails explicitly. diff --git a/specs/rustack-workflows-60-operations-design.md b/specs/rustack-workflows-60-operations-design.md index f1e1e17d..26fd55e3 100644 --- a/specs/rustack-workflows-60-operations-design.md +++ b/specs/rustack-workflows-60-operations-design.md @@ -97,6 +97,14 @@ Startup must use [runtime §4.7](./rustack-workflows-20-lambda-runtime-design.md Current actual execution evidence is on cgroup-v2 arm64 VZ and amd64/QEMU daemons. v1 normalization/private namespace behavior has parser and allocation-body coverage; an actual v1 daemon lane is still unmeasured. Network/resource receipts are implemented components; historical uncertain effects, registry pulls, endpoint reachability and the production startup coordinator remain mandatory before CLI Docker enablement. +### 1.5 Historical container records + +[Runtime §4.8](./rustack-workflows-20-lambda-runtime-design.md#48-durable-container-intents-and-restart-reconciliation) defines the bounded containers.v1 snapshot and its safe staging file. Environment creation durably records identity before daemon issuance; known IDs are persisted before deletion. Failed durable erasure retains allocation capacity even when Docker returned a no-effect status or the container is absent. Never remove these files to bypass an unresolved startup: unknown absence can precede a late create. + +Before ordinary admission, call reconcile_containers under the same DirectoryLease. It processes historical functions before their relays, leaves current-run records untouched and returns remaining/settled counts. Unknown404 remains unresolved; a positive exact identity is pinned durably before ID-only retirement. Repeating a cancelled/failed pass resumes the saved state. Foreign identity or corrupt/unsafe files fail closed. This operation covers recorded environment containers; private image aliases, other mutation classes, allowed pulls and the complete coordinator remain required before production readiness. + +`make test-lambda-recovery` covers bounded/corrupt files, restart transactions, cancellation, foreign resources and delayed commits. `make test-lambda-recovery-docker WORKFLOWS_DOCKER_SOCKET= WORKFLOWS_ARCH=arm64` provisions actual owned containers, releases/reacquires the directory under a new run and retires the historical pair. Failed actual qualification preserves durable ownership evidence rather than deleting the temporary data root. + ## 2. Deployment topology and readiness | Topology | Control path | Handler SDK path | Required test | diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 24e0df0e..248d7dcf 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -156,3 +156,9 @@ Do not introduce a benchmark crate early. First measure these integration worklo - ← [all component contracts](./rustack-workflows-00-prd.md#7-reading-order-and-cross-references) - → [roadmap exits](./rustack-workflows-80-roadmap.md), [implementation gates](./rustack-workflows-90-impl-plan.md) - ↔ [operations](./rustack-workflows-60-operations-design.md), [decisions](./rustack-workflows-99-decisions.md) + +## Durable container restart qualification + +The §4.8/D38 container journal must be exercised through the real EnvironmentManager create/cleanup path: a published intent exists when the first create arrives; failed durable erase retains the allocation until repaired. Transaction abandonment retains its lease/serial permit until its blocking worker completes. Reject overlong budgets before waiting for that permit. Verify maximum64 records/7,353 bytes, corruption/checksum/owner/UUID/duplicate identity, unsafe staging/published inodes, torn safe staging and authoritative-state preservation. + +Scripted restart cases require unknown404 plus blocked relay, later positive commit and exact function-before-relay cleanup; known-ID404; separately mutated name/ID/image and every authoritative label; current-run nonmutation; and an aborted recovery waiter leaving a durable known identity for the next pass. `make test-lambda-recovery` runs these independently of Docker. The explicit `test-lambda-recovery-docker` target requires socket/architecture/image, leaves two real journaled containers across directory reacquisition, then verifies both durable records are settled. Actual recovery tests preserve evidence and close the wire on every post-intent failure. Requalify actual pool/provider/HTTP/wire suites after integration. This is container historical-recovery evidence, not image-alias reconciliation, production startup or complete platform acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 73f9f6b2..49fd36c8 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -154,6 +154,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 supervised pool invocation | Implemented/reviewed; Executor consumer remains required | §4.5/D35 expose synchronous start, cancel-safe response wait and separate resumable cancel_and_wait with an independent cleanup deadline. Per-call settlement senders stay beside Retirement records above JoinSet through worker/root recovery. Nine new regressions,27 total pool cases, prove delayed/expired cleanup, waiter loss, no-allocation rejection, worker/root failure, warm success and missing-producer uncertainty. W86 observer rejection cleanup is fixed; final independent review has no remaining finding. Workspace2,286 tests/408 separately gated; strict boundary870 units+204 docs (1,074 total)/14 Docker-gated; standard build/test/fmt/Clippy/rustdoc and strict production checks pass. Each arm64 VZ and amd64/QEMU daemon explicitly passes7 owner/pool real-image cases, including per-call awaited retirement before global shutdown, container404 and replacement counter1 while admission stays open. Post-run owned containers/internal aliases are absent. This does not close Executor/provider/startup or M0 acceptance. | | 1.3 Docker Executor/provider | Implemented/reviewed under trusted runtime injection; production startup remains | §4.6/D36 consume one PoolInvocation with independent cleanup settlement, immutable plan/metadata validation, raw outcomes, resolved version, requested alias ARN and one base64 tail encoding. W87–W91 are fixed/rechecked.15 Executor +16 handler cases pass, including held reserved concurrency, expired cleanup/failed quiescence and resumed physical retirement. Workspace2,313 tests/409 separately gated; core611 units/141 docs; strict boundary897 units+208 docs (1,105 total)/15 Docker-gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU daemon passes actual boto3→HTTP→provider→Docker deploy/publish/alias/Invoke/error/timeout/replacement plus seven existing owner/pool cases (eight per daemon). Controlled fixture readiness is not startup/probe/native-x86 evidence; post-run owned containers/internal aliases are absent. | | 1.3/1.4 startup probes | Implemented/reviewed; no readiness grant | §4.7/D37 validate exact user bridge/none selections, bind full IDs, and execute the pinned Rust probe under retained EnvironmentManager ownership. Private cgroup namespaces and process root-membership checks cover v1/v2 parser semantics; exact kernel limits/flags, scoped proof matching, caller abandonment, independent cleanup and final deadline/live-input checks are enforced. W92–W94 closed;57 focused cases pass. Workspace2,370/411 gated, boundary954 units+217 docs (1,171 total)/17 gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU cgroup-v2 daemon passes17 distinct actual probe/SDK/pool/wire/runtime cases; scoped resources and the two dedicated probe bridges are removed. Actual v1/native-x86, historical effects, pulls, endpoint/config/capability coordinator acceptance remain open. | +| 1.3/1.4 historical container effects | Implemented/reviewed; other mutation classes remain | §4.8/D38 add bounded durable per-container intents before creation and known IDs before deletion. The restart pass preserves unknown absence, blocks unresolved relays, validates exact identities and resumes after cancellation. W95–W96 are fixed and independently rechecked.24 new scripted/unit cases pass, including create-before-effect and durable-erase reservation barriers. Workspace2,394/412 gated; boundary978 units+218 docs (1,196 total)/18 gated; standard/strict/audit/deny gates pass. Each arm64 VZ and amd64/QEMU daemon passes11 distinct actual recovery/pool/SDK/wire cases; scoped container/private-alias queries are empty afterward. Actual directory reacquisition and historical pair retirement pass on both. This does not cover historical image aliases or enable production startup. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | -P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Next implement historical uncertain-effect reconciliation and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. +P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index ee66bd7f..ae1d4004 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -451,6 +451,19 @@ Each retained Engine29.5.2 daemon passes17 explicitly selected actual cases with Production startup still requires historical uncertain-effect reconciliation, allowed pulls, function-visible endpoint reachability, final configuration/capability publication and complete platform qualification. Neither proof opens readiness by itself. M0 and P2–P5/ASL remain required by the full objective. +## P1 durable container intent and restart recovery review + +§4.8/D38 adds descriptor-relative bounded atomic snapshots, environment create/cleanup integration and resumable prior-run container retirement. Unknown absence remains unresolved; exact positive identity and durable full ID precede destructive requests. This is the container portion of historical reconciliation and does not grant production readiness. Independent final source review finds no remaining in-slice issue. + +| Finding | Priority / source | Correction | Verification | +|---|---|---|---| +| W95 | P2; independent review, `crates/rustack-lambda-core/src/docker/ownership/journal.rs:144` | Reject positive budgets above60s before waiting for the transaction permit; retain worker-side postqueue checks | Deterministic held-permit test requires immediate Configuration instead of waiting | +| W96 | P2; independent review, `crates/rustack-lambda-core/src/docker/recovery/tests.rs:277` | Capture reacquisition/recovery errors before propagation, retain TempDir on every post-intent failure and always attempt wire shutdown | Final-source independent review confirms the failure-path preservation; actual historical pair recovery passes on both retained daemons | + +Twenty-four new unit/scripted cases cover11 journal transactions/codecs/filesystem cases,12 historical recovery cases and one composed environment durable-erase barrier. Docker module332 tests pass; the actual restart case is separately gated. Final workspace build/test/fmt/Clippy/rustdoc pass with2,394 tests/zero failed/412 separately gated cases. Core670 units/150 doctests; strict six-library boundary978 units+218 doctests (1,196 total)/18 gated pass without new ignores. Audit scans474 locked crates with no vulnerabilities; deny passes all four checks with the existing policy notices. + +Each retained Engine29.5.2 daemon (arm64 VZ and amd64/QEMU) explicitly passes11 distinct actual cases: recovery1, owner/pool7, boto3/HTTP/provider/executor1 and wire2, all with zero ignored. The recovery fixture leaves a real stopped function and running relay across directory reacquisition, then verifies ordered retirement and durable erasure. Scoped owner-container and private-image-alias queries are empty after the suites. QEMU is not native x86 qualification. The existing network/kernel/shared-runtime evidence remains separately scoped; this change does not grant startup readiness or cover historical private image aliases, allowed pulls, endpoint/configuration coordination or P2–P5. The12 feature/index documents resolve265 local references. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 41845823..f5ae45f7 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -338,3 +338,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: probes consume the same policy/runtime path they qualify and retain existing cleanup ownership. Resource and network proofs remain independent from historical mutation reconciliation and cannot open readiness by themselves. - **Pinned by**: [runtime §4.7](./rustack-workflows-20-lambda-runtime-design.md#47-startup-network-qualification-and-actual-resource-probe), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. Promote the already-resolved runtime serde_json dev dependency to a normal workspace dependency for bounded typed report serialization; no new library/version is introduced. + +## D38 — Persist container intent before possible creation and identity before deletion + +- **Context**: in-memory uncertain-create receipts do not survive process death; an absent name cannot exclude a late daemon commit. +- **Alternatives considered**: label-only scans miss pending creation; negative lookup as proof loses ownership; always rejecting dirty restarts cannot reclaim positively identified resources; an unbounded append log adds retention and replay complexity. +- **Decision**: use a bounded atomic directory snapshot of container identities, publish before create, persist a positively verified full ID before removal, and erase only after defensible settlement. Historical unknown absence remains unresolved and blocks its relay. +- **Why**: the same retained directory capability protects filesystem transactions and daemon ownership; interrupted recovery is repeatable without replaying create or touching foreign resources. This is a container-specific part of full reconciliation, not production readiness. +- **Pinned by**: [runtime §4.8](./rustack-workflows-20-lambda-runtime-design.md#48-durable-container-intents-and-restart-reconciliation), [implementation plan](./rustack-workflows-90-impl-plan.md). +- **Date**: 2026-10-26. Reuse existing nix/sha2/hex dependencies and the bounded ownership worker pool. From 9937246705a6c707fe68ea2e1f12c2f8f9b8943f Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 05:49:15 -0700 Subject: [PATCH 032/106] phase 2: generate Step Functions model and exact JSON timestamps Implement API section 1.1 and D40 from the pinned 37-operation Smithy model. Generate exact numeric millisecond epoch adapters, strict documented nonexhaustive DTOs, typed builders, required collections and redacted diagnostics. Add reproducible generation, standalone tool checks and independent tool dependency audit/deny. Independent review fixes W97-W100. Five generator policy tests, 34 generated-model cases, one executed doctest, regeneration and strict model/tool checks pass. Current-worktree standard gates pass with 2447 tests; both dependency graphs audit/deny clean. Per user direction, advance Step Functions HTTP/core/ASL before further Docker-image work; the separate Docker checkpoint is retained and final image/platform acceptance remains required. --- Cargo.lock | 13 + Cargo.toml | 1 + Makefile | 27 + codegen/Cargo.lock | 48 +- codegen/Cargo.toml | 3 + codegen/services/stepfunctions.toml | 2 + codegen/src/codegen.rs | 248 +- codegen/src/config.rs | 30 +- codegen/src/main.rs | 66 +- codegen/src/model.rs | 33 +- codegen/src/shapes.rs | 36 +- codegen/src/tests.rs | 153 + codegen/templates/epoch_seconds.rs | 90 + crates/rustack-stepfunctions-model/Cargo.toml | 22 + .../src/epoch_seconds.rs | 151 + .../rustack-stepfunctions-model/src/error.rs | 316 ++ .../rustack-stepfunctions-model/src/input.rs | 852 ++++++ crates/rustack-stepfunctions-model/src/lib.rs | 18 + .../src/operations.rs | 177 ++ .../rustack-stepfunctions-model/src/output.rs | 926 ++++++ .../rustack-stepfunctions-model/src/types.rs | 2572 +++++++++++++++++ .../tests/timestamps.rs | 197 ++ specs/index.md | 2 +- specs/rustack-workflows-10-api-design.md | 10 + .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 7 +- specs/rustack-workflows-93-review.md | 13 + specs/rustack-workflows-99-decisions.md | 10 + 28 files changed, 5942 insertions(+), 87 deletions(-) create mode 100644 codegen/src/tests.rs create mode 100644 codegen/templates/epoch_seconds.rs create mode 100644 crates/rustack-stepfunctions-model/Cargo.toml create mode 100644 crates/rustack-stepfunctions-model/src/epoch_seconds.rs create mode 100644 crates/rustack-stepfunctions-model/src/error.rs create mode 100644 crates/rustack-stepfunctions-model/src/input.rs create mode 100644 crates/rustack-stepfunctions-model/src/lib.rs create mode 100644 crates/rustack-stepfunctions-model/src/operations.rs create mode 100644 crates/rustack-stepfunctions-model/src/output.rs create mode 100644 crates/rustack-stepfunctions-model/src/types.rs create mode 100644 crates/rustack-stepfunctions-model/tests/timestamps.rs diff --git a/Cargo.lock b/Cargo.lock index 25b2e407..27b3f2ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4190,6 +4190,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-stepfunctions-model" +version = "0.9.0" +dependencies = [ + "chrono", + "http 1.5.0", + "rstest", + "serde", + "serde_json", + "thiserror", + "typed-builder", +] + [[package]] name = "rustack-sts-core" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 322f44c4..1559ccb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ rustack-lambda-model = { version = "0.9.0", path = "crates/rustack-lambda-model" rustack-lambda-http = { version = "0.11.0", path = "crates/rustack-lambda-http" } rustack-lambda-core = { version = "0.11.0", path = "crates/rustack-lambda-core" } rustack-lambda-runtime = { version = "0.9.0", path = "crates/rustack-lambda-runtime" } +rustack-stepfunctions-model = { version = "0.9.0", path = "crates/rustack-stepfunctions-model" } rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } rustack-logs-http = { version = "0.11.0", path = "crates/rustack-logs-http" } diff --git a/Makefile b/Makefile index 2ebc7459..510b106d 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,33 @@ codegen-lambda: @cd codegen && cargo run -- --config services/lambda.toml --model smithy-model/lambda.json --output ../crates/rustack-lambda-model/src @cargo +nightly fmt -p rustack-lambda-model +codegen-stepfunctions: + @cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output crates/rustack-stepfunctions-model/src + @cargo +nightly fmt -p rustack-stepfunctions-model + +check-stepfunctions-codegen: + @set -eu; generated=$$(mktemp -d /tmp/rustack-sfn-codegen.XXXXXX); \ + trap 'rm -rf "$$generated"' EXIT; \ + cargo run --manifest-path codegen/Cargo.toml -- --config codegen/services/stepfunctions.toml --model codegen/smithy-model/stepfunctions.json --output "$$generated"; \ + rustup run nightly rustfmt --edition 2024 --config-path rustfmt.toml "$$generated"/*.rs; \ + diff -r "$$generated" crates/rustack-stepfunctions-model/src + +test-stepfunctions-model: check-stepfunctions-codegen + @cargo test --manifest-path codegen/Cargo.toml + @cargo clippy --manifest-path codegen/Cargo.toml --all-targets -- -D warnings + @cargo test -p rustack-stepfunctions-model + @cargo clippy -p rustack-stepfunctions-model --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: codegen-stepfunctions check-stepfunctions-codegen test-stepfunctions-model + +audit-codegen: + @cargo audit --file codegen/Cargo.lock + +deny-codegen: + @cargo deny --manifest-path codegen/Cargo.toml check + +.PHONY: audit-codegen deny-codegen + codegen-kms: @cd codegen && cargo run -- --config services/kms.toml --model smithy-model/kms.json --output ../crates/rustack-kms-model/src @cargo +nightly fmt -p rustack-kms-model diff --git a/codegen/Cargo.lock b/codegen/Cargo.lock index b8f117e6..cf9526a7 100644 --- a/codegen/Cargo.lock +++ b/codegen/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "equivalent" version = "1.0.2" @@ -48,6 +54,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -74,6 +86,7 @@ dependencies = [ "heck", "serde", "serde_json", + "tokio", "toml", ] @@ -104,7 +117,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -140,6 +153,39 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 018d838c..5da65e54 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -2,6 +2,7 @@ name = "rustack-codegen" version = "0.1.1" edition = "2024" +license = "MIT" [dependencies] serde = { version = "1.0", features = ["derive"] } @@ -9,3 +10,5 @@ serde_json = "1.0" toml = "0.9" heck = "0.5" anyhow = "1.0" + +tokio = { version = "~1.54.1", features = ["rt", "macros", "fs", "time", "io-util"] } diff --git a/codegen/services/stepfunctions.toml b/codegen/services/stepfunctions.toml index 0dc0cc33..a6204689 100644 --- a/codegen/services/stepfunctions.toml +++ b/codegen/services/stepfunctions.toml @@ -58,3 +58,5 @@ UnsupportedOperation = { status = 400, message = "Operation is outside the suppo [output] file_layout = "flat" always_serialize_arrays = true + +strict_model = true diff --git a/codegen/src/codegen.rs b/codegen/src/codegen.rs index 0331563c..05016961 100644 --- a/codegen/src/codegen.rs +++ b/codegen/src/codegen.rs @@ -3,15 +3,15 @@ //! This module takes a `ResolvedModel` and produces Rust source code strings //! for types, input/output structs, operations, errors, and request types. -use std::collections::BTreeMap; -use std::fmt::Write; - -use anyhow::Result; +use std::{collections::BTreeMap, fmt::Write}; +use anyhow::{Result, bail}; use heck::{ToLowerCamelCase, ToPascalCase}; -use crate::config::ServiceConfig; -use crate::shapes::{EnumVariantInfo, FieldInfo, HttpBinding, OperationCategories, ResolvedModel}; +use crate::{ + config::{Protocol, ServiceConfig}, + shapes::{EnumVariantInfo, FieldInfo, HttpBinding, OperationCategories, ResolvedModel}, +}; /// Header comment placed at the top of every generated file. fn file_header(config: &ServiceConfig) -> String { @@ -21,12 +21,39 @@ fn file_header(config: &ServiceConfig) -> String { ) } +fn json_epoch_protocol(config: &ServiceConfig) -> bool { + matches!(config.protocol, Protocol::AwsJson1_0 | Protocol::AwsJson1_1) +} +fn epoch_field(field: &FieldInfo, config: &ServiceConfig) -> bool { + json_epoch_protocol(config) + && field + .timestamp_format + .as_deref() + .is_some_and(|format| matches!(format, "" | "epoch-seconds")) +} +fn needs_epoch_module(resolved: &ResolvedModel, config: &ServiceConfig) -> bool { + config.uses_serde() + && resolved + .input_structs + .values() + .chain(resolved.output_structs.values()) + .chain(resolved.shared_structs.values()) + .flatten() + .any(|field| epoch_field(field, config)) +} + /// Generate all source files and return them as a map of path -> content. pub fn generate_all( resolved: &ResolvedModel, config: &ServiceConfig, ) -> Result> { let mut files = BTreeMap::new(); + if needs_epoch_module(resolved, config) { + files.insert( + "epoch_seconds.rs".to_owned(), + include_str!("../templates/epoch_seconds.rs").to_owned(), + ); + } files.insert("types.rs".to_owned(), generate_types(resolved, config)?); files.insert("operations.rs".to_owned(), generate_operations(config)?); @@ -113,7 +140,7 @@ fn generate_types(resolved: &ResolvedModel, config: &ServiceConfig) -> Result Result<()> { - writeln!(out, "/// {display_name} {name} enum.")?; + writeln!(out, "/// {display_name} `{name}` enum.")?; writeln!( out, "#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]" @@ -139,8 +167,12 @@ fn write_enum( writeln!(out, "pub enum {name} {{")?; for (i, variant) in variants.iter().enumerate() { - if i == 0 { + if config.strict_model { + writeln!(out, " /// The `{}` wire variant.", variant.string_value)?; + } else if i == 0 { writeln!(out, " /// Default variant.")?; + } + if i == 0 { writeln!(out, " #[default]")?; } if variant.rust_name != variant.string_value { @@ -229,23 +261,41 @@ fn write_struct( display_name: &str, config: &ServiceConfig, ) -> Result<()> { - writeln!(out, "/// {display_name} {name}.")?; + writeln!(out, "/// {display_name} `{name}`.")?; if config.uses_serde() { - writeln!( - out, - "#[derive(Debug, Clone, Default, Serialize, Deserialize)]" - )?; + if config.strict_model { + writeln!(out, "#[derive(Clone, Default, Serialize, Deserialize)]")?; + } else { + writeln!( + out, + "#[derive(Debug, Clone, Default, Serialize, Deserialize)]" + )?; + } if let Some(ref rename) = config.serde_rename { writeln!(out, "#[serde(rename_all = \"{rename}\")]")?; } + } else if config.strict_model { + writeln!(out, "#[derive(Clone, Default)]")?; } else { writeln!(out, "#[derive(Debug, Clone, Default)]")?; } + if config.strict_model { + writeln!(out, "#[non_exhaustive]")?; + if config.uses_serde() { + writeln!(out, "#[serde(deny_unknown_fields)]")?; + } + if fields.len() > 5 { + writeln!(out, "#[derive(typed_builder::TypedBuilder)]")?; + } + } writeln!(out, "pub struct {name} {{")?; for field in fields { + if config.strict_model { + writeln!(out, " /// The `{}` wire member.", field.smithy_name)?; + } // Write HTTP binding comment if requested if include_http_comments && let Some(ref binding) = field.http_binding { let comment = match binding { @@ -260,8 +310,35 @@ fn write_struct( writeln!(out, "{comment}")?; } + if config.strict_model && fields.len() > 5 && !field.required { + writeln!(out, " #[builder(default)]")?; + } // Write serde field attributes when serde is enabled if config.uses_serde() { + if json_epoch_protocol(config) { + match field.timestamp_format.as_deref() { + Some("" | "epoch-seconds") => { + if field.rust_type.starts_with("Option<") { + writeln!( + out, + " #[serde(default, with = \"crate::epoch_seconds::option\")]" + )?; + } else { + writeln!(out, " #[serde(with = \"crate::epoch_seconds\")]")?; + } + } + Some("date-time") => {} + Some(_) => bail!( + "unsupported JSON timestamp format for {}", + field.smithy_name + ), + None if field.rust_type.contains("chrono::DateTime") => bail!( + "timestamp collection requires an explicit adapter: {}", + field.smithy_name + ), + None => {} + } + } // Explicit rename if rename_all would not produce the correct Smithy name if let Some(ref strategy) = config.serde_rename && needs_explicit_rename(&field.rust_name, &field.smithy_name, strategy) @@ -274,6 +351,11 @@ fn write_struct( out, " #[serde(skip_serializing_if = \"Option::is_none\")]" )?; + } else if config.strict_model + && field.required + && (field.rust_type.starts_with("Vec<") || field.rust_type.starts_with("HashMap<")) + { + // Required collections must be present, including when empty. } else if field.rust_type.starts_with("Vec<") { if config.always_serialize_arrays { writeln!(out, " #[serde(default)]")?; @@ -296,6 +378,18 @@ fn write_struct( writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl std::fmt::Debug for {name} {{")?; + writeln!( + out, + " fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{" + )?; + writeln!( + out, + " f.debug_struct(\"{name}\").finish_non_exhaustive()" + )?; + writeln!(out, " }}\n}}")?; + } Ok(()) } @@ -549,12 +643,12 @@ fn generate_operations(config: &ServiceConfig) -> Result { writeln!(out, "{header}")?; writeln!(out)?; - writeln!(out, "/// All supported {prefix} operations.")?; + writeln!(out, "/// All supported `{prefix}Operation` values.")?; writeln!(out, "#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]")?; writeln!(out, "pub enum {prefix}Operation {{")?; for op in all_operations { - writeln!(out, " /// The {op} operation.")?; + writeln!(out, " /// The `{op}` operation.")?; writeln!(out, " {op},")?; } @@ -577,7 +671,7 @@ fn generate_operations(config: &ServiceConfig) -> Result { // Generate from_str writeln!( out, - " /// Parse an operation name string into an {prefix}Operation." + " /// Parse an operation name string into a `{prefix}Operation`." )?; writeln!(out, " #[must_use]")?; writeln!(out, " pub fn from_name(name: &str) -> Option {{")?; @@ -668,7 +762,7 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu } for err in errors { - writeln!(out, " /// {} error.", err.name)?; + writeln!(out, " /// `{}` error.", err.name)?; if err.name == default_name { writeln!(out, " #[default]")?; } @@ -746,9 +840,35 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl {prefix}ErrorCode {{")?; + writeln!(out, " /// Default bounded protocol message.")?; + writeln!(out, " #[must_use]")?; + writeln!(out, " pub fn default_message(&self) -> &'static str {{")?; + writeln!(out, " match self {{")?; + for error in errors { + let message = config + .custom_errors + .get(&error.name) + .map_or(error.wire_name.as_str(), |custom| custom.message.as_str()); + writeln!(out, " Self::{} => {message:?},", error.name)?; + } + if errors.is_empty() { + writeln!( + out, + " Self::InternalException => \"InternalException\"," + )?; + } + writeln!(out, " }}\n }}\n}}")?; + } + // Error struct (no resource/request_id for JSON services) writeln!(out, "/// An {display_name} error response.")?; - writeln!(out, "#[derive(Debug)]")?; + if config.strict_model { + writeln!(out, "#[non_exhaustive]")?; + } else { + writeln!(out, "#[derive(Debug)]")?; + } writeln!(out, "pub struct {prefix}Error {{")?; writeln!(out, " /// The error code.")?; writeln!(out, " pub code: {prefix}ErrorCode,")?; @@ -764,16 +884,29 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "}}")?; writeln!(out)?; + if config.strict_model { + writeln!(out, "impl fmt::Debug for {prefix}Error {{")?; + writeln!( + out, + " fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {{" + )?; + writeln!(out, " f.write_str(\"{prefix}Error()\")")?; + writeln!(out, " }}\n}}")?; + } // Display impl writeln!(out, "impl fmt::Display for {prefix}Error {{")?; writeln!( out, " fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {{" )?; - writeln!( - out, - " write!(f, \"{prefix}Error({{}}): {{}}\", self.code, self.message)" - )?; + if config.strict_model { + writeln!(out, " f.write_str(\"{prefix}Error()\")")?; + } else { + writeln!( + out, + " write!(f, \"{prefix}Error({{}}): {{}}\", self.code, self.message)" + )?; + } writeln!(out, " }}")?; writeln!(out, "}}")?; writeln!(out)?; @@ -802,7 +935,14 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, " pub fn new(code: {prefix}ErrorCode) -> Self {{")?; writeln!(out, " Self {{")?; writeln!(out, " status_code: code.default_status_code(),")?; - writeln!(out, " message: code.as_str().to_owned(),")?; + if config.strict_model { + writeln!( + out, + " message: code.default_message().to_owned()," + )?; + } else { + writeln!(out, " message: code.as_str().to_owned(),")?; + } writeln!(out, " code,")?; writeln!(out, " source: None,")?; writeln!(out, " }}")?; @@ -838,8 +978,8 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out)?; // Convenience constructors for common errors - // Generate validation(), internal_error(), missing_action(), unknown_operation(), not_implemented() - // based on which errors are present + // Generate validation(), internal_error(), missing_action(), unknown_operation(), + // not_implemented() based on which errors are present let has_error = |name: &str| errors.iter().any(|e| e.name == name); @@ -936,7 +1076,15 @@ fn generate_error_json(resolved: &ResolvedModel, config: &ServiceConfig) -> Resu writeln!(out, "///")?; writeln!(out, "/// # Examples")?; writeln!(out, "///")?; - writeln!(out, "/// ```ignore")?; + if config.strict_model { + writeln!(out, "/// ```")?; + writeln!( + out, + "/// use rustack_{name}_model::{{{name}_error, {prefix}ErrorCode}};" + )?; + } else { + writeln!(out, "/// ```ignore")?; + } writeln!(out, "/// let err = {name}_error!({default_name});")?; writeln!( out, @@ -1385,7 +1533,8 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { writeln!(out, " #[must_use]")?; writeln!( out, - " pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {{" + " pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) \ + -> Self {{" )?; writeln!(out, " self.source = Some(Box::new(source));")?; writeln!(out, " self")?; @@ -1601,7 +1750,8 @@ fn generate_request(config: &ServiceConfig) -> Result { )?; writeln!( out, - " .field(\"session_token\", &self.session_token.as_ref().map(|_| \"[REDACTED]\"))" + " .field(\"session_token\", &self.session_token.as_ref().map(|_| \ + \"[REDACTED]\"))" )?; writeln!(out, " .finish()")?; writeln!(out, " }}")?; @@ -1684,9 +1834,16 @@ fn generate_lib(resolved: &ResolvedModel, config: &ServiceConfig) -> Result Result Result, + /// Whether AWS Query-compatible errors are configured. + pub aws_query_compatible: bool, /// Serde rename strategy (None means no rename). pub serde_rename: Option, /// Whether to emit HTTP binding comments. @@ -174,8 +181,6 @@ pub struct ServiceConfig { pub emit_request_wrapper: bool, /// Ordered list of all operations to generate. pub all_operations: Vec, - /// Map from operation name to phase number. - pub operation_phases: BTreeMap, /// Operation categories for file organization (if categorized layout). pub categories: Option>>, /// File layout strategy. @@ -188,6 +193,8 @@ pub struct ServiceConfig { pub overlay_extra_modules: Vec, /// Whether to always serialize arrays. pub always_serialize_arrays: bool, + /// Opt-in strict public model generation. + pub strict_model: bool, } /// Supported protocol variants. @@ -217,6 +224,9 @@ impl ServiceConfig { other => bail!("Unknown protocol: {other}"), }; + if config.protocol.error_type_format.is_some() || config.output.dir.is_some() { + bail!("custom error formats and output.dir are unsupported; use --output"); + } let serde_rename = match config.protocol.serde_rename.as_str() { "none" | "" => None, s => Some(s.to_owned()), @@ -225,22 +235,20 @@ impl ServiceConfig { // Build all_operations list. // Priority: explicit `all` list > phases > categories let mut all_operations = Vec::new(); - let mut operation_phases = BTreeMap::new(); if !config.operations.all.is_empty() { // Use the explicit ordered list all_operations = config.operations.all.clone(); } else { // Build from phases - for (phase_num, phase_ops) in [ - (0, &config.operations.phase0), - (1, &config.operations.phase1), - (2, &config.operations.phase2), - (3, &config.operations.phase3), + for phase_ops in [ + &config.operations.phase0, + &config.operations.phase1, + &config.operations.phase2, + &config.operations.phase3, ] { for op in phase_ops { all_operations.push(op.clone()); - operation_phases.insert(op.clone(), phase_num); } } @@ -269,18 +277,20 @@ impl ServiceConfig { rust_prefix: config.service.rust_prefix, namespace: config.service.namespace, protocol, + target_prefix: config.protocol.target_prefix, + aws_query_compatible: config.protocol.aws_query_compatible, serde_rename, emit_http_bindings: config.protocol.emit_http_bindings, emit_serde_derives: config.protocol.emit_serde_derives, emit_request_wrapper: config.protocol.emit_request_wrapper, all_operations, - operation_phases, categories, file_layout: config.output.file_layout, custom_errors: config.errors.custom, overlay_preserve: config.overlay.preserve, overlay_extra_modules: config.overlay.extra_modules, always_serialize_arrays: config.output.always_serialize_arrays, + strict_model: config.output.strict_model, }) } diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 61bbf7b2..6d28bece 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -3,20 +3,43 @@ //! Reads an AWS Smithy JSON AST model and generates Rust source files //! based on a TOML service configuration. +#![forbid(unsafe_code)] + mod codegen; mod config; mod model; mod shapes; +#[cfg(test)] +mod tests; -use std::fs; -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, ensure}; +use tokio::{fs, io::AsyncReadExt, time::timeout}; use crate::config::{ServiceConfig, ServiceConfigFile}; -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + timeout(Duration::from_secs(60), generate()) + .await + .context("Code generation deadline exceeded")? +} + +async fn generate() -> Result<()> { + let args: Vec = std::env::args_os() + .map(|arg| { + arg.into_string() + .map_err(|_| anyhow::anyhow!("Arguments must be UTF-8")) + }) + .collect::>()?; + ensure!( + args.len() <= 16 && args.iter().all(|arg| arg.len() <= 4096), + "Argument budget exceeded" + ); // Parse named arguments: --config, --model, --output let config_path = find_arg(&args, "--config"); @@ -34,7 +57,8 @@ fn main() -> Result<()> { ); // Read and parse the TOML config - let config_toml = fs::read_to_string(&config_path) + let config_toml = read_utf8(&config_path, 1024 * 1024) + .await .with_context(|| format!("Failed to read config file: {}", config_path.display()))?; let config_file: ServiceConfigFile = toml::from_str(&config_toml).context("Failed to parse TOML config")?; @@ -65,7 +89,8 @@ fn main() -> Result<()> { eprintln!("Writing output to: {}", output_dir.display()); // Read and parse the Smithy JSON model. - let model_json = fs::read_to_string(&model_path) + let model_json = read_utf8(&model_path, 16 * 1024 * 1024) + .await .with_context(|| format!("Failed to read model file: {}", model_path.display()))?; let smithy_model: model::SmithyModel = @@ -73,6 +98,8 @@ fn main() -> Result<()> { eprintln!("Parsed model: {} shapes", smithy_model.shapes.len()); + ensure!(smithy_model.smithy == "2.0", "Unsupported Smithy version"); + // Resolve shapes and types. let resolved = shapes::resolve_model(&smithy_model, &service_config) .context("Failed to resolve model shapes")?; @@ -93,8 +120,12 @@ fn main() -> Result<()> { // Write output files. for (rel_path, content) in &files { let full_path = output_dir.join(rel_path); - ensure_parent_dir(&full_path)?; + if service_config.overlay_preserve.contains(rel_path) && fs::try_exists(&full_path).await? { + continue; + } + ensure_parent_dir(&full_path).await?; fs::write(&full_path, content) + .await .with_context(|| format!("Failed to write {}", full_path.display()))?; eprintln!(" Wrote: {}", full_path.display()); } @@ -113,10 +144,27 @@ fn find_arg(args: &[String], flag: &str) -> Option { } /// Ensure the parent directory of a path exists. -fn ensure_parent_dir(path: &Path) -> Result<()> { +async fn ensure_parent_dir(path: &Path) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) + .await .with_context(|| format!("Failed to create directory: {}", parent.display()))?; } Ok(()) } + +/// Read ordinary developer-owned input files under an explicit byte ceiling. +async fn read_utf8(path: &Path, maximum: u64) -> Result { + ensure!( + fs::metadata(path).await?.is_file(), + "Codegen input must be an ordinary file" + ); + let file = fs::File::open(path).await?; + let mut bytes = Vec::new(); + file.take(maximum + 1).read_to_end(&mut bytes).await?; + ensure!( + u64::try_from(bytes.len())? <= maximum, + "Codegen input byte limit exceeded" + ); + String::from_utf8(bytes).context("Invalid UTF-8 codegen input") +} diff --git a/codegen/src/model.rs b/codegen/src/model.rs index a43aab65..f0267256 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; -use serde::Deserialize; +use serde::{Deserialize, de::IgnoredAny}; /// Top-level Smithy model document. #[derive(Debug, Deserialize)] @@ -31,13 +31,13 @@ pub enum Shape { String(SimpleShape), /// A boolean shape. #[serde(rename = "boolean")] - Boolean(SimpleShape), + Boolean(IgnoredAny), /// A 32-bit integer shape. #[serde(rename = "integer")] - Integer(SimpleShape), + Integer(IgnoredAny), /// A 64-bit integer shape. #[serde(rename = "long")] - Long(SimpleShape), + Long(IgnoredAny), /// A timestamp shape. #[serde(rename = "timestamp")] Timestamp(SimpleShape), @@ -58,19 +58,19 @@ pub enum Shape { Union(UnionShape), /// A service shape (ignored). #[serde(rename = "service")] - Service(SimpleShape), + Service(IgnoredAny), /// A resource shape (ignored). #[serde(rename = "resource")] - Resource(SimpleShape), + Resource(IgnoredAny), /// An integer enum shape (ignored). #[serde(rename = "intEnum")] - IntEnum(SimpleShape), + IntEnum(IgnoredAny), /// A double shape. #[serde(rename = "double")] - Double(SimpleShape), + Double(IgnoredAny), /// A float shape. #[serde(rename = "float")] - Float(SimpleShape), + Float(IgnoredAny), } /// A simple shape with optional traits. @@ -112,9 +112,6 @@ pub struct OperationShape { /// Error shape references. #[serde(default)] pub errors: Vec, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A reference to another shape. @@ -130,9 +127,6 @@ pub struct EnumShape { /// Enum variants (member name -> member shape). #[serde(default)] pub members: HashMap, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A list shape. @@ -140,9 +134,6 @@ pub struct EnumShape { pub struct ListShape { /// The shape of the list's elements. pub member: ShapeRef, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A map shape. @@ -152,9 +143,6 @@ pub struct MapShape { pub key: ShapeRef, /// The shape of the map's values. pub value: ShapeRef, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } /// A union shape. @@ -163,9 +151,6 @@ pub struct UnionShape { /// Union variants. #[serde(default)] pub members: HashMap, - /// Traits applied to this shape. - #[serde(default)] - pub traits: HashMap, } impl SmithyModel { diff --git a/codegen/src/shapes.rs b/codegen/src/shapes.rs index 478895dd..dce3f9c7 100644 --- a/codegen/src/shapes.rs +++ b/codegen/src/shapes.rs @@ -7,8 +7,10 @@ use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result}; -use crate::config::ServiceConfig; -use crate::model::{MemberShape, Shape, SmithyModel, StructureShape}; +use crate::{ + config::ServiceConfig, + model::{MemberShape, Shape, SmithyModel, StructureShape}, +}; /// Maximum recursion depth when resolving shape references. const MAX_RESOLVE_DEPTH: usize = 15; @@ -38,6 +40,8 @@ pub struct FieldInfo { pub rust_type: String, /// Whether this field is required. pub required: bool, + /// Timestamp format; empty means protocol default, None means not a timestamp. + pub timestamp_format: Option, /// HTTP binding annotation if any. pub http_binding: Option, } @@ -75,8 +79,6 @@ pub struct ErrorInfo { pub wire_name: String, /// Default HTTP status code (from `smithy.api#httpError`, or 400/500 based on fault). pub status_code: u16, - /// Default human-readable message. - pub message: String, /// Fault type: `"client"` or `"server"`. pub fault: String, } @@ -331,12 +333,33 @@ fn resolve_struct_fields( rust_name: to_snake_case(name), rust_type, required, + timestamp_format: timestamp_format(model, member), http_binding, } }) .collect() } +/// Resolve a timestamp member override before its target-shape override. +fn timestamp_format(model: &SmithyModel, member: &MemberShape) -> Option { + let shape = model.shapes.get(&member.target); + let target_traits = match shape { + Some(Shape::Timestamp(shape)) => Some(&shape.traits), + _ if member.target == "smithy.api#Timestamp" => None, + _ => return None, + }; + Some( + member + .traits + .get("smithy.api#timestampFormat") + .or_else(|| target_traits.and_then(|traits| traits.get("smithy.api#timestampFormat"))) + .map_or("", |value| { + value.as_str().unwrap_or("") + }) + .to_owned(), + ) +} + /// Check if a type is a collection type that should not be wrapped in Option. fn is_collection_type(ty: &str) -> bool { ty.starts_with("Vec<") || ty.starts_with("HashMap<") @@ -506,6 +529,7 @@ pub fn resolve_model(model: &SmithyModel, config: &ServiceConfig) -> Result"), required: false, + timestamp_format: timestamp_format(model, member), http_binding: None, } }) @@ -565,14 +589,11 @@ fn collect_errors( // for a cleaner Rust API, but keep the wire name for serialization) let name = wire_name.clone(); - let message = name.clone(); - seen_names.insert(name.clone()); errors.push(ErrorInfo { name, wire_name, status_code, - message, fault, }); } @@ -585,7 +606,6 @@ fn collect_errors( name: name.clone(), wire_name: name.clone(), status_code: custom.status, - message: custom.message.clone(), fault: if custom.status >= 500 { "server".to_owned() } else { diff --git a/codegen/src/tests.rs b/codegen/src/tests.rs new file mode 100644 index 00000000..4793fd4a --- /dev/null +++ b/codegen/src/tests.rs @@ -0,0 +1,153 @@ +//! Generator protocol-policy tests independent of generated runtime fixtures. + +use serde_json::json; + +use crate::{ + codegen::generate_all, + config::{ServiceConfig, ServiceConfigFile}, + model::SmithyModel, + shapes::resolve_model, +}; + +fn config(protocol: &str, strict: bool) -> ServiceConfig { + let text = format!( + r#" +[service] +name="fixture" +display_name="Fixture" +rust_prefix="Fixture" +namespace="fixture" +protocol="{protocol}" +[protocol] +serde_rename="camelCase" +emit_serde_derives=true +[operations] +all=["Run"] +[output] +file_layout="flat" +strict_model={strict} +"# + ); + let file: ServiceConfigFile = toml::from_str(&text).unwrap(); + ServiceConfig::from_file(file).unwrap() +} +fn model(target: Option<&str>, member: Option<&str>, collection: bool) -> SmithyModel { + let mut value = json!({"smithy":"2.0","shapes":{ + "fixture#Run":{"type":"operation","input":{"target":"fixture#RunRequest"}}, + "fixture#RunRequest":{"type":"structure","members":{ + "required":{"target":"fixture#Time","traits":{"smithy.api#required":{}}}, + "optional":{"target":"fixture#Time"}, + "plain":{"target":"smithy.api#String"}}}, + "fixture#Time":{"type":"timestamp","traits":{}} + }}); + if let Some(format) = target { + value["shapes"]["fixture#Time"]["traits"]["smithy.api#timestampFormat"] = json!(format); + } + if let Some(format) = member { + value["shapes"]["fixture#RunRequest"]["members"]["required"]["traits"] + ["smithy.api#timestampFormat"] = json!(format); + } + if collection { + value["shapes"]["fixture#Times"] = + json!({"type":"list","member":{"target":"fixture#Time"}}); + value["shapes"]["fixture#RunRequest"]["members"]["times"] = + json!({"target":"fixture#Times"}); + } + serde_json::from_value(value).unwrap() +} +#[test] +fn test_should_emit_numeric_codecs_only_for_json_protocol_defaults() { + for protocol in [ + "awsJson1_0", + "awsJson1_1", + "restJson1", + "awsQuery", + "restXml", + ] { + let config = config(protocol, true); + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + let numeric = protocol.starts_with("awsJson"); + assert_eq!(files.contains_key("epoch_seconds.rs"), numeric); + assert_eq!( + files["input.rs"].contains("with = \"crate::epoch_seconds"), + numeric + ); + if numeric { + assert!(files["input.rs"].contains("default, with = \"crate::epoch_seconds::option\"")); + } + } +} +#[test] +fn test_should_resolve_member_override_before_shape_before_protocol_default() { + let config = config("awsJson1_0", true); + for (target, member, expected) in [ + (Some("date-time"), Some("epoch-seconds"), "epoch-seconds"), + (Some("epoch-seconds"), Some("date-time"), "date-time"), + (Some("date-time"), None, "date-time"), + (None, None, ""), + ] { + let resolved = resolve_model(&model(target, member, false), &config).unwrap(); + let field = resolved.input_structs["RunInput"] + .iter() + .find(|field| field.smithy_name == "required") + .unwrap(); + assert_eq!(field.timestamp_format.as_deref(), Some(expected)); + let files = generate_all(&resolved, &config).unwrap(); + let required = files["input.rs"].split("pub required:").next().unwrap(); + assert_eq!( + required + .lines() + .rev() + .find(|line| !line.trim().is_empty()) + .unwrap() + .contains("with = \"crate::epoch_seconds\""), + expected != "date-time" + ); + } +} +#[test] +fn test_should_reject_unsupported_explicit_formats_and_unadapted_collections() { + let config = config("awsJson1_0", true); + for model in [ + model(None, Some("http-date"), false), + model(None, None, true), + ] { + let resolved = resolve_model(&model, &config).unwrap(); + assert!(generate_all(&resolved, &config).is_err()); + } +} +#[test] +fn test_should_keep_strict_models_redacted_documented_and_without_blanket_ignores() { + let config = config("awsJson1_0", true); + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + assert!(files["input.rs"].contains("#[serde(deny_unknown_fields)]")); + assert!(files["input.rs"].contains("#[non_exhaustive]")); + assert!(files["input.rs"].contains("impl std::fmt::Debug for RunInput")); + assert!(files["input.rs"].contains("/// The `required` wire member.")); + assert!(files.values().all(|file| !file.contains("#[allow(") + && !file.contains("#![allow(") + && !file.contains("```ignore"))); + assert_eq!(files, generate_all(&resolved, &config).unwrap()); +} + +#[test] +fn test_should_emit_one_redacted_debug_without_serde_derives() { + for protocol in ["restXml", "awsJson1_0", "awsJson1_1"] { + let mut config = config(protocol, true); + config.emit_serde_derives = false; + let resolved = resolve_model(&model(None, None, false), &config).unwrap(); + let files = generate_all(&resolved, &config).unwrap(); + assert!(files["input.rs"].contains("#[derive(Clone, Default)]")); + assert!(!files["input.rs"].contains("derive(Debug")); + assert_eq!( + files["input.rs"] + .matches("impl std::fmt::Debug for RunInput") + .count(), + 1 + ); + assert!(!files.contains_key("epoch_seconds.rs")); + assert!(!files["lib.rs"].contains("mod epoch_seconds")); + } +} diff --git a/codegen/templates/epoch_seconds.rs b/codegen/templates/epoch_seconds.rs new file mode 100644 index 00000000..36ab79c9 --- /dev/null +++ b/codegen/templates/epoch_seconds.rs @@ -0,0 +1,90 @@ +//! Generated JSON epoch-second adapters with exact decimal millisecond precision. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as DeError, ser::Error as SerError}; +use serde_json::value::RawValue; + +const INVALID: &str = "invalid epoch-second timestamp"; + +/// Serialize a required timestamp as a numeric JSON epoch value. +/// +/// # Errors +/// Rejects leap seconds, invalid numeric representation or serializer failures. +pub fn serialize(value: &DateTime, serializer: S) -> Result { + if value.timestamp_subsec_nanos() >= 1_000_000_000 { return Err(S::Error::custom(INVALID)); } + let nanos = i128::from(value.timestamp()).checked_mul(1_000_000_000) + .and_then(|seconds| seconds.checked_add(i128::from(value.timestamp_subsec_nanos()))) + .ok_or_else(|| S::Error::custom(INVALID))?; + let millis = nanos / 1_000_000; + let sign = if millis < 0 { "-" } else { "" }; + let magnitude = millis.unsigned_abs(); + let seconds = magnitude / 1000; + let fraction = magnitude % 1000; + let number = if fraction == 0 { format!("{sign}{seconds}") } else { format!("{sign}{seconds}.{fraction:03}") }; + RawValue::from_string(number).map_err(|_| S::Error::custom(INVALID))?.serialize(serializer) +} + +/// Deserialize a required JSON number, truncating submillisecond precision. +/// +/// # Errors +/// Rejects nonnumeric, oversized, out-of-range or invalid epoch values. +pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let value = Box::::deserialize(deserializer)?; + parse(value.get()).ok_or_else(|| D::Error::custom(INVALID)) +} + +fn parse(raw: &str) -> Option> { + if raw.len() > 128 { return None; } + let (negative, raw) = raw.strip_prefix('-').map_or((false, raw), |rest| (true, rest)); + let (mantissa, exponent) = match raw.split_once(['e', 'E']) { + Some((value, exponent)) => { + let (negative, digits) = exponent.strip_prefix('-').map_or_else( + || (false, exponent.strip_prefix('+').unwrap_or(exponent)), |digits| (true, digits)); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { return None; } + let magnitude = digits.bytes().fold(0_i32, |value, digit| + value.saturating_mul(10).saturating_add(i32::from(digit - b'0')).min(1024)); + (value, if negative { -magnitude } else { magnitude }) + }, + None => (raw, 0), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if whole.is_empty() || !whole.bytes().chain(fraction.bytes()).all(|byte| byte.is_ascii_digit()) { return None; } + let digits: Vec<_> = whole.bytes().chain(fraction.bytes()).collect(); + if digits.iter().all(|digit| *digit == b'0') { return DateTime::from_timestamp_millis(0); } + let keep = i32::try_from(whole.len()).ok()?.checked_add(exponent)?.checked_add(3)?; + if keep <= 0 { return DateTime::from_timestamp_millis(0); } + // Leading zeroes must not make a small representable exponent look huge. + let leading = digits.iter().take_while(|digit| **digit == b'0').count(); + let significant = keep.checked_sub(i32::try_from(leading).ok()?)?; + if significant <= 0 { return DateTime::from_timestamp_millis(0); } + if significant > 19 { return None; } + let mut millis = 0_i128; + let count = usize::try_from(significant).ok()?; + for offset in 0..count { + let digit = digits.get(leading.checked_add(offset)?).copied().unwrap_or(b'0'); + millis = millis.checked_mul(10)?.checked_add(i128::from(digit.checked_sub(b'0')?))?; + } + let signed = if negative { millis.checked_neg()? } else { millis }; + DateTime::from_timestamp_millis(i64::try_from(signed).ok()?) +} + +/// Optional JSON timestamp adapter; missing members also need serde default. +pub mod option { + use std::borrow::Borrow; + use super::{DateTime, Deserialize, Deserializer, DeError, RawValue, Serializer, Utc, INVALID}; + + /// Serialize a timestamp or JSON null. + /// + /// # Errors + /// Returns the required adapter's validation/serialization errors. + pub fn serialize>>>(value: V, serializer: S) -> Result { + match value.borrow() { Some(value) => super::serialize(value, serializer), None => serializer.serialize_none() } + } + /// Deserialize a timestamp or JSON null. + /// + /// # Errors + /// Rejects invalid nonnull values with the required adapter's rules. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result>, D::Error> { + Option::>::deserialize(deserializer)?.map(|value| super::parse(value.get()).ok_or_else(|| D::Error::custom(INVALID))).transpose() + } +} diff --git a/crates/rustack-stepfunctions-model/Cargo.toml b/crates/rustack-stepfunctions-model/Cargo.toml new file mode 100644 index 00000000..2bf629a7 --- /dev/null +++ b/crates/rustack-stepfunctions-model/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustack-stepfunctions-model" +description = "Generated Step Functions protocol model for Rustack" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["raw_value"] } +thiserror = { workspace = true } +http = { workspace = true } +chrono = { workspace = true } +typed-builder = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } diff --git a/crates/rustack-stepfunctions-model/src/epoch_seconds.rs b/crates/rustack-stepfunctions-model/src/epoch_seconds.rs new file mode 100644 index 00000000..c5f3db90 --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/epoch_seconds.rs @@ -0,0 +1,151 @@ +//! Generated JSON epoch-second adapters with exact decimal millisecond precision. + +use chrono::{DateTime, Utc}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, de::Error as DeError, ser::Error as SerError, +}; +use serde_json::value::RawValue; + +const INVALID: &str = "invalid epoch-second timestamp"; + +/// Serialize a required timestamp as a numeric JSON epoch value. +/// +/// # Errors +/// Rejects leap seconds, invalid numeric representation or serializer failures. +pub fn serialize(value: &DateTime, serializer: S) -> Result { + if value.timestamp_subsec_nanos() >= 1_000_000_000 { + return Err(S::Error::custom(INVALID)); + } + let nanos = i128::from(value.timestamp()) + .checked_mul(1_000_000_000) + .and_then(|seconds| seconds.checked_add(i128::from(value.timestamp_subsec_nanos()))) + .ok_or_else(|| S::Error::custom(INVALID))?; + let millis = nanos / 1_000_000; + let sign = if millis < 0 { "-" } else { "" }; + let magnitude = millis.unsigned_abs(); + let seconds = magnitude / 1000; + let fraction = magnitude % 1000; + let number = if fraction == 0 { + format!("{sign}{seconds}") + } else { + format!("{sign}{seconds}.{fraction:03}") + }; + RawValue::from_string(number) + .map_err(|_| S::Error::custom(INVALID))? + .serialize(serializer) +} + +/// Deserialize a required JSON number, truncating submillisecond precision. +/// +/// # Errors +/// Rejects nonnumeric, oversized, out-of-range or invalid epoch values. +pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let value = Box::::deserialize(deserializer)?; + parse(value.get()).ok_or_else(|| D::Error::custom(INVALID)) +} + +fn parse(raw: &str) -> Option> { + if raw.len() > 128 { + return None; + } + let (negative, raw) = raw + .strip_prefix('-') + .map_or((false, raw), |rest| (true, rest)); + let (mantissa, exponent) = match raw.split_once(['e', 'E']) { + Some((value, exponent)) => { + let (negative, digits) = exponent.strip_prefix('-').map_or_else( + || (false, exponent.strip_prefix('+').unwrap_or(exponent)), + |digits| (true, digits), + ); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let magnitude = digits.bytes().fold(0_i32, |value, digit| { + value + .saturating_mul(10) + .saturating_add(i32::from(digit - b'0')) + .min(1024) + }); + (value, if negative { -magnitude } else { magnitude }) + } + None => (raw, 0), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if whole.is_empty() + || !whole + .bytes() + .chain(fraction.bytes()) + .all(|byte| byte.is_ascii_digit()) + { + return None; + } + let digits: Vec<_> = whole.bytes().chain(fraction.bytes()).collect(); + if digits.iter().all(|digit| *digit == b'0') { + return DateTime::from_timestamp_millis(0); + } + let keep = i32::try_from(whole.len()) + .ok()? + .checked_add(exponent)? + .checked_add(3)?; + if keep <= 0 { + return DateTime::from_timestamp_millis(0); + } + // Leading zeroes must not make a small representable exponent look huge. + let leading = digits.iter().take_while(|digit| **digit == b'0').count(); + let significant = keep.checked_sub(i32::try_from(leading).ok()?)?; + if significant <= 0 { + return DateTime::from_timestamp_millis(0); + } + if significant > 19 { + return None; + } + let mut millis = 0_i128; + let count = usize::try_from(significant).ok()?; + for offset in 0..count { + let digit = digits + .get(leading.checked_add(offset)?) + .copied() + .unwrap_or(b'0'); + millis = millis + .checked_mul(10)? + .checked_add(i128::from(digit.checked_sub(b'0')?))?; + } + let signed = if negative { + millis.checked_neg()? + } else { + millis + }; + DateTime::from_timestamp_millis(i64::try_from(signed).ok()?) +} + +/// Optional JSON timestamp adapter; missing members also need serde default. +pub mod option { + use std::borrow::Borrow; + + use super::{DateTime, DeError, Deserialize, Deserializer, INVALID, RawValue, Serializer, Utc}; + + /// Serialize a timestamp or JSON null. + /// + /// # Errors + /// Returns the required adapter's validation/serialization errors. + pub fn serialize>>>( + value: V, + serializer: S, + ) -> Result { + match value.borrow() { + Some(value) => super::serialize(value, serializer), + None => serializer.serialize_none(), + } + } + /// Deserialize a timestamp or JSON null. + /// + /// # Errors + /// Rejects invalid nonnull values with the required adapter's rules. + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result>, D::Error> { + Option::>::deserialize(deserializer)? + .map(|value| super::parse(value.get()).ok_or_else(|| D::Error::custom(INVALID))) + .transpose() + } +} diff --git a/crates/rustack-stepfunctions-model/src/error.rs b/crates/rustack-stepfunctions-model/src/error.rs new file mode 100644 index 00000000..314e9117 --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/error.rs @@ -0,0 +1,316 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. +//! +//! Step Functions errors use JSON format with a `__type` field containing the +//! short error type name (e.g., `ResourceNotFoundException`). + +use std::fmt; + +/// Well-known Step Functions error codes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[non_exhaustive] +pub enum StepFunctionsErrorCode { + /// `ActivityAlreadyExists` error. + ActivityAlreadyExists, + /// `ActivityDoesNotExist` error. + ActivityDoesNotExist, + /// `ActivityLimitExceeded` error. + ActivityLimitExceeded, + /// `ActivityWorkerLimitExceeded` error. + ActivityWorkerLimitExceeded, + /// `ConflictException` error. + ConflictException, + /// `ExecutionAlreadyExists` error. + ExecutionAlreadyExists, + /// `ExecutionDoesNotExist` error. + ExecutionDoesNotExist, + /// `ExecutionLimitExceeded` error. + ExecutionLimitExceeded, + /// `ExecutionNotRedrivable` error. + ExecutionNotRedrivable, + /// `InvalidArn` error. + InvalidArn, + /// `InvalidDefinition` error. + InvalidDefinition, + /// `InvalidEncryptionConfiguration` error. + InvalidEncryptionConfiguration, + /// `InvalidExecutionInput` error. + InvalidExecutionInput, + /// `InvalidLoggingConfiguration` error. + InvalidLoggingConfiguration, + /// `InvalidName` error. + InvalidName, + /// `InvalidOutput` error. + InvalidOutput, + /// `InvalidToken` error. + InvalidToken, + /// `InvalidTracingConfiguration` error. + InvalidTracingConfiguration, + /// `KmsAccessDeniedException` error. + KmsAccessDeniedException, + /// `KmsInvalidStateException` error. + KmsInvalidStateException, + /// `KmsThrottlingException` error. + KmsThrottlingException, + /// `MissingRequiredParameter` error. + MissingRequiredParameter, + /// `ResourceNotFound` error. + ResourceNotFound, + /// `ServiceQuotaExceededException` error. + ServiceQuotaExceededException, + /// `StateMachineAlreadyExists` error. + StateMachineAlreadyExists, + /// `StateMachineDeleting` error. + StateMachineDeleting, + /// `StateMachineDoesNotExist` error. + StateMachineDoesNotExist, + /// `StateMachineLimitExceeded` error. + StateMachineLimitExceeded, + /// `StateMachineTypeNotSupported` error. + StateMachineTypeNotSupported, + /// `TaskDoesNotExist` error. + TaskDoesNotExist, + /// `TaskTimedOut` error. + TaskTimedOut, + /// `TooManyTags` error. + TooManyTags, + /// `UnknownOperationException` error. + UnknownOperationException, + /// `UnsupportedOperation` error. + UnsupportedOperation, + /// `ValidationException` error. + #[default] + ValidationException, +} + +impl StepFunctionsErrorCode { + /// Returns the short error type string for the JSON `__type` field. + #[must_use] + pub fn error_type(&self) -> &'static str { + self.as_str() + } + + /// Returns the short error code string. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::ActivityAlreadyExists => "ActivityAlreadyExists", + Self::ActivityDoesNotExist => "ActivityDoesNotExist", + Self::ActivityLimitExceeded => "ActivityLimitExceeded", + Self::ActivityWorkerLimitExceeded => "ActivityWorkerLimitExceeded", + Self::ConflictException => "ConflictException", + Self::ExecutionAlreadyExists => "ExecutionAlreadyExists", + Self::ExecutionDoesNotExist => "ExecutionDoesNotExist", + Self::ExecutionLimitExceeded => "ExecutionLimitExceeded", + Self::ExecutionNotRedrivable => "ExecutionNotRedrivable", + Self::InvalidArn => "InvalidArn", + Self::InvalidDefinition => "InvalidDefinition", + Self::InvalidEncryptionConfiguration => "InvalidEncryptionConfiguration", + Self::InvalidExecutionInput => "InvalidExecutionInput", + Self::InvalidLoggingConfiguration => "InvalidLoggingConfiguration", + Self::InvalidName => "InvalidName", + Self::InvalidOutput => "InvalidOutput", + Self::InvalidToken => "InvalidToken", + Self::InvalidTracingConfiguration => "InvalidTracingConfiguration", + Self::KmsAccessDeniedException => "KmsAccessDeniedException", + Self::KmsInvalidStateException => "KmsInvalidStateException", + Self::KmsThrottlingException => "KmsThrottlingException", + Self::MissingRequiredParameter => "MissingRequiredParameter", + Self::ResourceNotFound => "ResourceNotFound", + Self::ServiceQuotaExceededException => "ServiceQuotaExceededException", + Self::StateMachineAlreadyExists => "StateMachineAlreadyExists", + Self::StateMachineDeleting => "StateMachineDeleting", + Self::StateMachineDoesNotExist => "StateMachineDoesNotExist", + Self::StateMachineLimitExceeded => "StateMachineLimitExceeded", + Self::StateMachineTypeNotSupported => "StateMachineTypeNotSupported", + Self::TaskDoesNotExist => "TaskDoesNotExist", + Self::TaskTimedOut => "TaskTimedOut", + Self::TooManyTags => "TooManyTags", + Self::UnknownOperationException => "UnknownOperationException", + Self::UnsupportedOperation => "UnsupportedOperation", + Self::ValidationException => "ValidationException", + } + } + + /// Returns the default HTTP status code for this error. + #[must_use] + pub fn default_status_code(&self) -> http::StatusCode { + match self { + Self::ActivityAlreadyExists + | Self::ActivityDoesNotExist + | Self::ActivityLimitExceeded + | Self::ActivityWorkerLimitExceeded + | Self::ExecutionAlreadyExists + | Self::ExecutionDoesNotExist + | Self::ExecutionLimitExceeded + | Self::ExecutionNotRedrivable + | Self::InvalidArn + | Self::InvalidDefinition + | Self::InvalidEncryptionConfiguration + | Self::InvalidExecutionInput + | Self::InvalidLoggingConfiguration + | Self::InvalidName + | Self::InvalidOutput + | Self::InvalidToken + | Self::InvalidTracingConfiguration + | Self::KmsAccessDeniedException + | Self::KmsInvalidStateException + | Self::KmsThrottlingException + | Self::MissingRequiredParameter + | Self::StateMachineAlreadyExists + | Self::StateMachineDeleting + | Self::StateMachineDoesNotExist + | Self::StateMachineLimitExceeded + | Self::StateMachineTypeNotSupported + | Self::TaskDoesNotExist + | Self::TaskTimedOut + | Self::TooManyTags + | Self::UnknownOperationException + | Self::UnsupportedOperation + | Self::ValidationException => http::StatusCode::BAD_REQUEST, + Self::ServiceQuotaExceededException => http::StatusCode::INTERNAL_SERVER_ERROR, + Self::ResourceNotFound => http::StatusCode::NOT_FOUND, + Self::ConflictException => http::StatusCode::CONFLICT, + } + } +} + +impl fmt::Display for StepFunctionsErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl StepFunctionsErrorCode { + /// Default bounded protocol message. + #[must_use] + pub fn default_message(&self) -> &'static str { + match self { + Self::ActivityAlreadyExists => "ActivityAlreadyExists", + Self::ActivityDoesNotExist => "ActivityDoesNotExist", + Self::ActivityLimitExceeded => "ActivityLimitExceeded", + Self::ActivityWorkerLimitExceeded => "ActivityWorkerLimitExceeded", + Self::ConflictException => "ConflictException", + Self::ExecutionAlreadyExists => "ExecutionAlreadyExists", + Self::ExecutionDoesNotExist => "ExecutionDoesNotExist", + Self::ExecutionLimitExceeded => "ExecutionLimitExceeded", + Self::ExecutionNotRedrivable => "ExecutionNotRedrivable", + Self::InvalidArn => "InvalidArn", + Self::InvalidDefinition => "InvalidDefinition", + Self::InvalidEncryptionConfiguration => "InvalidEncryptionConfiguration", + Self::InvalidExecutionInput => "InvalidExecutionInput", + Self::InvalidLoggingConfiguration => "InvalidLoggingConfiguration", + Self::InvalidName => "InvalidName", + Self::InvalidOutput => "InvalidOutput", + Self::InvalidToken => "InvalidToken", + Self::InvalidTracingConfiguration => "InvalidTracingConfiguration", + Self::KmsAccessDeniedException => "KmsAccessDeniedException", + Self::KmsInvalidStateException => "KmsInvalidStateException", + Self::KmsThrottlingException => "KmsThrottlingException", + Self::MissingRequiredParameter => "MissingRequiredParameter", + Self::ResourceNotFound => "ResourceNotFound", + Self::ServiceQuotaExceededException => "ServiceQuotaExceededException", + Self::StateMachineAlreadyExists => "StateMachineAlreadyExists", + Self::StateMachineDeleting => "StateMachineDeleting", + Self::StateMachineDoesNotExist => "StateMachineDoesNotExist", + Self::StateMachineLimitExceeded => "StateMachineLimitExceeded", + Self::StateMachineTypeNotSupported => "StateMachineTypeNotSupported", + Self::TaskDoesNotExist => "TaskDoesNotExist", + Self::TaskTimedOut => "TaskTimedOut", + Self::TooManyTags => "TooManyTags", + Self::UnknownOperationException => "Unknown Step Functions operation", + Self::UnsupportedOperation => { + "Operation is outside the supported Step Functions profile" + } + Self::ValidationException => "ValidationException", + } + } +} +/// An Step Functions error response. +#[non_exhaustive] +pub struct StepFunctionsError { + /// The error code. + pub code: StepFunctionsErrorCode, + /// A human-readable error message. + pub message: String, + /// The HTTP status code. + pub status_code: http::StatusCode, + /// The underlying source error, if any. + pub source: Option>, +} + +impl fmt::Debug for StepFunctionsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsError()") + } +} +impl fmt::Display for StepFunctionsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsError()") + } +} + +impl std::error::Error for StepFunctionsError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source + .as_ref() + .map(|e| e.as_ref() as &(dyn std::error::Error + 'static)) + } +} + +impl StepFunctionsError { + /// Create a new `StepFunctionsError` from an error code. + #[must_use] + pub fn new(code: StepFunctionsErrorCode) -> Self { + Self { + status_code: code.default_status_code(), + message: code.default_message().to_owned(), + code, + source: None, + } + } + + /// Create a new `StepFunctionsError` with a custom message. + #[must_use] + pub fn with_message(code: StepFunctionsErrorCode, message: impl Into) -> Self { + Self { + status_code: code.default_status_code(), + message: message.into(), + code, + source: None, + } + } + + /// Returns the `__type` string for the JSON error response. + #[must_use] + pub fn error_type(&self) -> &'static str { + self.code.error_type() + } + + /// Validation error. + #[must_use] + pub fn validation(message: impl Into) -> Self { + Self::with_message(StepFunctionsErrorCode::ValidationException, message) + } +} + +/// Create an `StepFunctionsError` from an error code. +/// +/// # Examples +/// +/// ``` +/// use rustack_stepfunctions_model::{stepfunctions_error, StepFunctionsErrorCode}; +/// let err = stepfunctions_error!(ValidationException); +/// assert_eq!(err.code, StepFunctionsErrorCode::ValidationException); +/// ``` +#[macro_export] +macro_rules! stepfunctions_error { + ($code:ident) => { + $crate::error::StepFunctionsError::new($crate::error::StepFunctionsErrorCode::$code) + }; + ($code:ident, $msg:expr) => { + $crate::error::StepFunctionsError::with_message( + $crate::error::StepFunctionsErrorCode::$code, + $msg, + ) + }; +} diff --git a/crates/rustack-stepfunctions-model/src/input.rs b/crates/rustack-stepfunctions-model/src/input.rs new file mode 100644 index 00000000..63760247 --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/input.rs @@ -0,0 +1,852 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. + +use serde::{Deserialize, Serialize}; + +use crate::types::{ + EncryptionConfiguration, ExecutionRedriveFilter, ExecutionStatus, IncludedData, + InspectionLevel, LoggingConfiguration, MockInput, RoutingConfigurationListItem, + StateMachineType, Tag, TestStateConfiguration, TracingConfiguration, + ValidateStateMachineDefinitionSeverity, +}; + +/// Step Functions `CreateActivityInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CreateActivityInput { + /// The `encryptionConfiguration` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `name` wire member. + pub name: String, + /// The `tags` wire member. + #[serde(default)] + pub tags: Vec, +} + +impl std::fmt::Debug for CreateActivityInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateActivityInput") + .finish_non_exhaustive() + } +} +/// Step Functions `CreateStateMachineAliasInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CreateStateMachineAliasInput { + /// The `description` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The `name` wire member. + pub name: String, + /// The `routingConfiguration` wire member. + pub routing_configuration: Vec, +} + +impl std::fmt::Debug for CreateStateMachineAliasInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateStateMachineAliasInput") + .finish_non_exhaustive() + } +} +/// Step Functions `CreateStateMachineInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct CreateStateMachineInput { + /// The `definition` wire member. + pub definition: String, + /// The `encryptionConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `loggingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub logging_configuration: Option, + /// The `name` wire member. + pub name: String, + /// The `publish` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub publish: Option, + /// The `roleArn` wire member. + pub role_arn: String, + /// The `tags` wire member. + #[builder(default)] + #[serde(default)] + pub tags: Vec, + /// The `tracingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tracing_configuration: Option, + /// The `type` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + /// The `versionDescription` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub version_description: Option, +} + +impl std::fmt::Debug for CreateStateMachineInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateStateMachineInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteActivityInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteActivityInput { + /// The `activityArn` wire member. + pub activity_arn: String, +} + +impl std::fmt::Debug for DeleteActivityInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteActivityInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineAliasInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineAliasInput { + /// The `stateMachineAliasArn` wire member. + pub state_machine_alias_arn: String, +} + +impl std::fmt::Debug for DeleteStateMachineAliasInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineAliasInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineInput { + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, +} + +impl std::fmt::Debug for DeleteStateMachineInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineVersionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineVersionInput { + /// The `stateMachineVersionArn` wire member. + pub state_machine_version_arn: String, +} + +impl std::fmt::Debug for DeleteStateMachineVersionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineVersionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeActivityInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeActivityInput { + /// The `activityArn` wire member. + pub activity_arn: String, +} + +impl std::fmt::Debug for DescribeActivityInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeActivityInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeExecutionInput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `includedData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub included_data: Option, +} + +impl std::fmt::Debug for DescribeExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeExecutionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeMapRunInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeMapRunInput { + /// The `mapRunArn` wire member. + pub map_run_arn: String, +} + +impl std::fmt::Debug for DescribeMapRunInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeMapRunInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineAliasInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeStateMachineAliasInput { + /// The `stateMachineAliasArn` wire member. + pub state_machine_alias_arn: String, +} + +impl std::fmt::Debug for DescribeStateMachineAliasInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineAliasInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineForExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeStateMachineForExecutionInput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `includedData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub included_data: Option, +} + +impl std::fmt::Debug for DescribeStateMachineForExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineForExecutionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeStateMachineInput { + /// The `includedData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub included_data: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, +} + +impl std::fmt::Debug for DescribeStateMachineInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineInput") + .finish_non_exhaustive() + } +} +/// Step Functions `GetActivityTaskInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct GetActivityTaskInput { + /// The `activityArn` wire member. + pub activity_arn: String, + /// The `workerName` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_name: Option, +} + +impl std::fmt::Debug for GetActivityTaskInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GetActivityTaskInput") + .finish_non_exhaustive() + } +} +/// Step Functions `GetExecutionHistoryInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct GetExecutionHistoryInput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `includeExecutionData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub include_execution_data: Option, + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `reverseOrder` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub reverse_order: Option, +} + +impl std::fmt::Debug for GetExecutionHistoryInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GetExecutionHistoryInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListActivitiesInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListActivitiesInput { + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListActivitiesInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListActivitiesInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListExecutionsInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct ListExecutionsInput { + /// The `mapRunArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, + /// The `maxResults` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `redriveFilter` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_filter: Option, + /// The `stateMachineArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_arn: Option, + /// The `statusFilter` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub status_filter: Option, +} + +impl std::fmt::Debug for ListExecutionsInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListExecutionsInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListMapRunsInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListMapRunsInput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListMapRunsInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListMapRunsInput").finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachineAliasesInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachineAliasesInput { + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, +} + +impl std::fmt::Debug for ListStateMachineAliasesInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachineAliasesInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachineVersionsInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachineVersionsInput { + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, +} + +impl std::fmt::Debug for ListStateMachineVersionsInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachineVersionsInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachinesInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachinesInput { + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListStateMachinesInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachinesInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListTagsForResourceInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListTagsForResourceInput { + /// The `resourceArn` wire member. + pub resource_arn: String, +} + +impl std::fmt::Debug for ListTagsForResourceInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListTagsForResourceInput") + .finish_non_exhaustive() + } +} +/// Step Functions `PublishStateMachineVersionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct PublishStateMachineVersionInput { + /// The `description` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The `revisionId` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, +} + +impl std::fmt::Debug for PublishStateMachineVersionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishStateMachineVersionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `RedriveExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct RedriveExecutionInput { + /// The `clientToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub client_token: Option, + /// The `executionArn` wire member. + pub execution_arn: String, +} + +impl std::fmt::Debug for RedriveExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RedriveExecutionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskFailureInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskFailureInput { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `taskToken` wire member. + pub task_token: String, +} + +impl std::fmt::Debug for SendTaskFailureInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskFailureInput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskHeartbeatInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskHeartbeatInput { + /// The `taskToken` wire member. + pub task_token: String, +} + +impl std::fmt::Debug for SendTaskHeartbeatInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskHeartbeatInput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskSuccessInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskSuccessInput { + /// The `output` wire member. + pub output: String, + /// The `taskToken` wire member. + pub task_token: String, +} + +impl std::fmt::Debug for SendTaskSuccessInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskSuccessInput") + .finish_non_exhaustive() + } +} +/// Step Functions `StartExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StartExecutionInput { + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `name` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `traceHeader` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_header: Option, +} + +impl std::fmt::Debug for StartExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StartExecutionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `StartSyncExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StartSyncExecutionInput { + /// The `includedData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub included_data: Option, + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `name` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `traceHeader` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_header: Option, +} + +impl std::fmt::Debug for StartSyncExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StartSyncExecutionInput") + .finish_non_exhaustive() + } +} +/// Step Functions `StopExecutionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StopExecutionInput { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `executionArn` wire member. + pub execution_arn: String, +} + +impl std::fmt::Debug for StopExecutionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StopExecutionInput").finish_non_exhaustive() + } +} +/// Step Functions `TagResourceInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TagResourceInput { + /// The `resourceArn` wire member. + pub resource_arn: String, + /// The `tags` wire member. + pub tags: Vec, +} + +impl std::fmt::Debug for TagResourceInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TagResourceInput").finish_non_exhaustive() + } +} +/// Step Functions `TestStateInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct TestStateInput { + /// The `context` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + /// The `definition` wire member. + pub definition: String, + /// The `input` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inspectionLevel` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub inspection_level: Option, + /// The `mock` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub mock: Option, + /// The `revealSecrets` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub reveal_secrets: Option, + /// The `roleArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub role_arn: Option, + /// The `stateConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_configuration: Option, + /// The `stateName` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_name: Option, + /// The `variables` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub variables: Option, +} + +impl std::fmt::Debug for TestStateInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestStateInput").finish_non_exhaustive() + } +} +/// Step Functions `UntagResourceInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UntagResourceInput { + /// The `resourceArn` wire member. + pub resource_arn: String, + /// The `tagKeys` wire member. + pub tag_keys: Vec, +} + +impl std::fmt::Debug for UntagResourceInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UntagResourceInput").finish_non_exhaustive() + } +} +/// Step Functions `UpdateMapRunInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UpdateMapRunInput { + /// The `mapRunArn` wire member. + pub map_run_arn: String, + /// The `maxConcurrency` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_concurrency: Option, + /// The `toleratedFailureCount` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub tolerated_failure_count: Option, + /// The `toleratedFailurePercentage` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub tolerated_failure_percentage: Option, +} + +impl std::fmt::Debug for UpdateMapRunInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateMapRunInput").finish_non_exhaustive() + } +} +/// Step Functions `UpdateStateMachineAliasInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UpdateStateMachineAliasInput { + /// The `description` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The `routingConfiguration` wire member. + #[serde(default)] + pub routing_configuration: Vec, + /// The `stateMachineAliasArn` wire member. + pub state_machine_alias_arn: String, +} + +impl std::fmt::Debug for UpdateStateMachineAliasInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateStateMachineAliasInput") + .finish_non_exhaustive() + } +} +/// Step Functions `UpdateStateMachineInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct UpdateStateMachineInput { + /// The `definition` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub definition: Option, + /// The `encryptionConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `loggingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub logging_configuration: Option, + /// The `publish` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub publish: Option, + /// The `roleArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub role_arn: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `tracingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tracing_configuration: Option, + /// The `versionDescription` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub version_description: Option, +} + +impl std::fmt::Debug for UpdateStateMachineInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateStateMachineInput") + .finish_non_exhaustive() + } +} +/// Step Functions `ValidateStateMachineDefinitionInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ValidateStateMachineDefinitionInput { + /// The `definition` wire member. + pub definition: String, + /// The `maxResults` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + /// The `severity` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub severity: Option, + /// The `type` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, +} + +impl std::fmt::Debug for ValidateStateMachineDefinitionInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValidateStateMachineDefinitionInput") + .finish_non_exhaustive() + } +} diff --git a/crates/rustack-stepfunctions-model/src/lib.rs b/crates/rustack-stepfunctions-model/src/lib.rs new file mode 100644 index 00000000..782b711f --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/lib.rs @@ -0,0 +1,18 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. +#![forbid(unsafe_code)] +pub(crate) mod epoch_seconds; + +pub mod error; +pub mod input; +pub mod operations; +pub mod output; +pub mod types; +/// JSON protocol target prefix. +pub const TARGET_PREFIX: &str = "AWSStepFunctions"; +/// Whether the service config requests AWS Query-compatible errors. +pub const AWS_QUERY_COMPATIBLE: bool = false; + +pub use error::{StepFunctionsError, StepFunctionsErrorCode}; +pub use operations::StepFunctionsOperation; + +// Generated: 37 operations, 16 enums, 63 shared structs, 37 input structs, 37 output structs diff --git a/crates/rustack-stepfunctions-model/src/operations.rs b/crates/rustack-stepfunctions-model/src/operations.rs new file mode 100644 index 00000000..37525787 --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/operations.rs @@ -0,0 +1,177 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. + +/// All supported `StepFunctionsOperation` values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StepFunctionsOperation { + /// The `CreateActivity` operation. + CreateActivity, + /// The `CreateStateMachine` operation. + CreateStateMachine, + /// The `CreateStateMachineAlias` operation. + CreateStateMachineAlias, + /// The `DeleteActivity` operation. + DeleteActivity, + /// The `DeleteStateMachine` operation. + DeleteStateMachine, + /// The `DeleteStateMachineAlias` operation. + DeleteStateMachineAlias, + /// The `DeleteStateMachineVersion` operation. + DeleteStateMachineVersion, + /// The `DescribeActivity` operation. + DescribeActivity, + /// The `DescribeExecution` operation. + DescribeExecution, + /// The `DescribeMapRun` operation. + DescribeMapRun, + /// The `DescribeStateMachine` operation. + DescribeStateMachine, + /// The `DescribeStateMachineAlias` operation. + DescribeStateMachineAlias, + /// The `DescribeStateMachineForExecution` operation. + DescribeStateMachineForExecution, + /// The `GetActivityTask` operation. + GetActivityTask, + /// The `GetExecutionHistory` operation. + GetExecutionHistory, + /// The `ListActivities` operation. + ListActivities, + /// The `ListExecutions` operation. + ListExecutions, + /// The `ListMapRuns` operation. + ListMapRuns, + /// The `ListStateMachineAliases` operation. + ListStateMachineAliases, + /// The `ListStateMachines` operation. + ListStateMachines, + /// The `ListStateMachineVersions` operation. + ListStateMachineVersions, + /// The `ListTagsForResource` operation. + ListTagsForResource, + /// The `PublishStateMachineVersion` operation. + PublishStateMachineVersion, + /// The `RedriveExecution` operation. + RedriveExecution, + /// The `SendTaskFailure` operation. + SendTaskFailure, + /// The `SendTaskHeartbeat` operation. + SendTaskHeartbeat, + /// The `SendTaskSuccess` operation. + SendTaskSuccess, + /// The `StartExecution` operation. + StartExecution, + /// The `StartSyncExecution` operation. + StartSyncExecution, + /// The `StopExecution` operation. + StopExecution, + /// The `TagResource` operation. + TagResource, + /// The `TestState` operation. + TestState, + /// The `UntagResource` operation. + UntagResource, + /// The `UpdateMapRun` operation. + UpdateMapRun, + /// The `UpdateStateMachine` operation. + UpdateStateMachine, + /// The `UpdateStateMachineAlias` operation. + UpdateStateMachineAlias, + /// The `ValidateStateMachineDefinition` operation. + ValidateStateMachineDefinition, +} + +impl StepFunctionsOperation { + /// Returns the AWS operation name string. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::CreateActivity => "CreateActivity", + Self::CreateStateMachine => "CreateStateMachine", + Self::CreateStateMachineAlias => "CreateStateMachineAlias", + Self::DeleteActivity => "DeleteActivity", + Self::DeleteStateMachine => "DeleteStateMachine", + Self::DeleteStateMachineAlias => "DeleteStateMachineAlias", + Self::DeleteStateMachineVersion => "DeleteStateMachineVersion", + Self::DescribeActivity => "DescribeActivity", + Self::DescribeExecution => "DescribeExecution", + Self::DescribeMapRun => "DescribeMapRun", + Self::DescribeStateMachine => "DescribeStateMachine", + Self::DescribeStateMachineAlias => "DescribeStateMachineAlias", + Self::DescribeStateMachineForExecution => "DescribeStateMachineForExecution", + Self::GetActivityTask => "GetActivityTask", + Self::GetExecutionHistory => "GetExecutionHistory", + Self::ListActivities => "ListActivities", + Self::ListExecutions => "ListExecutions", + Self::ListMapRuns => "ListMapRuns", + Self::ListStateMachineAliases => "ListStateMachineAliases", + Self::ListStateMachines => "ListStateMachines", + Self::ListStateMachineVersions => "ListStateMachineVersions", + Self::ListTagsForResource => "ListTagsForResource", + Self::PublishStateMachineVersion => "PublishStateMachineVersion", + Self::RedriveExecution => "RedriveExecution", + Self::SendTaskFailure => "SendTaskFailure", + Self::SendTaskHeartbeat => "SendTaskHeartbeat", + Self::SendTaskSuccess => "SendTaskSuccess", + Self::StartExecution => "StartExecution", + Self::StartSyncExecution => "StartSyncExecution", + Self::StopExecution => "StopExecution", + Self::TagResource => "TagResource", + Self::TestState => "TestState", + Self::UntagResource => "UntagResource", + Self::UpdateMapRun => "UpdateMapRun", + Self::UpdateStateMachine => "UpdateStateMachine", + Self::UpdateStateMachineAlias => "UpdateStateMachineAlias", + Self::ValidateStateMachineDefinition => "ValidateStateMachineDefinition", + } + } + + /// Parse an operation name string into a `StepFunctionsOperation`. + #[must_use] + pub fn from_name(name: &str) -> Option { + match name { + "CreateActivity" => Some(Self::CreateActivity), + "CreateStateMachine" => Some(Self::CreateStateMachine), + "CreateStateMachineAlias" => Some(Self::CreateStateMachineAlias), + "DeleteActivity" => Some(Self::DeleteActivity), + "DeleteStateMachine" => Some(Self::DeleteStateMachine), + "DeleteStateMachineAlias" => Some(Self::DeleteStateMachineAlias), + "DeleteStateMachineVersion" => Some(Self::DeleteStateMachineVersion), + "DescribeActivity" => Some(Self::DescribeActivity), + "DescribeExecution" => Some(Self::DescribeExecution), + "DescribeMapRun" => Some(Self::DescribeMapRun), + "DescribeStateMachine" => Some(Self::DescribeStateMachine), + "DescribeStateMachineAlias" => Some(Self::DescribeStateMachineAlias), + "DescribeStateMachineForExecution" => Some(Self::DescribeStateMachineForExecution), + "GetActivityTask" => Some(Self::GetActivityTask), + "GetExecutionHistory" => Some(Self::GetExecutionHistory), + "ListActivities" => Some(Self::ListActivities), + "ListExecutions" => Some(Self::ListExecutions), + "ListMapRuns" => Some(Self::ListMapRuns), + "ListStateMachineAliases" => Some(Self::ListStateMachineAliases), + "ListStateMachines" => Some(Self::ListStateMachines), + "ListStateMachineVersions" => Some(Self::ListStateMachineVersions), + "ListTagsForResource" => Some(Self::ListTagsForResource), + "PublishStateMachineVersion" => Some(Self::PublishStateMachineVersion), + "RedriveExecution" => Some(Self::RedriveExecution), + "SendTaskFailure" => Some(Self::SendTaskFailure), + "SendTaskHeartbeat" => Some(Self::SendTaskHeartbeat), + "SendTaskSuccess" => Some(Self::SendTaskSuccess), + "StartExecution" => Some(Self::StartExecution), + "StartSyncExecution" => Some(Self::StartSyncExecution), + "StopExecution" => Some(Self::StopExecution), + "TagResource" => Some(Self::TagResource), + "TestState" => Some(Self::TestState), + "UntagResource" => Some(Self::UntagResource), + "UpdateMapRun" => Some(Self::UpdateMapRun), + "UpdateStateMachine" => Some(Self::UpdateStateMachine), + "UpdateStateMachineAlias" => Some(Self::UpdateStateMachineAlias), + "ValidateStateMachineDefinition" => Some(Self::ValidateStateMachineDefinition), + _ => None, + } + } +} + +impl std::fmt::Display for StepFunctionsOperation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} diff --git a/crates/rustack-stepfunctions-model/src/output.rs b/crates/rustack-stepfunctions-model/src/output.rs new file mode 100644 index 00000000..e24d5dbe --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/output.rs @@ -0,0 +1,926 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::types::{ + ActivityListItem, BillingDetails, CloudWatchEventsExecutionDataDetails, + EncryptionConfiguration, ExecutionListItem, ExecutionRedriveStatus, ExecutionStatus, + HistoryEvent, InspectionData, LoggingConfiguration, MapRunExecutionCounts, MapRunItemCounts, + MapRunListItem, MapRunStatus, RoutingConfigurationListItem, StateMachineAliasListItem, + StateMachineListItem, StateMachineStatus, StateMachineType, StateMachineVersionListItem, + SyncExecutionStatus, Tag, TestExecutionStatus, TracingConfiguration, + ValidateStateMachineDefinitionDiagnostic, ValidateStateMachineDefinitionResultCode, +}; + +/// Step Functions `CreateActivityOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CreateActivityOutput { + /// The `activityArn` wire member. + pub activity_arn: String, + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, +} + +impl std::fmt::Debug for CreateActivityOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateActivityOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `CreateStateMachineAliasOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CreateStateMachineAliasOutput { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `stateMachineAliasArn` wire member. + pub state_machine_alias_arn: String, +} + +impl std::fmt::Debug for CreateStateMachineAliasOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateStateMachineAliasOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `CreateStateMachineOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CreateStateMachineOutput { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `stateMachineVersionArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_version_arn: Option, +} + +impl std::fmt::Debug for CreateStateMachineOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CreateStateMachineOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteActivityOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteActivityOutput {} + +impl std::fmt::Debug for DeleteActivityOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteActivityOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineAliasOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineAliasOutput {} + +impl std::fmt::Debug for DeleteStateMachineAliasOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineAliasOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineOutput {} + +impl std::fmt::Debug for DeleteStateMachineOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DeleteStateMachineVersionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DeleteStateMachineVersionOutput {} + +impl std::fmt::Debug for DeleteStateMachineVersionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeleteStateMachineVersionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeActivityOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct DescribeActivityOutput { + /// The `activityArn` wire member. + pub activity_arn: String, + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `encryptionConfiguration` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `name` wire member. + pub name: String, +} + +impl std::fmt::Debug for DescribeActivityOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeActivityOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct DescribeExecutionOutput { + /// The `cause` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `input` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `mapRunArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, + /// The `name` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The `output` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, + /// The `redriveCount` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_count: Option, + /// The `redriveDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_date: Option>, + /// The `redriveStatus` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_status: Option, + /// The `redriveStatusReason` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_status_reason: Option, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, + /// The `stateMachineAliasArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_alias_arn: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `stateMachineVersionArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_version_arn: Option, + /// The `status` wire member. + pub status: ExecutionStatus, + /// The `stopDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_date: Option>, + /// The `traceHeader` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_header: Option, +} + +impl std::fmt::Debug for DescribeExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeMapRunOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct DescribeMapRunOutput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `executionCounts` wire member. + pub execution_counts: MapRunExecutionCounts, + /// The `itemCounts` wire member. + pub item_counts: MapRunItemCounts, + /// The `mapRunArn` wire member. + pub map_run_arn: String, + /// The `maxConcurrency` wire member. + pub max_concurrency: i32, + /// The `redriveCount` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_count: Option, + /// The `redriveDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_date: Option>, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, + /// The `status` wire member. + pub status: MapRunStatus, + /// The `stopDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_date: Option>, + /// The `toleratedFailureCount` wire member. + pub tolerated_failure_count: i64, + /// The `toleratedFailurePercentage` wire member. + pub tolerated_failure_percentage: f32, +} + +impl std::fmt::Debug for DescribeMapRunOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeMapRunOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineAliasOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct DescribeStateMachineAliasOutput { + /// The `creationDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub creation_date: Option>, + /// The `description` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The `name` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The `routingConfiguration` wire member. + #[builder(default)] + #[serde(default)] + pub routing_configuration: Vec, + /// The `stateMachineAliasArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_alias_arn: Option, + /// The `updateDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub update_date: Option>, +} + +impl std::fmt::Debug for DescribeStateMachineAliasOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineAliasOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineForExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct DescribeStateMachineForExecutionOutput { + /// The `definition` wire member. + pub definition: String, + /// The `encryptionConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `label` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + /// The `loggingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub logging_configuration: Option, + /// The `mapRunArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, + /// The `name` wire member. + pub name: String, + /// The `revisionId` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, + /// The `roleArn` wire member. + pub role_arn: String, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `tracingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tracing_configuration: Option, + /// The `updateDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub update_date: chrono::DateTime, + /// The `variableReferences` wire member. + #[builder(default)] + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub variable_references: HashMap>, +} + +impl std::fmt::Debug for DescribeStateMachineForExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineForExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `DescribeStateMachineOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct DescribeStateMachineOutput { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `definition` wire member. + pub definition: String, + /// The `description` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The `encryptionConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_configuration: Option, + /// The `label` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + /// The `loggingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub logging_configuration: Option, + /// The `name` wire member. + pub name: String, + /// The `revisionId` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, + /// The `roleArn` wire member. + pub role_arn: String, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `status` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + /// The `tracingConfiguration` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tracing_configuration: Option, + /// The `type` wire member. + pub r#type: StateMachineType, + /// The `variableReferences` wire member. + #[builder(default)] + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub variable_references: HashMap>, +} + +impl std::fmt::Debug for DescribeStateMachineOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DescribeStateMachineOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `GetActivityTaskOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct GetActivityTaskOutput { + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `taskToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub task_token: Option, +} + +impl std::fmt::Debug for GetActivityTaskOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GetActivityTaskOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `GetExecutionHistoryOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct GetExecutionHistoryOutput { + /// The `events` wire member. + pub events: Vec, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for GetExecutionHistoryOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GetExecutionHistoryOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListActivitiesOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListActivitiesOutput { + /// The `activities` wire member. + pub activities: Vec, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListActivitiesOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListActivitiesOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListExecutionsOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListExecutionsOutput { + /// The `executions` wire member. + pub executions: Vec, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListExecutionsOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListExecutionsOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListMapRunsOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListMapRunsOutput { + /// The `mapRuns` wire member. + pub map_runs: Vec, + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, +} + +impl std::fmt::Debug for ListMapRunsOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListMapRunsOutput").finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachineAliasesOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachineAliasesOutput { + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `stateMachineAliases` wire member. + pub state_machine_aliases: Vec, +} + +impl std::fmt::Debug for ListStateMachineAliasesOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachineAliasesOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachineVersionsOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachineVersionsOutput { + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `stateMachineVersions` wire member. + pub state_machine_versions: Vec, +} + +impl std::fmt::Debug for ListStateMachineVersionsOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachineVersionsOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListStateMachinesOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListStateMachinesOutput { + /// The `nextToken` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_token: Option, + /// The `stateMachines` wire member. + pub state_machines: Vec, +} + +impl std::fmt::Debug for ListStateMachinesOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListStateMachinesOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ListTagsForResourceOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ListTagsForResourceOutput { + /// The `tags` wire member. + #[serde(default)] + pub tags: Vec, +} + +impl std::fmt::Debug for ListTagsForResourceOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListTagsForResourceOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `PublishStateMachineVersionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct PublishStateMachineVersionOutput { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `stateMachineVersionArn` wire member. + pub state_machine_version_arn: String, +} + +impl std::fmt::Debug for PublishStateMachineVersionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishStateMachineVersionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `RedriveExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct RedriveExecutionOutput { + /// The `redriveDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub redrive_date: chrono::DateTime, +} + +impl std::fmt::Debug for RedriveExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RedriveExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskFailureOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskFailureOutput {} + +impl std::fmt::Debug for SendTaskFailureOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskFailureOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskHeartbeatOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskHeartbeatOutput {} + +impl std::fmt::Debug for SendTaskHeartbeatOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskHeartbeatOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `SendTaskSuccessOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct SendTaskSuccessOutput {} + +impl std::fmt::Debug for SendTaskSuccessOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendTaskSuccessOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `StartExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StartExecutionOutput { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, +} + +impl std::fmt::Debug for StartExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StartExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `StartSyncExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct StartSyncExecutionOutput { + /// The `billingDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub billing_details: Option, + /// The `cause` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `input` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `name` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The `output` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, + /// The `stateMachineArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_arn: Option, + /// The `status` wire member. + pub status: SyncExecutionStatus, + /// The `stopDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub stop_date: chrono::DateTime, + /// The `traceHeader` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_header: Option, +} + +impl std::fmt::Debug for StartSyncExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StartSyncExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `StopExecutionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StopExecutionOutput { + /// The `stopDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub stop_date: chrono::DateTime, +} + +impl std::fmt::Debug for StopExecutionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StopExecutionOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `TagResourceOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TagResourceOutput {} + +impl std::fmt::Debug for TagResourceOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TagResourceOutput").finish_non_exhaustive() + } +} +/// Step Functions `TestStateOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct TestStateOutput { + /// The `cause` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `inspectionData` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub inspection_data: Option, + /// The `nextState` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub next_state: Option, + /// The `output` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `status` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +impl std::fmt::Debug for TestStateOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestStateOutput").finish_non_exhaustive() + } +} +/// Step Functions `UntagResourceOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UntagResourceOutput {} + +impl std::fmt::Debug for UntagResourceOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UntagResourceOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `UpdateMapRunOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UpdateMapRunOutput {} + +impl std::fmt::Debug for UpdateMapRunOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateMapRunOutput").finish_non_exhaustive() + } +} +/// Step Functions `UpdateStateMachineAliasOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UpdateStateMachineAliasOutput { + /// The `updateDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub update_date: chrono::DateTime, +} + +impl std::fmt::Debug for UpdateStateMachineAliasOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateStateMachineAliasOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `UpdateStateMachineOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct UpdateStateMachineOutput { + /// The `revisionId` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub revision_id: Option, + /// The `stateMachineVersionArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_version_arn: Option, + /// The `updateDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub update_date: chrono::DateTime, +} + +impl std::fmt::Debug for UpdateStateMachineOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateStateMachineOutput") + .finish_non_exhaustive() + } +} +/// Step Functions `ValidateStateMachineDefinitionOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ValidateStateMachineDefinitionOutput { + /// The `diagnostics` wire member. + pub diagnostics: Vec, + /// The `result` wire member. + pub result: ValidateStateMachineDefinitionResultCode, + /// The `truncated` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub truncated: Option, +} + +impl std::fmt::Debug for ValidateStateMachineDefinitionOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValidateStateMachineDefinitionOutput") + .finish_non_exhaustive() + } +} diff --git a/crates/rustack-stepfunctions-model/src/types.rs b/crates/rustack-stepfunctions-model/src/types.rs new file mode 100644 index 00000000..4024d0d5 --- /dev/null +++ b/crates/rustack-stepfunctions-model/src/types.rs @@ -0,0 +1,2572 @@ +//! Auto-generated from AWS Step Functions Smithy model. DO NOT EDIT. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Step Functions `EncryptionType` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum EncryptionType { + /// The `AWS_OWNED_KEY` wire variant. + #[default] + #[serde(rename = "AWS_OWNED_KEY")] + AwsOwnedKey, + /// The `CUSTOMER_MANAGED_KMS_KEY` wire variant. + #[serde(rename = "CUSTOMER_MANAGED_KMS_KEY")] + CustomerManagedKmsKey, +} + +impl EncryptionType { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::AwsOwnedKey => "AWS_OWNED_KEY", + Self::CustomerManagedKmsKey => "CUSTOMER_MANAGED_KMS_KEY", + } + } +} + +impl std::fmt::Display for EncryptionType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for EncryptionType { + fn from(s: &str) -> Self { + match s { + "AWS_OWNED_KEY" => Self::AwsOwnedKey, + "CUSTOMER_MANAGED_KMS_KEY" => Self::CustomerManagedKmsKey, + _ => Self::default(), + } + } +} + +/// Step Functions `ExecutionRedriveFilter` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum ExecutionRedriveFilter { + /// The `NOT_REDRIVEN` wire variant. + #[default] + #[serde(rename = "NOT_REDRIVEN")] + NotRedriven, + /// The `REDRIVEN` wire variant. + #[serde(rename = "REDRIVEN")] + Redriven, +} + +impl ExecutionRedriveFilter { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::NotRedriven => "NOT_REDRIVEN", + Self::Redriven => "REDRIVEN", + } + } +} + +impl std::fmt::Display for ExecutionRedriveFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for ExecutionRedriveFilter { + fn from(s: &str) -> Self { + match s { + "NOT_REDRIVEN" => Self::NotRedriven, + "REDRIVEN" => Self::Redriven, + _ => Self::default(), + } + } +} + +/// Step Functions `ExecutionRedriveStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum ExecutionRedriveStatus { + /// The `NOT_REDRIVABLE` wire variant. + #[default] + #[serde(rename = "NOT_REDRIVABLE")] + NotRedrivable, + /// The `REDRIVABLE` wire variant. + #[serde(rename = "REDRIVABLE")] + Redrivable, + /// The `REDRIVABLE_BY_MAP_RUN` wire variant. + #[serde(rename = "REDRIVABLE_BY_MAP_RUN")] + RedrivableByMapRun, +} + +impl ExecutionRedriveStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::NotRedrivable => "NOT_REDRIVABLE", + Self::Redrivable => "REDRIVABLE", + Self::RedrivableByMapRun => "REDRIVABLE_BY_MAP_RUN", + } + } +} + +impl std::fmt::Display for ExecutionRedriveStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for ExecutionRedriveStatus { + fn from(s: &str) -> Self { + match s { + "NOT_REDRIVABLE" => Self::NotRedrivable, + "REDRIVABLE" => Self::Redrivable, + "REDRIVABLE_BY_MAP_RUN" => Self::RedrivableByMapRun, + _ => Self::default(), + } + } +} + +/// Step Functions `ExecutionStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum ExecutionStatus { + /// The `ABORTED` wire variant. + #[default] + #[serde(rename = "ABORTED")] + Aborted, + /// The `FAILED` wire variant. + #[serde(rename = "FAILED")] + Failed, + /// The `PENDING_REDRIVE` wire variant. + #[serde(rename = "PENDING_REDRIVE")] + PendingRedrive, + /// The `RUNNING` wire variant. + #[serde(rename = "RUNNING")] + Running, + /// The `SUCCEEDED` wire variant. + #[serde(rename = "SUCCEEDED")] + Succeeded, + /// The `TIMED_OUT` wire variant. + #[serde(rename = "TIMED_OUT")] + TimedOut, +} + +impl ExecutionStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Aborted => "ABORTED", + Self::Failed => "FAILED", + Self::PendingRedrive => "PENDING_REDRIVE", + Self::Running => "RUNNING", + Self::Succeeded => "SUCCEEDED", + Self::TimedOut => "TIMED_OUT", + } + } +} + +impl std::fmt::Display for ExecutionStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for ExecutionStatus { + fn from(s: &str) -> Self { + match s { + "ABORTED" => Self::Aborted, + "FAILED" => Self::Failed, + "PENDING_REDRIVE" => Self::PendingRedrive, + "RUNNING" => Self::Running, + "SUCCEEDED" => Self::Succeeded, + "TIMED_OUT" => Self::TimedOut, + _ => Self::default(), + } + } +} + +/// Step Functions `HistoryEventType` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum HistoryEventType { + /// The `ActivityFailed` wire variant. + #[default] + ActivityFailed, + /// The `ActivityScheduleFailed` wire variant. + ActivityScheduleFailed, + /// The `ActivityScheduled` wire variant. + ActivityScheduled, + /// The `ActivityStarted` wire variant. + ActivityStarted, + /// The `ActivitySucceeded` wire variant. + ActivitySucceeded, + /// The `ActivityTimedOut` wire variant. + ActivityTimedOut, + /// The `ChoiceStateEntered` wire variant. + ChoiceStateEntered, + /// The `ChoiceStateExited` wire variant. + ChoiceStateExited, + /// The `EvaluationFailed` wire variant. + EvaluationFailed, + /// The `ExecutionAborted` wire variant. + ExecutionAborted, + /// The `ExecutionFailed` wire variant. + ExecutionFailed, + /// The `ExecutionRedriven` wire variant. + ExecutionRedriven, + /// The `ExecutionStarted` wire variant. + ExecutionStarted, + /// The `ExecutionSucceeded` wire variant. + ExecutionSucceeded, + /// The `ExecutionTimedOut` wire variant. + ExecutionTimedOut, + /// The `FailStateEntered` wire variant. + FailStateEntered, + /// The `LambdaFunctionFailed` wire variant. + LambdaFunctionFailed, + /// The `LambdaFunctionScheduleFailed` wire variant. + LambdaFunctionScheduleFailed, + /// The `LambdaFunctionScheduled` wire variant. + LambdaFunctionScheduled, + /// The `LambdaFunctionStartFailed` wire variant. + LambdaFunctionStartFailed, + /// The `LambdaFunctionStarted` wire variant. + LambdaFunctionStarted, + /// The `LambdaFunctionSucceeded` wire variant. + LambdaFunctionSucceeded, + /// The `LambdaFunctionTimedOut` wire variant. + LambdaFunctionTimedOut, + /// The `MapIterationAborted` wire variant. + MapIterationAborted, + /// The `MapIterationFailed` wire variant. + MapIterationFailed, + /// The `MapIterationStarted` wire variant. + MapIterationStarted, + /// The `MapIterationSucceeded` wire variant. + MapIterationSucceeded, + /// The `MapRunAborted` wire variant. + MapRunAborted, + /// The `MapRunFailed` wire variant. + MapRunFailed, + /// The `MapRunRedriven` wire variant. + MapRunRedriven, + /// The `MapRunStarted` wire variant. + MapRunStarted, + /// The `MapRunSucceeded` wire variant. + MapRunSucceeded, + /// The `MapStateAborted` wire variant. + MapStateAborted, + /// The `MapStateEntered` wire variant. + MapStateEntered, + /// The `MapStateExited` wire variant. + MapStateExited, + /// The `MapStateFailed` wire variant. + MapStateFailed, + /// The `MapStateStarted` wire variant. + MapStateStarted, + /// The `MapStateSucceeded` wire variant. + MapStateSucceeded, + /// The `ParallelStateAborted` wire variant. + ParallelStateAborted, + /// The `ParallelStateEntered` wire variant. + ParallelStateEntered, + /// The `ParallelStateExited` wire variant. + ParallelStateExited, + /// The `ParallelStateFailed` wire variant. + ParallelStateFailed, + /// The `ParallelStateStarted` wire variant. + ParallelStateStarted, + /// The `ParallelStateSucceeded` wire variant. + ParallelStateSucceeded, + /// The `PassStateEntered` wire variant. + PassStateEntered, + /// The `PassStateExited` wire variant. + PassStateExited, + /// The `SucceedStateEntered` wire variant. + SucceedStateEntered, + /// The `SucceedStateExited` wire variant. + SucceedStateExited, + /// The `TaskFailed` wire variant. + TaskFailed, + /// The `TaskScheduled` wire variant. + TaskScheduled, + /// The `TaskStartFailed` wire variant. + TaskStartFailed, + /// The `TaskStarted` wire variant. + TaskStarted, + /// The `TaskStateAborted` wire variant. + TaskStateAborted, + /// The `TaskStateEntered` wire variant. + TaskStateEntered, + /// The `TaskStateExited` wire variant. + TaskStateExited, + /// The `TaskSubmitFailed` wire variant. + TaskSubmitFailed, + /// The `TaskSubmitted` wire variant. + TaskSubmitted, + /// The `TaskSucceeded` wire variant. + TaskSucceeded, + /// The `TaskTimedOut` wire variant. + TaskTimedOut, + /// The `WaitStateAborted` wire variant. + WaitStateAborted, + /// The `WaitStateEntered` wire variant. + WaitStateEntered, + /// The `WaitStateExited` wire variant. + WaitStateExited, +} + +impl HistoryEventType { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::ActivityFailed => "ActivityFailed", + Self::ActivityScheduleFailed => "ActivityScheduleFailed", + Self::ActivityScheduled => "ActivityScheduled", + Self::ActivityStarted => "ActivityStarted", + Self::ActivitySucceeded => "ActivitySucceeded", + Self::ActivityTimedOut => "ActivityTimedOut", + Self::ChoiceStateEntered => "ChoiceStateEntered", + Self::ChoiceStateExited => "ChoiceStateExited", + Self::EvaluationFailed => "EvaluationFailed", + Self::ExecutionAborted => "ExecutionAborted", + Self::ExecutionFailed => "ExecutionFailed", + Self::ExecutionRedriven => "ExecutionRedriven", + Self::ExecutionStarted => "ExecutionStarted", + Self::ExecutionSucceeded => "ExecutionSucceeded", + Self::ExecutionTimedOut => "ExecutionTimedOut", + Self::FailStateEntered => "FailStateEntered", + Self::LambdaFunctionFailed => "LambdaFunctionFailed", + Self::LambdaFunctionScheduleFailed => "LambdaFunctionScheduleFailed", + Self::LambdaFunctionScheduled => "LambdaFunctionScheduled", + Self::LambdaFunctionStartFailed => "LambdaFunctionStartFailed", + Self::LambdaFunctionStarted => "LambdaFunctionStarted", + Self::LambdaFunctionSucceeded => "LambdaFunctionSucceeded", + Self::LambdaFunctionTimedOut => "LambdaFunctionTimedOut", + Self::MapIterationAborted => "MapIterationAborted", + Self::MapIterationFailed => "MapIterationFailed", + Self::MapIterationStarted => "MapIterationStarted", + Self::MapIterationSucceeded => "MapIterationSucceeded", + Self::MapRunAborted => "MapRunAborted", + Self::MapRunFailed => "MapRunFailed", + Self::MapRunRedriven => "MapRunRedriven", + Self::MapRunStarted => "MapRunStarted", + Self::MapRunSucceeded => "MapRunSucceeded", + Self::MapStateAborted => "MapStateAborted", + Self::MapStateEntered => "MapStateEntered", + Self::MapStateExited => "MapStateExited", + Self::MapStateFailed => "MapStateFailed", + Self::MapStateStarted => "MapStateStarted", + Self::MapStateSucceeded => "MapStateSucceeded", + Self::ParallelStateAborted => "ParallelStateAborted", + Self::ParallelStateEntered => "ParallelStateEntered", + Self::ParallelStateExited => "ParallelStateExited", + Self::ParallelStateFailed => "ParallelStateFailed", + Self::ParallelStateStarted => "ParallelStateStarted", + Self::ParallelStateSucceeded => "ParallelStateSucceeded", + Self::PassStateEntered => "PassStateEntered", + Self::PassStateExited => "PassStateExited", + Self::SucceedStateEntered => "SucceedStateEntered", + Self::SucceedStateExited => "SucceedStateExited", + Self::TaskFailed => "TaskFailed", + Self::TaskScheduled => "TaskScheduled", + Self::TaskStartFailed => "TaskStartFailed", + Self::TaskStarted => "TaskStarted", + Self::TaskStateAborted => "TaskStateAborted", + Self::TaskStateEntered => "TaskStateEntered", + Self::TaskStateExited => "TaskStateExited", + Self::TaskSubmitFailed => "TaskSubmitFailed", + Self::TaskSubmitted => "TaskSubmitted", + Self::TaskSucceeded => "TaskSucceeded", + Self::TaskTimedOut => "TaskTimedOut", + Self::WaitStateAborted => "WaitStateAborted", + Self::WaitStateEntered => "WaitStateEntered", + Self::WaitStateExited => "WaitStateExited", + } + } +} + +impl std::fmt::Display for HistoryEventType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for HistoryEventType { + fn from(s: &str) -> Self { + match s { + "ActivityFailed" => Self::ActivityFailed, + "ActivityScheduleFailed" => Self::ActivityScheduleFailed, + "ActivityScheduled" => Self::ActivityScheduled, + "ActivityStarted" => Self::ActivityStarted, + "ActivitySucceeded" => Self::ActivitySucceeded, + "ActivityTimedOut" => Self::ActivityTimedOut, + "ChoiceStateEntered" => Self::ChoiceStateEntered, + "ChoiceStateExited" => Self::ChoiceStateExited, + "EvaluationFailed" => Self::EvaluationFailed, + "ExecutionAborted" => Self::ExecutionAborted, + "ExecutionFailed" => Self::ExecutionFailed, + "ExecutionRedriven" => Self::ExecutionRedriven, + "ExecutionStarted" => Self::ExecutionStarted, + "ExecutionSucceeded" => Self::ExecutionSucceeded, + "ExecutionTimedOut" => Self::ExecutionTimedOut, + "FailStateEntered" => Self::FailStateEntered, + "LambdaFunctionFailed" => Self::LambdaFunctionFailed, + "LambdaFunctionScheduleFailed" => Self::LambdaFunctionScheduleFailed, + "LambdaFunctionScheduled" => Self::LambdaFunctionScheduled, + "LambdaFunctionStartFailed" => Self::LambdaFunctionStartFailed, + "LambdaFunctionStarted" => Self::LambdaFunctionStarted, + "LambdaFunctionSucceeded" => Self::LambdaFunctionSucceeded, + "LambdaFunctionTimedOut" => Self::LambdaFunctionTimedOut, + "MapIterationAborted" => Self::MapIterationAborted, + "MapIterationFailed" => Self::MapIterationFailed, + "MapIterationStarted" => Self::MapIterationStarted, + "MapIterationSucceeded" => Self::MapIterationSucceeded, + "MapRunAborted" => Self::MapRunAborted, + "MapRunFailed" => Self::MapRunFailed, + "MapRunRedriven" => Self::MapRunRedriven, + "MapRunStarted" => Self::MapRunStarted, + "MapRunSucceeded" => Self::MapRunSucceeded, + "MapStateAborted" => Self::MapStateAborted, + "MapStateEntered" => Self::MapStateEntered, + "MapStateExited" => Self::MapStateExited, + "MapStateFailed" => Self::MapStateFailed, + "MapStateStarted" => Self::MapStateStarted, + "MapStateSucceeded" => Self::MapStateSucceeded, + "ParallelStateAborted" => Self::ParallelStateAborted, + "ParallelStateEntered" => Self::ParallelStateEntered, + "ParallelStateExited" => Self::ParallelStateExited, + "ParallelStateFailed" => Self::ParallelStateFailed, + "ParallelStateStarted" => Self::ParallelStateStarted, + "ParallelStateSucceeded" => Self::ParallelStateSucceeded, + "PassStateEntered" => Self::PassStateEntered, + "PassStateExited" => Self::PassStateExited, + "SucceedStateEntered" => Self::SucceedStateEntered, + "SucceedStateExited" => Self::SucceedStateExited, + "TaskFailed" => Self::TaskFailed, + "TaskScheduled" => Self::TaskScheduled, + "TaskStartFailed" => Self::TaskStartFailed, + "TaskStarted" => Self::TaskStarted, + "TaskStateAborted" => Self::TaskStateAborted, + "TaskStateEntered" => Self::TaskStateEntered, + "TaskStateExited" => Self::TaskStateExited, + "TaskSubmitFailed" => Self::TaskSubmitFailed, + "TaskSubmitted" => Self::TaskSubmitted, + "TaskSucceeded" => Self::TaskSucceeded, + "TaskTimedOut" => Self::TaskTimedOut, + "WaitStateAborted" => Self::WaitStateAborted, + "WaitStateEntered" => Self::WaitStateEntered, + "WaitStateExited" => Self::WaitStateExited, + _ => Self::default(), + } + } +} + +/// Step Functions `IncludedData` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum IncludedData { + /// The `ALL_DATA` wire variant. + #[default] + #[serde(rename = "ALL_DATA")] + AllData, + /// The `METADATA_ONLY` wire variant. + #[serde(rename = "METADATA_ONLY")] + MetadataOnly, +} + +impl IncludedData { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::AllData => "ALL_DATA", + Self::MetadataOnly => "METADATA_ONLY", + } + } +} + +impl std::fmt::Display for IncludedData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for IncludedData { + fn from(s: &str) -> Self { + match s { + "ALL_DATA" => Self::AllData, + "METADATA_ONLY" => Self::MetadataOnly, + _ => Self::default(), + } + } +} + +/// Step Functions `InspectionLevel` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum InspectionLevel { + /// The `DEBUG` wire variant. + #[default] + #[serde(rename = "DEBUG")] + Debug, + /// The `INFO` wire variant. + #[serde(rename = "INFO")] + Info, + /// The `TRACE` wire variant. + #[serde(rename = "TRACE")] + Trace, +} + +impl InspectionLevel { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Debug => "DEBUG", + Self::Info => "INFO", + Self::Trace => "TRACE", + } + } +} + +impl std::fmt::Display for InspectionLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for InspectionLevel { + fn from(s: &str) -> Self { + match s { + "DEBUG" => Self::Debug, + "INFO" => Self::Info, + "TRACE" => Self::Trace, + _ => Self::default(), + } + } +} + +/// Step Functions `LogLevel` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum LogLevel { + /// The `ALL` wire variant. + #[default] + #[serde(rename = "ALL")] + All, + /// The `ERROR` wire variant. + #[serde(rename = "ERROR")] + Error, + /// The `FATAL` wire variant. + #[serde(rename = "FATAL")] + Fatal, + /// The `OFF` wire variant. + #[serde(rename = "OFF")] + Off, +} + +impl LogLevel { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::All => "ALL", + Self::Error => "ERROR", + Self::Fatal => "FATAL", + Self::Off => "OFF", + } + } +} + +impl std::fmt::Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for LogLevel { + fn from(s: &str) -> Self { + match s { + "ALL" => Self::All, + "ERROR" => Self::Error, + "FATAL" => Self::Fatal, + "OFF" => Self::Off, + _ => Self::default(), + } + } +} + +/// Step Functions `MapRunStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum MapRunStatus { + /// The `ABORTED` wire variant. + #[default] + #[serde(rename = "ABORTED")] + Aborted, + /// The `FAILED` wire variant. + #[serde(rename = "FAILED")] + Failed, + /// The `RUNNING` wire variant. + #[serde(rename = "RUNNING")] + Running, + /// The `SUCCEEDED` wire variant. + #[serde(rename = "SUCCEEDED")] + Succeeded, +} + +impl MapRunStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Aborted => "ABORTED", + Self::Failed => "FAILED", + Self::Running => "RUNNING", + Self::Succeeded => "SUCCEEDED", + } + } +} + +impl std::fmt::Display for MapRunStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for MapRunStatus { + fn from(s: &str) -> Self { + match s { + "ABORTED" => Self::Aborted, + "FAILED" => Self::Failed, + "RUNNING" => Self::Running, + "SUCCEEDED" => Self::Succeeded, + _ => Self::default(), + } + } +} + +/// Step Functions `MockResponseValidationMode` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum MockResponseValidationMode { + /// The `NONE` wire variant. + #[default] + #[serde(rename = "NONE")] + None, + /// The `PRESENT` wire variant. + #[serde(rename = "PRESENT")] + Present, + /// The `STRICT` wire variant. + #[serde(rename = "STRICT")] + Strict, +} + +impl MockResponseValidationMode { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::None => "NONE", + Self::Present => "PRESENT", + Self::Strict => "STRICT", + } + } +} + +impl std::fmt::Display for MockResponseValidationMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for MockResponseValidationMode { + fn from(s: &str) -> Self { + match s { + "NONE" => Self::None, + "PRESENT" => Self::Present, + "STRICT" => Self::Strict, + _ => Self::default(), + } + } +} + +/// Step Functions `StateMachineStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum StateMachineStatus { + /// The `ACTIVE` wire variant. + #[default] + #[serde(rename = "ACTIVE")] + Active, + /// The `DELETING` wire variant. + #[serde(rename = "DELETING")] + Deleting, +} + +impl StateMachineStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Active => "ACTIVE", + Self::Deleting => "DELETING", + } + } +} + +impl std::fmt::Display for StateMachineStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for StateMachineStatus { + fn from(s: &str) -> Self { + match s { + "ACTIVE" => Self::Active, + "DELETING" => Self::Deleting, + _ => Self::default(), + } + } +} + +/// Step Functions `StateMachineType` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum StateMachineType { + /// The `EXPRESS` wire variant. + #[default] + #[serde(rename = "EXPRESS")] + Express, + /// The `STANDARD` wire variant. + #[serde(rename = "STANDARD")] + Standard, +} + +impl StateMachineType { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Express => "EXPRESS", + Self::Standard => "STANDARD", + } + } +} + +impl std::fmt::Display for StateMachineType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for StateMachineType { + fn from(s: &str) -> Self { + match s { + "EXPRESS" => Self::Express, + "STANDARD" => Self::Standard, + _ => Self::default(), + } + } +} + +/// Step Functions `SyncExecutionStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum SyncExecutionStatus { + /// The `FAILED` wire variant. + #[default] + #[serde(rename = "FAILED")] + Failed, + /// The `SUCCEEDED` wire variant. + #[serde(rename = "SUCCEEDED")] + Succeeded, + /// The `TIMED_OUT` wire variant. + #[serde(rename = "TIMED_OUT")] + TimedOut, +} + +impl SyncExecutionStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Failed => "FAILED", + Self::Succeeded => "SUCCEEDED", + Self::TimedOut => "TIMED_OUT", + } + } +} + +impl std::fmt::Display for SyncExecutionStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for SyncExecutionStatus { + fn from(s: &str) -> Self { + match s { + "FAILED" => Self::Failed, + "SUCCEEDED" => Self::Succeeded, + "TIMED_OUT" => Self::TimedOut, + _ => Self::default(), + } + } +} + +/// Step Functions `TestExecutionStatus` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum TestExecutionStatus { + /// The `CAUGHT_ERROR` wire variant. + #[default] + #[serde(rename = "CAUGHT_ERROR")] + CaughtError, + /// The `FAILED` wire variant. + #[serde(rename = "FAILED")] + Failed, + /// The `RETRIABLE` wire variant. + #[serde(rename = "RETRIABLE")] + Retriable, + /// The `SUCCEEDED` wire variant. + #[serde(rename = "SUCCEEDED")] + Succeeded, +} + +impl TestExecutionStatus { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::CaughtError => "CAUGHT_ERROR", + Self::Failed => "FAILED", + Self::Retriable => "RETRIABLE", + Self::Succeeded => "SUCCEEDED", + } + } +} + +impl std::fmt::Display for TestExecutionStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for TestExecutionStatus { + fn from(s: &str) -> Self { + match s { + "CAUGHT_ERROR" => Self::CaughtError, + "FAILED" => Self::Failed, + "RETRIABLE" => Self::Retriable, + "SUCCEEDED" => Self::Succeeded, + _ => Self::default(), + } + } +} + +/// Step Functions `ValidateStateMachineDefinitionResultCode` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum ValidateStateMachineDefinitionResultCode { + /// The `FAIL` wire variant. + #[default] + #[serde(rename = "FAIL")] + Fail, + /// The `OK` wire variant. + #[serde(rename = "OK")] + Ok, +} + +impl ValidateStateMachineDefinitionResultCode { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Fail => "FAIL", + Self::Ok => "OK", + } + } +} + +impl std::fmt::Display for ValidateStateMachineDefinitionResultCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for ValidateStateMachineDefinitionResultCode { + fn from(s: &str) -> Self { + match s { + "FAIL" => Self::Fail, + "OK" => Self::Ok, + _ => Self::default(), + } + } +} + +/// Step Functions `ValidateStateMachineDefinitionSeverity` enum. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum ValidateStateMachineDefinitionSeverity { + /// The `ERROR` wire variant. + #[default] + #[serde(rename = "ERROR")] + Error, + /// The `WARNING` wire variant. + #[serde(rename = "WARNING")] + Warning, +} + +impl ValidateStateMachineDefinitionSeverity { + /// Returns the string value of this enum variant. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Error => "ERROR", + Self::Warning => "WARNING", + } + } +} + +impl std::fmt::Display for ValidateStateMachineDefinitionSeverity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for ValidateStateMachineDefinitionSeverity { + fn from(s: &str) -> Self { + match s { + "ERROR" => Self::Error, + "WARNING" => Self::Warning, + _ => Self::default(), + } + } +} + +/// Step Functions `ActivityFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ActivityFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ActivityListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityListItem { + /// The `activityArn` wire member. + pub activity_arn: String, + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `name` wire member. + pub name: String, +} + +impl std::fmt::Debug for ActivityListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityListItem").finish_non_exhaustive() + } +} +/// Step Functions `ActivityScheduleFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityScheduleFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ActivityScheduleFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityScheduleFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ActivityScheduledEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityScheduledEventDetails { + /// The `heartbeatInSeconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub heartbeat_in_seconds: Option, + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `resource` wire member. + pub resource: String, + /// The `timeoutInSeconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_in_seconds: Option, +} + +impl std::fmt::Debug for ActivityScheduledEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityScheduledEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ActivityStartedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityStartedEventDetails { + /// The `workerName` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_name: Option, +} + +impl std::fmt::Debug for ActivityStartedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityStartedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ActivitySucceededEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivitySucceededEventDetails { + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, +} + +impl std::fmt::Debug for ActivitySucceededEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivitySucceededEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ActivityTimedOutEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ActivityTimedOutEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ActivityTimedOutEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActivityTimedOutEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `AssignedVariablesDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct AssignedVariablesDetails { + /// The `truncated` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub truncated: Option, +} + +impl std::fmt::Debug for AssignedVariablesDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AssignedVariablesDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `BillingDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct BillingDetails { + /// The `billedDurationInMilliseconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub billed_duration_in_milliseconds: Option, + /// The `billedMemoryUsedInMB` wire member. + #[serde(rename = "billedMemoryUsedInMB")] + #[serde(skip_serializing_if = "Option::is_none")] + pub billed_memory_used_in_mb: Option, +} + +impl std::fmt::Debug for BillingDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BillingDetails").finish_non_exhaustive() + } +} +/// Step Functions `CloudWatchEventsExecutionDataDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CloudWatchEventsExecutionDataDetails { + /// The `included` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub included: Option, +} + +impl std::fmt::Debug for CloudWatchEventsExecutionDataDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CloudWatchEventsExecutionDataDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `CloudWatchLogsLogGroup`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct CloudWatchLogsLogGroup { + /// The `logGroupArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_group_arn: Option, +} + +impl std::fmt::Debug for CloudWatchLogsLogGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CloudWatchLogsLogGroup") + .finish_non_exhaustive() + } +} +/// Step Functions `EncryptionConfiguration`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct EncryptionConfiguration { + /// The `kmsDataKeyReusePeriodSeconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub kms_data_key_reuse_period_seconds: Option, + /// The `kmsKeyId` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub kms_key_id: Option, + /// The `type` wire member. + pub r#type: EncryptionType, +} + +impl std::fmt::Debug for EncryptionConfiguration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EncryptionConfiguration") + .finish_non_exhaustive() + } +} +/// Step Functions `EvaluationFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct EvaluationFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `location` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, + /// The `state` wire member. + pub state: String, +} + +impl std::fmt::Debug for EvaluationFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EvaluationFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionAbortedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionAbortedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ExecutionAbortedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionAbortedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ExecutionFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct ExecutionListItem { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `itemCount` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub item_count: Option, + /// The `mapRunArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, + /// The `name` wire member. + pub name: String, + /// The `redriveCount` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_count: Option, + /// The `redriveDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_date: Option>, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, + /// The `stateMachineAliasArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_alias_arn: Option, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `stateMachineVersionArn` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_version_arn: Option, + /// The `status` wire member. + pub status: ExecutionStatus, + /// The `stopDate` wire member. + #[builder(default)] + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_date: Option>, +} + +impl std::fmt::Debug for ExecutionListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionListItem").finish_non_exhaustive() + } +} +/// Step Functions `ExecutionRedrivenEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionRedrivenEventDetails { + /// The `redriveCount` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_count: Option, +} + +impl std::fmt::Debug for ExecutionRedrivenEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionRedrivenEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionStartedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionStartedEventDetails { + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `roleArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub role_arn: Option, + /// The `stateMachineAliasArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_alias_arn: Option, + /// The `stateMachineVersionArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub state_machine_version_arn: Option, +} + +impl std::fmt::Debug for ExecutionStartedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionStartedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionSucceededEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionSucceededEventDetails { + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, +} + +impl std::fmt::Debug for ExecutionSucceededEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionSucceededEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `ExecutionTimedOutEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ExecutionTimedOutEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for ExecutionTimedOutEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExecutionTimedOutEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `HistoryEvent`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct HistoryEvent { + /// The `activityFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_failed_event_details: Option, + /// The `activityScheduleFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_schedule_failed_event_details: Option, + /// The `activityScheduledEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_scheduled_event_details: Option, + /// The `activityStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_started_event_details: Option, + /// The `activitySucceededEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_succeeded_event_details: Option, + /// The `activityTimedOutEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub activity_timed_out_event_details: Option, + /// The `evaluationFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub evaluation_failed_event_details: Option, + /// The `executionAbortedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_aborted_event_details: Option, + /// The `executionFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_failed_event_details: Option, + /// The `executionRedrivenEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_redriven_event_details: Option, + /// The `executionStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_started_event_details: Option, + /// The `executionSucceededEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_succeeded_event_details: Option, + /// The `executionTimedOutEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_timed_out_event_details: Option, + /// The `id` wire member. + pub id: i64, + /// The `lambdaFunctionFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_failed_event_details: Option, + /// The `lambdaFunctionScheduleFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_schedule_failed_event_details: + Option, + /// The `lambdaFunctionScheduledEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_scheduled_event_details: Option, + /// The `lambdaFunctionStartFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_start_failed_event_details: Option, + /// The `lambdaFunctionSucceededEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_succeeded_event_details: Option, + /// The `lambdaFunctionTimedOutEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub lambda_function_timed_out_event_details: Option, + /// The `mapIterationAbortedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_iteration_aborted_event_details: Option, + /// The `mapIterationFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_iteration_failed_event_details: Option, + /// The `mapIterationStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_iteration_started_event_details: Option, + /// The `mapIterationSucceededEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_iteration_succeeded_event_details: Option, + /// The `mapRunFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_failed_event_details: Option, + /// The `mapRunRedrivenEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_redriven_event_details: Option, + /// The `mapRunStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_started_event_details: Option, + /// The `mapStateStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub map_state_started_event_details: Option, + /// The `previousEventId` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_event_id: Option, + /// The `stateEnteredEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_entered_event_details: Option, + /// The `stateExitedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub state_exited_event_details: Option, + /// The `taskFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_failed_event_details: Option, + /// The `taskScheduledEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_scheduled_event_details: Option, + /// The `taskStartFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_start_failed_event_details: Option, + /// The `taskStartedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_started_event_details: Option, + /// The `taskSubmitFailedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_submit_failed_event_details: Option, + /// The `taskSubmittedEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_submitted_event_details: Option, + /// The `taskSucceededEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_succeeded_event_details: Option, + /// The `taskTimedOutEventDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_timed_out_event_details: Option, + /// The `timestamp` wire member. + #[serde(with = "crate::epoch_seconds")] + pub timestamp: chrono::DateTime, + /// The `type` wire member. + pub r#type: HistoryEventType, +} + +impl std::fmt::Debug for HistoryEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HistoryEvent").finish_non_exhaustive() + } +} +/// Step Functions `HistoryEventExecutionDataDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct HistoryEventExecutionDataDetails { + /// The `truncated` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub truncated: Option, +} + +impl std::fmt::Debug for HistoryEventExecutionDataDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HistoryEventExecutionDataDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `InspectionData`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct InspectionData { + /// The `afterArguments` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_arguments: Option, + /// The `afterInputPath` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_input_path: Option, + /// The `afterItemBatcher` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_item_batcher: Option, + /// The `afterItemSelector` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_item_selector: Option, + /// The `afterItemsPath` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_items_path: Option, + /// The `afterItemsPointer` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_items_pointer: Option, + /// The `afterParameters` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_parameters: Option, + /// The `afterResultPath` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_result_path: Option, + /// The `afterResultSelector` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub after_result_selector: Option, + /// The `errorDetails` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, + /// The `input` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `maxConcurrency` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub max_concurrency: Option, + /// The `request` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub request: Option, + /// The `response` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub response: Option, + /// The `result` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, + /// The `toleratedFailureCount` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tolerated_failure_count: Option, + /// The `toleratedFailurePercentage` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub tolerated_failure_percentage: Option, + /// The `variables` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub variables: Option, +} + +impl std::fmt::Debug for InspectionData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InspectionData").finish_non_exhaustive() + } +} +/// Step Functions `InspectionDataRequest`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct InspectionDataRequest { + /// The `body` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option, + /// The `headers` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option, + /// The `method` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub method: Option, + /// The `protocol` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub protocol: Option, + /// The `url` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +impl std::fmt::Debug for InspectionDataRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InspectionDataRequest") + .finish_non_exhaustive() + } +} +/// Step Functions `InspectionDataResponse`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct InspectionDataResponse { + /// The `body` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option, + /// The `headers` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option, + /// The `protocol` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub protocol: Option, + /// The `statusCode` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub status_code: Option, + /// The `statusMessage` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub status_message: Option, +} + +impl std::fmt::Debug for InspectionDataResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InspectionDataResponse") + .finish_non_exhaustive() + } +} +/// Step Functions `InspectionErrorDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct InspectionErrorDetails { + /// The `catchIndex` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub catch_index: Option, + /// The `retryBackoffIntervalSeconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_backoff_interval_seconds: Option, + /// The `retryIndex` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_index: Option, +} + +impl std::fmt::Debug for InspectionErrorDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InspectionErrorDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for LambdaFunctionFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionScheduleFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionScheduleFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for LambdaFunctionScheduleFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionScheduleFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionScheduledEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionScheduledEventDetails { + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `resource` wire member. + pub resource: String, + /// The `taskCredentials` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub task_credentials: Option, + /// The `timeoutInSeconds` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_in_seconds: Option, +} + +impl std::fmt::Debug for LambdaFunctionScheduledEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionScheduledEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionStartFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionStartFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for LambdaFunctionStartFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionStartFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionSucceededEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionSucceededEventDetails { + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, +} + +impl std::fmt::Debug for LambdaFunctionSucceededEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionSucceededEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LambdaFunctionTimedOutEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LambdaFunctionTimedOutEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for LambdaFunctionTimedOutEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LambdaFunctionTimedOutEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `LogDestination`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LogDestination { + /// The `cloudWatchLogsLogGroup` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cloud_watch_logs_log_group: Option, +} + +impl std::fmt::Debug for LogDestination { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LogDestination").finish_non_exhaustive() + } +} +/// Step Functions `LoggingConfiguration`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct LoggingConfiguration { + /// The `destinations` wire member. + #[serde(default)] + pub destinations: Vec, + /// The `includeExecutionData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub include_execution_data: Option, + /// The `level` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub level: Option, +} + +impl std::fmt::Debug for LoggingConfiguration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LoggingConfiguration") + .finish_non_exhaustive() + } +} +/// Step Functions `MapIterationEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapIterationEventDetails { + /// The `index` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub index: Option, + /// The `name` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, +} + +impl std::fmt::Debug for MapIterationEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapIterationEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `MapRunExecutionCounts`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct MapRunExecutionCounts { + /// The `aborted` wire member. + pub aborted: i64, + /// The `failed` wire member. + pub failed: i64, + /// The `failuresNotRedrivable` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub failures_not_redrivable: Option, + /// The `pending` wire member. + pub pending: i64, + /// The `pendingRedrive` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub pending_redrive: Option, + /// The `resultsWritten` wire member. + pub results_written: i64, + /// The `running` wire member. + pub running: i64, + /// The `succeeded` wire member. + pub succeeded: i64, + /// The `timedOut` wire member. + pub timed_out: i64, + /// The `total` wire member. + pub total: i64, +} + +impl std::fmt::Debug for MapRunExecutionCounts { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunExecutionCounts") + .finish_non_exhaustive() + } +} +/// Step Functions `MapRunFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapRunFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for MapRunFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `MapRunItemCounts`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct MapRunItemCounts { + /// The `aborted` wire member. + pub aborted: i64, + /// The `failed` wire member. + pub failed: i64, + /// The `failuresNotRedrivable` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub failures_not_redrivable: Option, + /// The `pending` wire member. + pub pending: i64, + /// The `pendingRedrive` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub pending_redrive: Option, + /// The `resultsWritten` wire member. + pub results_written: i64, + /// The `running` wire member. + pub running: i64, + /// The `succeeded` wire member. + pub succeeded: i64, + /// The `timedOut` wire member. + pub timed_out: i64, + /// The `total` wire member. + pub total: i64, +} + +impl std::fmt::Debug for MapRunItemCounts { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunItemCounts").finish_non_exhaustive() + } +} +/// Step Functions `MapRunListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapRunListItem { + /// The `executionArn` wire member. + pub execution_arn: String, + /// The `mapRunArn` wire member. + pub map_run_arn: String, + /// The `startDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub start_date: chrono::DateTime, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `stopDate` wire member. + #[serde(default, with = "crate::epoch_seconds::option")] + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_date: Option>, +} + +impl std::fmt::Debug for MapRunListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunListItem").finish_non_exhaustive() + } +} +/// Step Functions `MapRunRedrivenEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapRunRedrivenEventDetails { + /// The `mapRunArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, + /// The `redriveCount` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub redrive_count: Option, +} + +impl std::fmt::Debug for MapRunRedrivenEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunRedrivenEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `MapRunStartedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapRunStartedEventDetails { + /// The `mapRunArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub map_run_arn: Option, +} + +impl std::fmt::Debug for MapRunStartedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapRunStartedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `MapStateStartedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MapStateStartedEventDetails { + /// The `length` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub length: Option, +} + +impl std::fmt::Debug for MapStateStartedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MapStateStartedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `MockErrorOutput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MockErrorOutput { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl std::fmt::Debug for MockErrorOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MockErrorOutput").finish_non_exhaustive() + } +} +/// Step Functions `MockInput`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct MockInput { + /// The `errorOutput` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_output: Option, + /// The `fieldValidationMode` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub field_validation_mode: Option, + /// The `result` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, +} + +impl std::fmt::Debug for MockInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MockInput").finish_non_exhaustive() + } +} +/// Step Functions `RoutingConfigurationListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct RoutingConfigurationListItem { + /// The `stateMachineVersionArn` wire member. + pub state_machine_version_arn: String, + /// The `weight` wire member. + pub weight: i32, +} + +impl std::fmt::Debug for RoutingConfigurationListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RoutingConfigurationListItem") + .finish_non_exhaustive() + } +} +/// Step Functions `StateEnteredEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StateEnteredEventDetails { + /// The `input` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// The `inputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_details: Option, + /// The `name` wire member. + pub name: String, +} + +impl std::fmt::Debug for StateEnteredEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StateEnteredEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `StateExitedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StateExitedEventDetails { + /// The `assignedVariables` wire member. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub assigned_variables: HashMap, + /// The `assignedVariablesDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub assigned_variables_details: Option, + /// The `name` wire member. + pub name: String, + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, +} + +impl std::fmt::Debug for StateExitedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StateExitedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `StateMachineAliasListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StateMachineAliasListItem { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `stateMachineAliasArn` wire member. + pub state_machine_alias_arn: String, +} + +impl std::fmt::Debug for StateMachineAliasListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StateMachineAliasListItem") + .finish_non_exhaustive() + } +} +/// Step Functions `StateMachineListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StateMachineListItem { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `name` wire member. + pub name: String, + /// The `stateMachineArn` wire member. + pub state_machine_arn: String, + /// The `type` wire member. + pub r#type: StateMachineType, +} + +impl std::fmt::Debug for StateMachineListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StateMachineListItem") + .finish_non_exhaustive() + } +} +/// Step Functions `StateMachineVersionListItem`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct StateMachineVersionListItem { + /// The `creationDate` wire member. + #[serde(with = "crate::epoch_seconds")] + pub creation_date: chrono::DateTime, + /// The `stateMachineVersionArn` wire member. + pub state_machine_version_arn: String, +} + +impl std::fmt::Debug for StateMachineVersionListItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StateMachineVersionListItem") + .finish_non_exhaustive() + } +} +/// Step Functions `Tag`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct Tag { + /// The `key` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub key: Option, + /// The `value` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, +} + +impl std::fmt::Debug for Tag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Tag").finish_non_exhaustive() + } +} +/// Step Functions `TaskCredentials`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskCredentials { + /// The `roleArn` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub role_arn: Option, +} + +impl std::fmt::Debug for TaskCredentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskCredentials").finish_non_exhaustive() + } +} +/// Step Functions `TaskFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskScheduledEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +#[derive(typed_builder::TypedBuilder)] +pub struct TaskScheduledEventDetails { + /// The `heartbeatInSeconds` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub heartbeat_in_seconds: Option, + /// The `parameters` wire member. + pub parameters: String, + /// The `region` wire member. + pub region: String, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, + /// The `taskCredentials` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub task_credentials: Option, + /// The `timeoutInSeconds` wire member. + #[builder(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_in_seconds: Option, +} + +impl std::fmt::Debug for TaskScheduledEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskScheduledEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskStartFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskStartFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskStartFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskStartFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskStartedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskStartedEventDetails { + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskStartedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskStartedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskSubmitFailedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskSubmitFailedEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskSubmitFailedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskSubmitFailedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskSubmittedEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskSubmittedEventDetails { + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskSubmittedEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskSubmittedEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskSucceededEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskSucceededEventDetails { + /// The `output` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The `outputDetails` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_details: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskSucceededEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskSucceededEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TaskTimedOutEventDetails`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TaskTimedOutEventDetails { + /// The `cause` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// The `error` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The `resource` wire member. + pub resource: String, + /// The `resourceType` wire member. + pub resource_type: String, +} + +impl std::fmt::Debug for TaskTimedOutEventDetails { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TaskTimedOutEventDetails") + .finish_non_exhaustive() + } +} +/// Step Functions `TestStateConfiguration`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TestStateConfiguration { + /// The `errorCausedByState` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_caused_by_state: Option, + /// The `mapItemReaderData` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub map_item_reader_data: Option, + /// The `mapIterationFailureCount` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub map_iteration_failure_count: Option, + /// The `retrierRetryCount` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub retrier_retry_count: Option, +} + +impl std::fmt::Debug for TestStateConfiguration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestStateConfiguration") + .finish_non_exhaustive() + } +} +/// Step Functions `TracingConfiguration`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct TracingConfiguration { + /// The `enabled` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, +} + +impl std::fmt::Debug for TracingConfiguration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TracingConfiguration") + .finish_non_exhaustive() + } +} +/// Step Functions `ValidateStateMachineDefinitionDiagnostic`. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct ValidateStateMachineDefinitionDiagnostic { + /// The `code` wire member. + pub code: String, + /// The `location` wire member. + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, + /// The `message` wire member. + pub message: String, + /// The `severity` wire member. + pub severity: ValidateStateMachineDefinitionSeverity, +} + +impl std::fmt::Debug for ValidateStateMachineDefinitionDiagnostic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValidateStateMachineDefinitionDiagnostic") + .finish_non_exhaustive() + } +} diff --git a/crates/rustack-stepfunctions-model/tests/timestamps.rs b/crates/rustack-stepfunctions-model/tests/timestamps.rs new file mode 100644 index 00000000..247d3638 --- /dev/null +++ b/crates/rustack-stepfunctions-model/tests/timestamps.rs @@ -0,0 +1,197 @@ +//! Required/optional numeric wire timestamps from the actual generated model. + +use chrono::{DateTime, Utc}; +use rstest::rstest; +use rustack_stepfunctions_model::{ + output::{DescribeExecutionOutput, StartExecutionOutput}, + types::{ExecutionStatus, HistoryEvent, HistoryEventType}, +}; +use serde_json::json; + +#[rstest] +#[case("0", 0)] +#[case("1", 1000)] +#[case("1700000000.123", 1_700_000_000_123)] +#[case("-0.25",-250)] +#[case("-1.999",-1999)] +#[case("-1.999999",-1999)] +#[case("0.000999", 0)] +#[case("-0.000999", 0)] +#[case("1.23e2", 123_000)] +#[case("123e-3", 123)] +#[case("0.00123e3", 1230)] +#[case("1e-100", 0)] +#[case("-0e100", 0)] +#[case("0e2147483648", 0)] +#[case("-0e-2147483649", 0)] +#[case("1e-2147483649", 0)] +fn test_should_decode_and_roundtrip_generated_required_timestamp( + #[case] number: &str, + #[case] millis: i64, +) { + let wire = format!(r#"{{"executionArn":"execution","startDate":{number}}}"#); + let decoded: StartExecutionOutput = serde_json::from_str(&wire).unwrap(); + assert_eq!(decoded.start_date.timestamp_millis(), millis); + let serialized = serde_json::to_string(&decoded).unwrap(); + assert!(!serialized.contains('Z')); + let roundtrip: StartExecutionOutput = serde_json::from_str(&serialized).unwrap(); + assert_eq!(roundtrip.start_date, decoded.start_date); +} +#[rstest] +#[case("null")] +#[case("true")] +#[case("[]")] +#[case("{}")] +#[case("\"1700000000\"")] +#[case("\"2026-01-01T00:00:00Z\"")] +#[case("1e100")] +#[case("9223372036854775807")] +#[case("-9223372036854775808")] +#[case("NaN")] +fn test_should_reject_invalid_required_timestamp(#[case] value: &str) { + let wire = format!(r#"{{"executionArn":"execution","startDate":{value}}}"#); + assert!(serde_json::from_str::(&wire).is_err()); +} +#[test] +fn test_should_omit_optional_timestamp_and_accept_null_or_exponent_values() { + let output = DescribeExecutionOutput::builder() + .execution_arn("execution".into()) + .state_machine_arn("machine".into()) + .status(ExecutionStatus::Running) + .start_date(DateTime::from_timestamp_millis(-250).unwrap()) + .build(); + let serialized = serde_json::to_string(&output).unwrap(); + assert!(!serialized.contains("stopDate")); + let mut value = serde_json::to_value(&output).unwrap(); + value["stopDate"] = json!(null); + let decoded: DescribeExecutionOutput = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(decoded.stop_date, None); + value["stopDate"] = json!(-0.75); + let decoded: DescribeExecutionOutput = serde_json::from_value(value).unwrap(); + assert_eq!(decoded.stop_date.unwrap().timestamp_millis(), -750); + let wire = serialized.trim_end_matches('}').to_owned() + ",\"stopDate\":-2.5e-1}"; + assert_eq!( + serde_json::from_str::(&wire) + .unwrap() + .stop_date + .unwrap() + .timestamp_millis(), + -250 + ); +} +#[test] +fn test_should_serialize_exact_decimal_and_truncate_submillisecond_precision() { + let output = HistoryEvent::builder() + .id(1) + .timestamp(DateTime::from_timestamp(-2, 765_432_100).unwrap()) + .r#type(HistoryEventType::ExecutionStarted) + .build(); + let serialized = serde_json::to_string(&output).unwrap(); + assert!(serialized.contains(r#""timestamp":-1.234"#)); + let decoded: HistoryEvent = serde_json::from_str(&serialized).unwrap(); + assert_eq!(decoded.timestamp.timestamp_millis(), -1234); +} +#[test] +fn test_should_reject_leap_seconds_missing_required_fields_and_unknown_fields() { + let mut output = StartExecutionOutput::default(); + output.start_date = DateTime::from_timestamp(59, 1_000_000_000).unwrap(); + assert!(serde_json::to_string(&output).is_err()); + assert!( + serde_json::from_str::(r#"{"executionArn":"execution"}"#).is_err() + ); + assert!( + serde_json::from_str::( + r#"{"executionArn":"execution","startDate":0,"extra":0}"# + ) + .is_err() + ); +} +#[test] +fn test_should_preserve_chrono_range_millisecond_boundaries_without_float_rounding() { + for value in [DateTime::::MIN_UTC, DateTime::::MAX_UTC] { + let mut output = StartExecutionOutput::default(); + output.start_date = value; + let serialized = serde_json::to_string(&output).unwrap(); + let decoded: StartExecutionOutput = serde_json::from_str(&serialized).unwrap(); + let total = i128::from(value.timestamp()) * 1_000_000_000 + + i128::from(value.timestamp_subsec_nanos()); + assert_eq!( + i128::from(decoded.start_date.timestamp_millis()), + total / 1_000_000 + ); + } +} +#[test] +fn test_should_reject_oversized_timestamp_lexeme_with_redacted_error() { + let wire = format!( + r#"{{"executionArn":"execution","startDate":0.{}}}"#, + "1".repeat(129) + ); + let error = serde_json::from_str::(&wire) + .unwrap_err() + .to_string(); + assert!(error.contains("invalid epoch-second timestamp")); + assert!(!error.contains(&"1".repeat(20))); +} + +#[test] +fn test_should_accept_optional_oversized_exponents_only_when_value_normalizes_to_zero() { + for number in ["0e2147483648", "-0e-2147483649", "1e-2147483649"] { + let wire = format!( + r#"{{"executionArn":"execution","stateMachineArn":"machine","startDate":0,"status":"RUNNING","stopDate":{number}}}"# + ); + let decoded: DescribeExecutionOutput = serde_json::from_str(&wire).unwrap(); + assert_eq!(decoded.stop_date.unwrap().timestamp_millis(), 0); + } + let wire = r#"{"executionArn":"execution","startDate":1e2147483648}"#; + assert!(serde_json::from_str::(wire).is_err()); +} +#[test] +fn test_should_redact_generated_inputs_nested_history_and_error_diagnostics() { + use std::error::Error; + + use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode, input::SendTaskSuccessInput, + types::ExecutionStartedEventDetails, + }; + let token = "secret-task-token"; + let input: SendTaskSuccessInput = + serde_json::from_value(json!({"taskToken": token, "output": "secret-output"})).unwrap(); + let debug = format!("{input:?}"); + assert!(!debug.contains(token) && !debug.contains("secret-output")); + let nested: ExecutionStartedEventDetails = + serde_json::from_value(json!({"input":"secret-input"})).unwrap(); + let event = HistoryEvent::builder() + .id(1) + .timestamp(DateTime::from_timestamp_millis(0).unwrap()) + .r#type(HistoryEventType::ExecutionStarted) + .execution_started_event_details(Some(nested)) + .build(); + assert!(!format!("{event:?}").contains("secret-input")); + let mut error = StepFunctionsError::with_message(StepFunctionsErrorCode::InvalidToken, token); + error.source = Some(Box::new(std::io::Error::other("secret-source"))); + assert_eq!(format!("{error:?}"), "StepFunctionsError()"); + assert_eq!(error.to_string(), "StepFunctionsError()"); + assert!(error.source().is_some()); +} +#[test] +fn test_should_require_present_collections_and_keep_supported_operation_names_exact() { + use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode, StepFunctionsOperation, input::TagResourceInput, + }; + assert!(serde_json::from_str::(r#"{"resourceArn":"resource"}"#).is_err()); + assert!( + serde_json::from_str::(r#"{"resourceArn":"resource","tags":[]}"#).is_ok() + ); + assert!(StepFunctionsOperation::from_name("StartExecution").is_some()); + assert_eq!(StepFunctionsOperation::from_name("startexecution"), None); + assert_eq!( + rustack_stepfunctions_model::TARGET_PREFIX, + "AWSStepFunctions" + ); + let error = StepFunctionsError::new(StepFunctionsErrorCode::UnsupportedOperation); + assert_eq!( + error.message, + "Operation is outside the supported Step Functions profile" + ); +} diff --git a/specs/index.md b/specs/index.md index 7618dc86..e5e29225 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The ASL engine and P2–P5 remain pending. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. HTTP routing, registry, ASL engine and the remaining P2–P5 requirements remain pending. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-10-api-design.md b/specs/rustack-workflows-10-api-design.md index 524878df..aff64460 100644 --- a/specs/rustack-workflows-10-api-design.md +++ b/specs/rustack-workflows-10-api-design.md @@ -18,6 +18,16 @@ Add `rustack-stepfunctions-model`, `rustack-stepfunctions-http` and `rustack-ste Future automation uses Makefile targets. The current generator needs protocol-aware numeric epoch-second timestamp serde for awsJson1_0; ordinary chrono serde produces strings. Before service generation, prove required/optional timestamps, fractional seconds and negative epochs in generated round trips. Keep protocol-default behavior scoped correctly so restJson/query timestamp bindings do not change accidentally. Resolve any union support required by the selected operations in the generator; do not hand-edit generated source. +### 1.1 Executable timestamp/model generation contract + +Generate the pinned37-operation model into rustack-stepfunctions-model without advertising those operations as implemented. Add opt-in strict_model output configuration for this service: forbid unsafe, emit non-exhaustive public structs, deny unknown struct fields, derive typed-builder for structs with more than five fields (defaults only on nonrequired members), redact every DTO Debug field, keep error Debug/Display static while preserving typed source(), document fields/variants, and emit no blanket lint suppressions. Existing services retain their current generation policy. + +For awsJson1_0/1_1 timestamp members, resolve member timestampFormat before target-shape format before protocol default. Default/epoch-seconds uses a generated epoch_seconds module; explicit date-time retains chrono serde; unsupported explicit formats fail generation. Other protocols retain their existing binding behavior. Apply required and optional serde adapters (optional includes default for omission); do not change plain strings which resemble dates. The pinned SFN model has no timestamp-valued list/map members; unsupported timestamp collections must fail generation rather than emit chrono string defaults. + +The JSON adapter uses serde_json raw_value to preserve decimal numeric lexemes without binary-float rounding or a workspace-wide arbitrary_precision feature. Accept JSON numbers including exponent notation, at most128 lexical bytes; reject strings, required null, booleans, collections, invalid numeric grammar and chrono-range overflow with static diagnostics. Normalize to millisecond precision by truncating toward zero, including negative fractional epochs. Compute checked decimal scaling with bounded work; never cast a negative fraction to unsigned nanoseconds. Serialize finite chrono timestamps as exact numeric decimal seconds at the same precision, reject leap-second representations, omit absent optional members and accept optional null. This JSON-specific wire adapter is not a generic CBOR timestamp contract. HTTP still enforces its independent request envelope limit before model parsing. + +`make codegen-stepfunctions` generates all source including the codec from repository templates and formats it. `make test-stepfunctions-model` checks actual generated required/optional fields, positive/negative/fractional/exponent/range/omitted/null cases and regeneration consistency. Generator tests cover format precedence, protocol isolation, serde-disabled combinations and strict output without lint ignores. The developer CLI accepts bounded UTF-8 arguments, ordinary config/model files up to1/16MiB, and a60s generation deadline using Tokio1.54.1 with explicit rt/macros/fs/time/io-util features. Validate Smithy2.0, honor configured overlay preservation/module exports and reject unsupported output.dir/custom error formats rather than silently ignoring them. Check the standalone tool graph with make audit-codegen/deny-codegen. Primary contracts: [Smithy JSON1.0](https://smithy.io/2.0/aws/protocols/aws-json-1_0-protocol.html#shape-serialization), [timestamp precision/precedence](https://smithy.io/2.0/spec/protocol-traits.html#timestampformat-trait), [RawValue](https://docs.rs/serde_json/latest/serde_json/value/struct.RawValue.html). + ```text ┌─ AWS SDK clients ──────────────────────────────────────────────────┐ │ POST / + AWSStepFunctions.* JSON strings(input/definition) │ diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 248d7dcf..189f5fd9 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -162,3 +162,9 @@ Do not introduce a benchmark crate early. First measure these integration worklo The §4.8/D38 container journal must be exercised through the real EnvironmentManager create/cleanup path: a published intent exists when the first create arrives; failed durable erase retains the allocation until repaired. Transaction abandonment retains its lease/serial permit until its blocking worker completes. Reject overlong budgets before waiting for that permit. Verify maximum64 records/7,353 bytes, corruption/checksum/owner/UUID/duplicate identity, unsafe staging/published inodes, torn safe staging and authoritative-state preservation. Scripted restart cases require unknown404 plus blocked relay, later positive commit and exact function-before-relay cleanup; known-ID404; separately mutated name/ID/image and every authoritative label; current-run nonmutation; and an aborted recovery waiter leaving a durable known identity for the next pass. `make test-lambda-recovery` runs these independently of Docker. The explicit `test-lambda-recovery-docker` target requires socket/architecture/image, leaves two real journaled containers across directory reacquisition, then verifies both durable records are settled. Actual recovery tests preserve evidence and close the wire on every post-intent failure. Requalify actual pool/provider/HTTP/wire suites after integration. This is container historical-recovery evidence, not image-alias reconciliation, production startup or complete platform acceptance. + +## Step Functions generated protocol/model qualification + +`make workflows-model-check` verifies the pinned312-shape/37-operation source. `make test-stepfunctions-model` compares formatted regeneration byte-for-byte, exercises five generator policy cases and34 actual model cases plus a runnable macro doctest, then applies ordinary tool and strict production-model Clippy. Cover required/optional/omitted/null, negative and fractional epochs, scientific notation including huge zero/underflow exponents, positive overflow, chrono boundaries, submillisecond truncation, hostile lexical size, required collections, unknown fields and diagnostic redaction of task tokens/nested data/error sources. Verify format precedence and unchanged other-protocol timestamp bindings, both serde-disabled JSON protocols and strict no-serde Debug ownership. No generated lint suppression or ignored doctest may satisfy a check. + +Run `make audit-codegen deny-codegen` for the excluded developer-tool graph as well as the workspace gates. Model presence and known operation names do not establish endpoint, execution, authorization, ASL or Lambda integration acceptance. The user-prioritized API/compiler/engine work proceeds before final Docker-image qualification, whose original acceptance requirements remain required. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 49fd36c8..1af47b66 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -4,7 +4,7 @@ Status: P0 complete; P1 in progress; complete M0–M4 implementation objective r ## 1. Readiness and dependencies -Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Docker Executor/provider/HTTP composition is implemented and qualified through controlled-readiness real SDK fixtures. Startup network/full-ID qualification and actual pinned Rust kernel resource proofs are implemented. Missing: production runtime startup construction; historical-effect reconciliation, allowed pulls, endpoint reachability and final startup/configuration/capabilities; Step Functions protocol/model/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. +Available: existing gateway/YAML/snapshot infrastructure and independent Native/Squib backends; Lambda control plane, immutable revision/artifact capture and supervised admission/cancellation; shared RuntimeSession/HTTP/relay/client, bounded DockerWire, platform-aware image resolution/leases and provider asynchronous deployment/status/CAS. Persistent directory ownership, allocation plans, retained environment lifecycle, bounded FIFO/warm/LRU dispatch, delivery-based handler timing and invocation-scoped observed log windows and resumable per-call pool cleanup settlement are implemented. Docker Executor/provider/HTTP composition is implemented and qualified through controlled-readiness real SDK fixtures. Startup network/full-ID qualification and actual pinned Rust kernel resource proofs are implemented. Missing: production runtime startup construction; historical-effect reconciliation, allowed pulls, endpoint reachability and final startup/configuration/capabilities; Step Functions HTTP/core, ASL engine, integrations, workflow snapshots and complete acceptance fixtures. The execution ledger distinguishes implemented foundations from full phase exits. The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) validates documentation, not a working relay. Existing [Cargo Lambda evidence](../docs/research/spike-cargo-lambda-runtime-execution.md) establishes the bootstrap protocol but cannot close Docker/platform risks. P0 must precede production implementation. P0 subsequently initialized the pinned vendor tree and inspected the specifically cited AWS-validated nested test/snapshot pairs; no complete upstream interpreter review is claimed. @@ -35,6 +35,8 @@ The [survey](../docs/research/survey-stepfunctions-lambda-container-runtime.md) ## 2. Effort model +Execution priority update (2026-10-26, explicit user direction): defer further Docker-image work and advance the Step Functions API/compiler/engine and subsequent workflow features first. P2 protocol, registry and pure semantics do not require Docker readiness. Qualify LambdaTaskPort against existing real execution backends while preserving the final real-image acceptance requirements. Do not mark M0/M1–M4 complete until their original image/platform gates are backfilled. The in-progress image-pin journal worktree is retained as a deferred checkpoint; no new Docker experimentation is required before proceeding with Step Functions. + | Phase | Outcome | Engineering days | |---|---|---:| | P0 | Resolve load-bearing risks |4–6| @@ -155,6 +157,9 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3 Docker Executor/provider | Implemented/reviewed under trusted runtime injection; production startup remains | §4.6/D36 consume one PoolInvocation with independent cleanup settlement, immutable plan/metadata validation, raw outcomes, resolved version, requested alias ARN and one base64 tail encoding. W87–W91 are fixed/rechecked.15 Executor +16 handler cases pass, including held reserved concurrency, expired cleanup/failed quiescence and resumed physical retirement. Workspace2,313 tests/409 separately gated; core611 units/141 docs; strict boundary897 units+208 docs (1,105 total)/15 Docker-gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU daemon passes actual boto3→HTTP→provider→Docker deploy/publish/alias/Invoke/error/timeout/replacement plus seven existing owner/pool cases (eight per daemon). Controlled fixture readiness is not startup/probe/native-x86 evidence; post-run owned containers/internal aliases are absent. | | 1.3/1.4 startup probes | Implemented/reviewed; no readiness grant | §4.7/D37 validate exact user bridge/none selections, bind full IDs, and execute the pinned Rust probe under retained EnvironmentManager ownership. Private cgroup namespaces and process root-membership checks cover v1/v2 parser semantics; exact kernel limits/flags, scoped proof matching, caller abandonment, independent cleanup and final deadline/live-input checks are enforced. W92–W94 closed;57 focused cases pass. Workspace2,370/411 gated, boundary954 units+217 docs (1,171 total)/17 gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU cgroup-v2 daemon passes17 distinct actual probe/SDK/pool/wire/runtime cases; scoped resources and the two dedicated probe bridges are removed. Actual v1/native-x86, historical effects, pulls, endpoint/config/capability coordinator acceptance remain open. | | 1.3/1.4 historical container effects | Implemented/reviewed; other mutation classes remain | §4.8/D38 add bounded durable per-container intents before creation and known IDs before deletion. The restart pass preserves unknown absence, blocks unresolved relays, validates exact identities and resumes after cancellation. W95–W96 are fixed and independently rechecked.24 new scripted/unit cases pass, including create-before-effect and durable-erase reservation barriers. Workspace2,394/412 gated; boundary978 units+218 docs (1,196 total)/18 gated; standard/strict/audit/deny gates pass. Each arm64 VZ and amd64/QEMU daemon passes11 distinct actual recovery/pool/SDK/wire cases; scoped container/private-alias queries are empty afterward. Actual directory reacquisition and historical pair retirement pass on both. This does not cover historical image aliases or enable production startup. | +| 2.1 generated model/timestamps | Implemented/reviewed; HTTP/registry still pending | API§1.1/D40 generates37 official operations from312 pinned shapes with exact numeric required/optional epoch dates and strict DTO policy.5 generator cases,34 model cases,1 doctest, regeneration consistency and strict model/tool checks pass. W97–W100 are fixed/rechecked. Final current-worktree workspace2,447/414 gated, including the retained deferred image checkpoint; build/test/fmt/Clippy/rustdoc, root475/tool29-crate audit and both deny graphs pass. This is protocol/model evidence, not API/engine or real-image workflow acceptance. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. + +Current execution priority: complete the Step Functions HTTP boundary, service registration, typed registry and ASL compiler/engine under the user's explicit reorder above. Keep Docker-image work deferred until the subsequent workflow implementation is complete; preserve its current worktree checkpoint and all original final image/platform gates. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index ae1d4004..fc6c2a43 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -464,6 +464,19 @@ Twenty-four new unit/scripted cases cover11 journal transactions/codecs/filesyst Each retained Engine29.5.2 daemon (arm64 VZ and amd64/QEMU) explicitly passes11 distinct actual cases: recovery1, owner/pool7, boto3/HTTP/provider/executor1 and wire2, all with zero ignored. The recovery fixture leaves a real stopped function and running relay across directory reacquisition, then verifies ordered retirement and durable erasure. Scoped owner-container and private-image-alias queries are empty after the suites. QEMU is not native x86 qualification. The existing network/kernel/shared-runtime evidence remains separately scoped; this change does not grant startup readiness or cover historical private image aliases, allowed pulls, endpoint/configuration coordination or P2–P5. The12 feature/index documents resolve265 local references. +## P2 generated model and numeric timestamp review + +API§1.1/D40 lands the official model and protocol-aware numeric serde generation. This is the first P2 contract under the user's explicit priority change; HTTP/core/ASL remain required. Docker-image changes remain a separate deferred worktree checkpoint. Independent final read-only review reports no remaining model/generator issue. + +| Finding | Priority / source | Correction | Verification | +|---|---|---|---| +| W97 | P2; `codegen/src/codegen.rs:256`, separate JSON-error generation | Strict DTO Debug hides all values; strict Error is nonexhaustive with redacted Debug/Display and retained source() | Actual task-token/output/nested-history/error-source fixtures; no blanket ignores | +| W98 | P2; `codegen/templates/epoch_seconds.rs:36` | Classify large exponents with bounded saturating arithmetic instead of rejecting representable zero/underflow at i32 parsing | Required and optional oversized exponent regressions; genuine positive overflow still rejects | +| W99 | P2; strict no-serde branch in `codegen/src/codegen.rs:278` | Emit one manual redacted Debug without deriving another implementation | Strict no-serde policy fixture | +| W100 | P3; `codegen/src/codegen.rs:34` | Gate codec emission on serde usage | Both AWS JSON no-serde protocols and restXml produce no unused codec module | + +Final focused evidence:5 generator tests,34 generated-model cases and1 executed doctest; formatted regeneration and strict tool/model checks pass. Final workspace build/test/fmt/Clippy/rustdoc pass with2,447 tests/414 gated, including retained deferred Docker changes. Audit scans475 workspace/29 standalone-tool dependencies without vulnerabilities; both deny graphs pass with existing unmatched/duplicate-policy notices and no new ignores. The developer tool uses bounded ordinary-file Tokio IO, validates Smithy2.0 and no longer retains unused trait/phase data. This does not close P2, M1 or final real-image/platform requirements. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index f5ae45f7..0e2e6b46 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -347,3 +347,13 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: the same retained directory capability protects filesystem transactions and daemon ownership; interrupted recovery is repeatable without replaying create or touching foreign resources. This is a container-specific part of full reconciliation, not production readiness. - **Pinned by**: [runtime §4.8](./rustack-workflows-20-lambda-runtime-design.md#48-durable-container-intents-and-restart-reconciliation), [implementation plan](./rustack-workflows-90-impl-plan.md). - **Date**: 2026-10-26. Reuse existing nix/sha2/hex dependencies and the bounded ownership worker pool. + + +## D40 — Generate exact JSON epoch timestamps before the Step Functions API + +- **Context**: Step Functions uses AWS JSON1.0 numeric timestamps; default chrono serde emits strings, and the existing optional KMS float helper mishandles negative fractions. +- **Alternatives considered**: binary f64 arithmetic introduces avoidable rounding; copying the KMS helper preserves its negative-epoch defect; workspace-wide arbitrary_precision changes unrelated payload semantics; hand-editing generated files cannot survive regeneration. +- **Decision**: resolve timestamp-format precedence in the generator and emit a JSON-specific RawValue decimal adapter, normalized by checked bounded arithmetic to millisecond precision toward zero. Select strict model generation for Step Functions: documented/nonexhaustive DTOs, required collections, typed builders, redacted diagnostics and no blanket ignores. +- **Why**: SDK wire dates are numeric and exact at the promised precision, including negative/exponent/range edges, while other protocols retain their bindings. Oversized exponents are classified with bounded work. Serde-disabled output gets neither duplicate Debug nor unused codec modules. +- **Pinned by**: [API §1.1](./rustack-workflows-10-api-design.md#11-executable-timestampmodel-generation-contract), [verification](./rustack-workflows-70-verification-plan.md), [implementation plan](./rustack-workflows-90-impl-plan.md). +- **Date**: 2026-10-26. The model reuses workspace serde/chrono/typed-builder; the excluded developer generator adds the already-workspace [Tokio1.54.1](https://docs.rs/tokio/1.54.1/tokio/attr.main.html) with explicit local-IO features and independent audit/deny. No Step Functions endpoint or engine is advertised by the model alone. From fb03acc1d40f49bfe33cb9024d5a8c9c27bc273b Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 06:38:06 -0700 Subject: [PATCH 033/106] phase 2: implement bounded typed Step Functions HTTP transport Implement API section 1.2 and D41 with strict signing, source-level limits, correlated typed replies and retained accepted-command ownership. Independently review and close W101-W104. Ordinary signed boto3 qualifies the transport against an explicit controlled handler. Standard workspace gates pass with 2505 tests and 415 gated cases in the current worktree; model regeneration, focused strict checks, root/tool audit and deny pass. Registry, compiler, real core registration and complete M1 acceptance remain required. Deferred Docker image changes remain outside this commit. --- Cargo.lock | 25 + Cargo.toml | 1 + Makefile | 9 + codegen/services/stepfunctions.toml | 2 +- crates/rustack-stepfunctions-http/Cargo.toml | 35 + crates/rustack-stepfunctions-http/src/body.rs | 120 ++++ .../rustack-stepfunctions-http/src/context.rs | 179 ++++++ .../rustack-stepfunctions-http/src/error.rs | 125 ++++ crates/rustack-stepfunctions-http/src/lib.rs | 27 + .../src/protocol.rs | 131 ++++ .../src/response.rs | 127 ++++ .../rustack-stepfunctions-http/src/service.rs | 171 +++++ .../rustack-stepfunctions-http/src/tests.rs | 596 ++++++++++++++++++ .../src/tests/bounds.rs | 255 ++++++++ .../src/transport.rs | 79 +++ .../src/validation.rs | 228 +++++++ .../tests/network.rs | 281 +++++++++ .../rustack-stepfunctions-model/src/error.rs | 3 +- specs/index.md | 2 +- specs/rustack-workflows-10-api-design.md | 16 + .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 3 +- specs/rustack-workflows-93-review.md | 13 + specs/rustack-workflows-99-decisions.md | 9 + tests/workflows/stepfunctions_http_sdk.py | 45 ++ 25 files changed, 2483 insertions(+), 5 deletions(-) create mode 100644 crates/rustack-stepfunctions-http/Cargo.toml create mode 100644 crates/rustack-stepfunctions-http/src/body.rs create mode 100644 crates/rustack-stepfunctions-http/src/context.rs create mode 100644 crates/rustack-stepfunctions-http/src/error.rs create mode 100644 crates/rustack-stepfunctions-http/src/lib.rs create mode 100644 crates/rustack-stepfunctions-http/src/protocol.rs create mode 100644 crates/rustack-stepfunctions-http/src/response.rs create mode 100644 crates/rustack-stepfunctions-http/src/service.rs create mode 100644 crates/rustack-stepfunctions-http/src/tests.rs create mode 100644 crates/rustack-stepfunctions-http/src/tests/bounds.rs create mode 100644 crates/rustack-stepfunctions-http/src/transport.rs create mode 100644 crates/rustack-stepfunctions-http/src/validation.rs create mode 100644 crates/rustack-stepfunctions-http/tests/network.rs create mode 100644 tests/workflows/stepfunctions_http_sdk.py diff --git a/Cargo.lock b/Cargo.lock index 27b3f2ce..2668736e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4190,6 +4190,31 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-stepfunctions-http" +version = "0.9.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "hyper", + "hyper-util", + "rstest", + "rustack-auth", + "rustack-core", + "rustack-stepfunctions-model", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "rustack-stepfunctions-model" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 1559ccb6..fee7477d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ rustack-lambda-http = { version = "0.11.0", path = "crates/rustack-lambda-http" rustack-lambda-core = { version = "0.11.0", path = "crates/rustack-lambda-core" } rustack-lambda-runtime = { version = "0.9.0", path = "crates/rustack-lambda-runtime" } rustack-stepfunctions-model = { version = "0.9.0", path = "crates/rustack-stepfunctions-model" } +rustack-stepfunctions-http = { version = "0.9.0", path = "crates/rustack-stepfunctions-http" } rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } rustack-logs-http = { version = "0.11.0", path = "crates/rustack-logs-http" } diff --git a/Makefile b/Makefile index 510b106d..d0aab46d 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,15 @@ test-stepfunctions-model: check-stepfunctions-codegen .PHONY: codegen-stepfunctions check-stepfunctions-codegen test-stepfunctions-model +test-stepfunctions-http: + @cargo test -p rustack-stepfunctions-http + @cargo clippy -p rustack-stepfunctions-http --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +test-stepfunctions-http-sdk: + @cargo test -p rustack-stepfunctions-http --test network test_should_exchange_signed_boto3_requests_with_typed_transport -- --ignored --test-threads=1 + +.PHONY: test-stepfunctions-http test-stepfunctions-http-sdk + audit-codegen: @cargo audit --file codegen/Cargo.lock diff --git a/codegen/services/stepfunctions.toml b/codegen/services/stepfunctions.toml index a6204689..e220ace4 100644 --- a/codegen/services/stepfunctions.toml +++ b/codegen/services/stepfunctions.toml @@ -52,7 +52,7 @@ all = [ ] [errors.custom] -UnknownOperationException = { status = 400, message = "Unknown Step Functions operation" } +UnknownOperationException = { status = 404, message = "Unknown Step Functions operation" } UnsupportedOperation = { status = 400, message = "Operation is outside the supported Step Functions profile" } [output] diff --git a/crates/rustack-stepfunctions-http/Cargo.toml b/crates/rustack-stepfunctions-http/Cargo.toml new file mode 100644 index 00000000..5ef03a3a --- /dev/null +++ b/crates/rustack-stepfunctions-http/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "rustack-stepfunctions-http" +description = "Bounded typed Step Functions AWS JSON1.0 HTTP transport" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +rustack-core = { workspace = true } +rustack-auth = { workspace = true } +rustack-stepfunctions-model = { workspace = true } +async-trait = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +http-body = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } +chrono = { workspace = true } +anyhow = { workspace = true } +hyper-util = { workspace = true } diff --git a/crates/rustack-stepfunctions-http/src/body.rs b/crates/rustack-stepfunctions-http/src/body.rs new file mode 100644 index 00000000..30f01e90 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/body.rs @@ -0,0 +1,120 @@ +//! Fully encoded response bytes with redacted diagnostics. + +use std::{ + convert::Infallible, + fmt, + pin::Pin, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use http_body::{Body, Frame, SizeHint}; +use http_body_util::Full; + +/// Bounded buffered response; Debug does not print its workflow data. +#[non_exhaustive] +pub struct StepFunctionsBody { + inner: Full, +} +impl fmt::Debug for StepFunctionsBody { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsBody()") + } +} +impl StepFunctionsBody { + pub(crate) fn new(bytes: Bytes) -> Self { + Self { + inner: Full::new(bytes), + } + } +} +impl Body for StepFunctionsBody { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Infallible>>> { + Pin::new(&mut self.get_mut().inner).poll_frame(cx) + } + fn size_hint(&self) -> SizeHint { + self.inner.size_hint() + } + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } +} + +/// Ordinary JSON requests have no trailer semantics; never silently discard them. +pub(crate) struct RequestBody { + inner: Pin>, + finished: bool, +} +impl RequestBody { + pub(crate) fn new(inner: B) -> Self { + Self { + inner: Box::pin(inner), + finished: false, + } + } +} +impl fmt::Debug for RequestBody { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RequestBody()") + } +} +#[derive(thiserror::Error)] +pub(crate) enum InputError { + #[error("request trailer frames are unsupported")] + Trailers, + #[error("request source failed")] + Source(#[source] Box), +} +impl fmt::Debug for InputError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl Body for RequestBody +where + B: Body, + B::Error: std::error::Error + Send + Sync + 'static, +{ + type Data = Bytes; + type Error = InputError; + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, InputError>>> { + let this = self.get_mut(); + if this.finished { + return Poll::Ready(None); + } + match this.inner.as_mut().poll_frame(cx) { + Poll::Ready(Some(Ok(frame))) if !frame.is_data() => { + this.finished = true; + Poll::Ready(Some(Err(InputError::Trailers))) + } + Poll::Ready(Some(Err(error))) => { + this.finished = true; + Poll::Ready(Some(Err(InputError::Source(Box::new(error))))) + } + Poll::Ready(None) => { + this.finished = true; + Poll::Ready(None) + } + Poll::Ready(Some(Ok(frame))) => Poll::Ready(Some(Ok(frame))), + Poll::Pending => Poll::Pending, + } + } + fn is_end_stream(&self) -> bool { + self.finished || self.inner.is_end_stream() + } + fn size_hint(&self) -> SizeHint { + if self.finished { + SizeHint::with_exact(0) + } else { + self.inner.size_hint() + } + } +} diff --git a/crates/rustack-stepfunctions-http/src/context.rs b/crates/rustack-stepfunctions-http/src/context.rs new file mode 100644 index 00000000..8c101b0a --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/context.rs @@ -0,0 +1,179 @@ +//! Immutable operator scope and request provenance; no serialized trust claims. + +use std::{fmt, num::NonZeroU64, sync::Arc, time::Duration}; + +use rustack_auth::{AuthMode, CredentialProvider}; +use rustack_core::http::BodyBudget; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::HttpError; + +/// Validated partition/region/account configured by the runtime, not the caller. +#[derive(Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct RequestScope { + partition: String, + region: String, + account: String, +} +impl fmt::Debug for RequestScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RequestScope()") + } +} +impl RequestScope { + /// Validate an operator scope for shared HTTP/core use. + /// + /// # Errors + /// Rejects unsupported partitions, invalid region identifiers or non12-digit accounts. + /// + /// ``` + /// let scope = rustack_stepfunctions_http::RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into())?; + /// assert_eq!(scope.region(), "us-east-1"); + /// # Ok::<(), rustack_stepfunctions_http::HttpError>(()) + /// ``` + pub fn new(partition: String, region: String, account: String) -> Result { + if !matches!(partition.as_str(), "aws" | "aws-cn" | "aws-us-gov") + || !(1..=64).contains(®ion.len()) + || !region + .as_bytes() + .first() + .is_some_and(u8::is_ascii_lowercase) + || !region + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + || !region + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + || account.len() != 12 + || !account.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(HttpError::Configuration); + } + Ok(Self { + partition, + region, + account, + }) + } + /// Configured AWS partition. + #[must_use] + pub fn partition(&self) -> &str { + &self.partition + } + /// Configured AWS region. + #[must_use] + pub fn region(&self) -> &str { + &self.region + } + /// Configured account identifier. + #[must_use] + pub fn account(&self) -> &str { + &self.account + } +} + +/// Validated, explicitly selected authentication/body/deadline configuration. +#[derive(Clone)] +#[non_exhaustive] +pub struct StepFunctionsHttpConfig { + pub(crate) scope: Arc, + pub(crate) skip: bool, + pub(crate) credentials: Option>, + pub(crate) body: BodyBudget, + pub(crate) total: Duration, +} +impl fmt::Debug for StepFunctionsHttpConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsHttpConfig()") + } +} +impl StepFunctionsHttpConfig { + /// Construct explicit local-development or strict authentication policy. + /// + /// # Errors + /// Strict mode requires a credential provider; scope and budgets must be valid. + /// + /// ``` + /// use rustack_stepfunctions_http::{RequestScope, StepFunctionsHttpConfig}; + /// let scope = RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into())?; + /// let config = StepFunctionsHttpConfig::new(scope, true, None)?; + /// # Ok::<(), rustack_stepfunctions_http::HttpError>(()) + /// ``` + pub fn new( + scope: RequestScope, + skip_signature_validation: bool, + credentials: Option>, + ) -> Result { + AuthMode::resolve(skip_signature_validation, credentials.as_deref()) + .map_err(|_| HttpError::Configuration)?; + let cap = NonZeroU64::new(2 * 1024 * 1024).ok_or(HttpError::Configuration)?; + Ok(Self { + scope: Arc::new(scope), + skip: skip_signature_validation, + credentials, + body: BodyBudget::control().capped(cap), + total: Duration::from_secs(rustack_core::settings::budgets().request_seconds), + }) + } + /// Apply validated operator budgets without relaxing the2MiB request limit. + /// + /// # Errors + /// Total request duration must be positive and at most one hour. + /// + /// ``` + /// # fn budget(config: rustack_stepfunctions_http::StepFunctionsHttpConfig) -> Result<(), rustack_stepfunctions_http::HttpError> { + /// let config = config.with_budgets(rustack_core::http::BodyBudget::control(), std::time::Duration::from_secs(30))?; + /// # Ok(()) } + /// ``` + pub fn with_budgets(mut self, body: BodyBudget, total: Duration) -> Result { + if total.is_zero() || total > Duration::from_secs(3600) { + return Err(HttpError::Configuration); + } + self.body = body.capped(NonZeroU64::new(2 * 1024 * 1024).ok_or(HttpError::Configuration)?); + self.total = total; + Ok(self) + } +} + +/// Per-call deadline/provenance. Dropping this value does not cancel accepted work. +#[derive(Clone)] +#[non_exhaustive] +pub struct RequestContext { + pub(crate) scope: Arc, + pub(crate) id: Uuid, + pub(crate) deadline: Instant, + pub(crate) authenticated: bool, +} +impl fmt::Debug for RequestContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RequestContext") + .field("request_id", &self.id) + .field("authenticated", &self.authenticated) + .finish_non_exhaustive() + } +} +impl RequestContext { + /// Trusted operator scope shared with the core adapter. + #[must_use] + pub fn scope(&self) -> &RequestScope { + &self.scope + } + /// Generated request correlation ID. + #[must_use] + pub fn request_id(&self) -> Uuid { + self.id + } + /// Absolute monotonic caller deadline, not the workflow's lifetime. + #[must_use] + pub fn deadline(&self) -> Instant { + self.deadline + } + /// Whether this request passed strict signature verification. + #[must_use] + pub fn authenticated(&self) -> bool { + self.authenticated + } +} diff --git a/crates/rustack-stepfunctions-http/src/error.rs b/crates/rustack-stepfunctions-http/src/error.rs new file mode 100644 index 00000000..7f53c686 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/error.rs @@ -0,0 +1,125 @@ +//! Typed sources and static transport diagnostics, independent of user payloads. + +use std::fmt; + +use http::StatusCode; +use rustack_auth::AuthError; +use rustack_core::{http::BodyReadError, json::JsonError}; +use rustack_stepfunctions_model::{StepFunctionsError, StepFunctionsErrorCode}; + +/// HTTP boundary failures; Debug/Display never expose underlying input values. +#[derive(thiserror::Error)] +#[non_exhaustive] +pub enum HttpError { + /// Invalid trusted scope, authentication configuration or budget. + #[error("invalid Step Functions HTTP configuration")] + Configuration, + /// Request method/path/headers are ambiguous or outside this protocol. + #[error("invalid Step Functions HTTP envelope")] + Envelope, + /// No known action exactly matches the target. + #[error("unknown Step Functions operation")] + UnknownOperation, + /// A modeled action is outside this transport profile. + #[error("unsupported Step Functions operation")] + UnsupportedOperation, + /// Authentication failed; the typed source is deliberately not displayed. + #[error("Step Functions authentication failed")] + Authentication(#[source] AuthError), + /// Source body bounds/IO failed without accepting partial bytes. + #[error("Step Functions body collection failed")] + Body(#[source] BodyReadError), + /// Bounded JSON preflight/schema failed. + #[error("invalid Step Functions request JSON")] + Json(#[source] JsonError), + /// An operation-specific input bound failed. + #[error("invalid Step Functions request field")] + Field(StepFunctionsErrorCode), + /// The caller's absolute deadline has elapsed. + #[error("Step Functions HTTP deadline exceeded")] + Deadline, + /// A typed handler rejected a request; diagnostics remain redacted. + #[error("Step Functions handler rejected request")] + Handler(#[source] StepFunctionsError), + /// An invalid/mismatched or oversized handler reply cannot become success. + #[error("invalid Step Functions handler response")] + Response, + /// Encoding failed; source may contain values and is never displayed. + #[error("Step Functions response encoding failed")] + Encoding(#[source] serde_json::Error), +} +impl fmt::Debug for HttpError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} +impl HttpError { + pub(crate) fn wire(&self) -> (&'static str, &str, StatusCode) { + match self { + Self::Configuration | Self::Response | Self::Encoding(_) => ( + "InternalFailure", + "Request processing failed", + StatusCode::INTERNAL_SERVER_ERROR, + ), + Self::Envelope => ( + "ValidationException", + "Invalid HTTP request envelope", + StatusCode::BAD_REQUEST, + ), + Self::UnknownOperation => ( + "UnknownOperationException", + "Unknown Step Functions operation", + StatusCode::NOT_FOUND, + ), + Self::UnsupportedOperation => ( + "UnsupportedOperation", + "Operation is outside the supported Step Functions profile", + StatusCode::BAD_REQUEST, + ), + Self::Authentication(_) => ( + "AccessDeniedException", + "Request authentication failed", + StatusCode::FORBIDDEN, + ), + Self::Body(error) => { + let code = match error { + BodyReadError::TooLarge => "RequestEntityTooLargeException", + BodyReadError::Deadline | BodyReadError::Idle => "RequestTimeoutException", + _ => "MalformedHttpRequestException", + }; + ( + code, + "Request body could not be accepted", + error.status_code(), + ) + } + Self::Json(_) => ( + "ValidationException", + "Invalid request JSON", + StatusCode::BAD_REQUEST, + ), + Self::Field(code) => ( + code.as_str(), + "Invalid request field", + code.default_status_code(), + ), + Self::Deadline => ( + "RequestTimeoutException", + "Request deadline exceeded", + StatusCode::REQUEST_TIMEOUT, + ), + Self::Handler(error) + if error.message.len() <= 1024 + && (error.status_code.is_client_error() + || error.status_code.is_server_error()) => + { + (error.error_type(), &error.message, error.status_code) + } + Self::Handler(_) => ( + "InternalFailure", + "Invalid handler error", + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } +} diff --git a/crates/rustack-stepfunctions-http/src/lib.rs b/crates/rustack-stepfunctions-http/src/lib.rs new file mode 100644 index 00000000..4446fae2 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/lib.rs @@ -0,0 +1,27 @@ +//! Bounded AWS JSON1.0 transport for typed Step Functions handlers. +//! +//! Core owns registry/domain semantics and accepted command lifetime. This crate +//! supplies no default handler and cannot advertise workflow execution by itself. +//! The async-trait handler is deliberately object-safe for trusted runtime DI. +//! The serving gateway owns bounded connection/request admission; this layer +//! does not spawn request jobs or own accepted workflow execution lifetime. +#![forbid(unsafe_code)] + +mod body; +mod context; +mod error; +mod protocol; +mod response; +mod service; +mod transport; +mod validation; + +pub use body::StepFunctionsBody; +pub use context::{RequestContext, RequestScope, StepFunctionsHttpConfig}; +pub use error::HttpError; +pub use protocol::{CONTENT_TYPE, claims_request, resolve_operation}; +pub use service::{StepFunctionsHandler, StepFunctionsHttpService}; +pub use transport::{Request, Response}; + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-http/src/protocol.rs b/crates/rustack-stepfunctions-http/src/protocol.rs new file mode 100644 index 00000000..1f56e841 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/protocol.rs @@ -0,0 +1,131 @@ +//! Exact target-family claiming and bounded HTTP framing policy. + +use http::{ + HeaderMap, Method, + header::{ + AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE as TYPE_HEADER, HOST, + TRANSFER_ENCODING, + }, + request::Parts, +}; +use rustack_stepfunctions_model::StepFunctionsOperation; + +use crate::HttpError; + +/// AWS JSON1.0 media type emitted by every transport response. +pub const CONTENT_TYPE: &str = "application/x-amz-json-1.0"; +const PREFIX: &str = "AWSStepFunctions."; + +/// Claim a target family even when its action or framing will be rejected. +/// +/// ``` +/// let mut headers=http::HeaderMap::new(); +/// headers.insert("x-amz-target",http::HeaderValue::from_static("AWSStepFunctions.Unknown")); +/// assert!(rustack_stepfunctions_http::claims_request(&headers)); +/// ``` +#[must_use] +pub fn claims_request(headers: &HeaderMap) -> bool { + headers.get_all("x-amz-target").iter().any(|value| { + value.as_bytes().starts_with(PREFIX.as_bytes()) || value.as_bytes() == b"AWSStepFunctions" + }) +} +/// Resolve exactly one bounded target into a modeled action. +/// +/// # Errors +/// Returns envelope errors for ambiguity/non-ASCII/oversize or `UnknownOperation`. +/// +/// ``` +/// let mut headers=http::HeaderMap::new(); +/// headers.insert("x-amz-target",http::HeaderValue::from_static("AWSStepFunctions.StartExecution")); +/// let operation=rustack_stepfunctions_http::resolve_operation(&headers)?; +/// assert_eq!(operation,rustack_stepfunctions_model::StepFunctionsOperation::StartExecution); +/// # Ok::<(),rustack_stepfunctions_http::HttpError>(()) +/// ``` +pub fn resolve_operation(headers: &HeaderMap) -> Result { + let target = single(headers, "x-amz-target", 256)?.ok_or(HttpError::Envelope)?; + let name = target + .strip_prefix(PREFIX) + .ok_or(HttpError::UnknownOperation)?; + StepFunctionsOperation::from_name(name).ok_or(HttpError::UnknownOperation) +} +fn single<'a>( + headers: &'a HeaderMap, + name: &str, + maximum: usize, +) -> Result, HttpError> { + let mut values = headers.get_all(name).iter(); + let Some(value) = values.next() else { + return Ok(None); + }; + if values.next().is_some() || value.as_bytes().len() > maximum || !value.as_bytes().is_ascii() { + return Err(HttpError::Envelope); + } + value.to_str().map(Some).map_err(|_| HttpError::Envelope) +} +pub(crate) fn envelope(parts: &Parts) -> Result, HttpError> { + if parts.method != Method::POST || parts.uri.path() != "/" || parts.uri.query().is_some() { + return Err(HttpError::Envelope); + } + let mut bytes = 0_usize; + if parts.headers.len() > 64 { + return Err(HttpError::Envelope); + } + for (name, value) in &parts.headers { + if value.as_bytes().len() > 8192 { + return Err(HttpError::Envelope); + } + bytes = bytes + .checked_add(name.as_str().len()) + .and_then(|count| count.checked_add(value.as_bytes().len())) + .ok_or(HttpError::Envelope)?; + if bytes > 32768 { + return Err(HttpError::Envelope); + } + } + single(&parts.headers, "x-amz-target", 256)?.ok_or(HttpError::Envelope)?; + let content = single(&parts.headers, TYPE_HEADER.as_str(), 256)?.ok_or(HttpError::Envelope)?; + let mut fields = content.split(';'); + if !fields + .next() + .is_some_and(|media| media.trim().eq_ignore_ascii_case(CONTENT_TYPE)) + { + return Err(HttpError::Envelope); + } + if let Some(charset) = fields.next() { + let charset = charset.trim(); + if !(charset.eq_ignore_ascii_case("charset=utf-8") + || charset.eq_ignore_ascii_case("charset=\"utf-8\"")) + || fields.next().is_some() + { + return Err(HttpError::Envelope); + } + } + if single(&parts.headers, CONTENT_ENCODING.as_str(), 64)? + .is_some_and(|value| !value.eq_ignore_ascii_case("identity")) + { + return Err(HttpError::Envelope); + } + for name in [ + AUTHORIZATION.as_str(), + HOST.as_str(), + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + ] { + single(&parts.headers, name, 8192)?; + } + let length = single(&parts.headers, CONTENT_LENGTH.as_str(), 20)? + .map(|value| { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(HttpError::Envelope); + } + value.parse::().map_err(|_| HttpError::Envelope) + }) + .transpose()?; + if let Some(encoding) = single(&parts.headers, TRANSFER_ENCODING.as_str(), 64)? { + if length.is_some() || !encoding.eq_ignore_ascii_case("chunked") { + return Err(HttpError::Envelope); + } + } + Ok(length) +} diff --git a/crates/rustack-stepfunctions-http/src/response.rs b/crates/rustack-stepfunctions-http/src/response.rs new file mode 100644 index 00000000..fe24d44e --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/response.rs @@ -0,0 +1,127 @@ +//! Bounded complete encoding, exact operation replies and static error envelopes. + +use std::io::{self, Write}; + +use bytes::Bytes; +use http::{ + HeaderValue, StatusCode, + header::{CONTENT_LENGTH, CONTENT_TYPE as TYPE_HEADER}, +}; +use serde::Serialize; +use uuid::Uuid; + +use crate::{CONTENT_TYPE, HttpError, StepFunctionsBody, transport::Response}; + +const LIMIT: usize = 8 * 1024 * 1024; +struct Writer { + bytes: Vec, + failed: bool, +} +impl Write for Writer { + fn write(&mut self, data: &[u8]) -> io::Result { + let Some(length) = self + .bytes + .len() + .checked_add(data.len()) + .filter(|length| *length <= LIMIT) + else { + self.failed = true; + return Err(io::Error::other("response byte limit exceeded")); + }; + let capacity = self.bytes.capacity(); + let target = length.max(capacity.saturating_mul(2).max(4096)).min(LIMIT); + if length > capacity + && self + .bytes + .try_reserve_exact(target - self.bytes.len()) + .is_err() + { + self.failed = true; + return Err(io::Error::other("response allocation failed")); + } + self.bytes.extend_from_slice(data); + Ok(data.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn encode( + write: impl FnOnce(&mut Writer) -> Result<(), serde_json::Error>, +) -> Result { + let mut writer = Writer { + bytes: Vec::new(), + failed: false, + }; + if let Err(error) = write(&mut writer) { + return Err(if writer.failed { + HttpError::Response + } else { + HttpError::Encoding(error) + }); + } + Ok(Bytes::from(writer.bytes)) +} +pub(crate) fn success( + value: &Response, + id: Uuid, +) -> Result, HttpError> { + let bytes = encode(|writer| value.encode(writer))?; + build(bytes, StatusCode::OK, id, None) +} +#[derive(Serialize)] +struct ErrorEnvelope<'a> { + #[serde(rename = "__type")] + kind: &'a str, + message: &'a str, +} +pub(crate) fn failure(error: &HttpError, id: Uuid) -> http::Response { + let (code, message, status) = error.wire(); + tracing::debug!(error_code = code, "Step Functions request rejected"); + let result = encode(|writer| { + serde_json::to_writer( + writer, + &ErrorEnvelope { + kind: code, + message, + }, + ) + }) + .and_then(|bytes| build(bytes, status, id, Some(code))); + match result { + Ok(response) => response, + Err(_) => emergency(), + } +} +fn build( + bytes: Bytes, + status: StatusCode, + id: Uuid, + error: Option<&'static str>, +) -> Result, HttpError> { + let length = bytes.len(); + let mut response = http::Response::new(StepFunctionsBody::new(bytes)); + *response.status_mut() = status; + let headers = response.headers_mut(); + headers.insert(TYPE_HEADER, HeaderValue::from_static(CONTENT_TYPE)); + headers.insert(CONTENT_LENGTH, HeaderValue::from(length)); + headers.insert( + "x-amzn-requestid", + HeaderValue::try_from(id.to_string()).map_err(|_| HttpError::Response)?, + ); + headers.insert("server", HeaderValue::from_static("Rustack")); + if let Some(code) = error { + headers.insert("x-amzn-errortype", HeaderValue::from_static(code)); + } + Ok(response) +} +fn emergency() -> http::Response { + let mut response = http::Response::new(StepFunctionsBody::new(Bytes::from_static( + br#"{"__type":"InternalFailure","message":"Request processing failed"}"#, + ))); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + response + .headers_mut() + .insert(TYPE_HEADER, HeaderValue::from_static(CONTENT_TYPE)); + response +} diff --git a/crates/rustack-stepfunctions-http/src/service.rs b/crates/rustack-stepfunctions-http/src/service.rs new file mode 100644 index 00000000..2267704e --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/service.rs @@ -0,0 +1,171 @@ +//! Owned per-request transport futures; accepted commands remain core-owned. + +use std::{convert::Infallible, error::Error, fmt, future::Future, pin::Pin, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use http_body::Body; +use hyper::body::Incoming; +use rustack_auth::{AuthMode, hash_payload}; +use rustack_core::http::collect_body; +use rustack_stepfunctions_model::StepFunctionsError; +use tokio::time::{Instant, timeout_at}; +use tracing::Instrument; +use uuid::Uuid; + +use crate::{ + HttpError, Request, RequestContext, Response, StepFunctionsBody, StepFunctionsHttpConfig, + protocol, response, +}; + +/// Object-safe typed command boundary for the runtime's real core adapter. +/// +/// async-trait is required because services retain `Arc`. +/// Accepted registry commands survive a dropped/timed-out HTTP future; handlers +/// must not equate reply-channel closure with execution cancellation. +#[async_trait] +pub trait StepFunctionsHandler: Send + Sync + 'static { + /// Handle one decoded, bounded request under its caller deadline. + /// + /// # Errors + /// Return a typed domain/protocol error; never include tokens or input in diagnostics. + async fn handle( + &self, + context: RequestContext, + request: Request, + ) -> Result; +} + +/// Hyper service requiring an explicit real handler and validated configuration. +#[derive(Clone)] +#[non_exhaustive] +pub struct StepFunctionsHttpService { + handler: Arc, + config: Arc, +} +impl fmt::Debug for StepFunctionsHttpService { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("StepFunctionsHttpService()") + } +} +impl StepFunctionsHttpService { + /// Compose transport with a trusted typed core adapter. + /// + /// ```no_run + /// # fn compose(handler: std::sync::Arc, config: rustack_stepfunctions_http::StepFunctionsHttpConfig) { + /// let service=rustack_stepfunctions_http::StepFunctionsHttpService::new(handler,config); + /// # } + /// ``` + #[must_use] + pub fn new(handler: Arc, config: StepFunctionsHttpConfig) -> Self { + Self { + handler, + config: Arc::new(config), + } + } + /// Process any bounded byte body; network serving uses the same path. + /// + /// ```no_run + /// # async fn serve(service: rustack_stepfunctions_http::StepFunctionsHttpService, request: http::Request>) { + /// let response=service.handle(request).await; + /// # } + /// ``` + pub async fn handle(&self, request: http::Request) -> http::Response + where + B: Body + Send + 'static, + B::Error: Error + Send + Sync + 'static, + { + let id = Uuid::new_v4(); + let deadline = Instant::now() + self.config.total; + let span = tracing::info_span!("stepfunctions.request",request_id=%id); + let result = timeout_at( + deadline, + self.process(request, id, deadline).instrument(span), + ) + .await; + match result { + Ok(Ok(response)) => response, + Ok(Err(error)) => response::failure(&error, id), + Err(_) => response::failure(&HttpError::Deadline, id), + } + } + async fn process( + &self, + request: http::Request, + id: Uuid, + deadline: Instant, + ) -> Result, HttpError> + where + B: Body + Send + 'static, + B::Error: Error + Send + Sync + 'static, + { + let (parts, body) = request.into_parts(); + let declared = protocol::envelope(&parts)?; + if declared.is_some_and(|length| { + u64::try_from(length).map_or(true, |length| length > self.config.body.max_bytes()) + }) { + return Err(HttpError::Body(rustack_core::http::BodyReadError::TooLarge)); + } + let bytes = collect_body(crate::body::RequestBody::new(body), self.config.body) + .await + .map_err(HttpError::Body)?; + if declared.is_some_and(|length| length != bytes.len()) { + return Err(HttpError::Envelope); + } + let authenticated = AuthMode::resolve(self.config.skip, self.config.credentials.as_deref()) + .map_err(HttpError::Authentication)? + .verify(&parts, &hash_payload(&bytes)) + .map_err(HttpError::Authentication)?; + if let Some(auth) = &authenticated { + if auth.service != "states" + || auth.region != self.config.scope.region() + || !auth + .signed_headers + .iter() + .any(|name| name == "x-amz-target") + { + return Err(HttpError::Authentication( + rustack_auth::AuthError::InvalidCredential, + )); + } + } + check(deadline)?; + let operation = protocol::resolve_operation(&parts.headers)?; + let request = Request::decode(operation, &bytes)?; + check(deadline)?; + let context = RequestContext { + scope: self.config.scope.clone(), + id, + deadline, + authenticated: authenticated.is_some(), + }; + let reply = self + .handler + .handle(context, request) + .await + .map_err(HttpError::Handler)?; + check(deadline)?; + if reply.operation() != operation { + return Err(HttpError::Response); + } + let response = response::success(&reply, id)?; + check(deadline)?; + Ok(response) + } +} +fn check(deadline: Instant) -> Result<(), HttpError> { + if Instant::now() >= deadline { + Err(HttpError::Deadline) + } else { + Ok(()) + } +} +impl hyper::service::Service> for StepFunctionsHttpService { + type Response = http::Response; + type Error = Infallible; + type Future = Pin> + Send>>; + fn call(&self, request: http::Request) -> Self::Future { + let service = self.clone(); + Box::pin(async move { Ok(service.handle(request).await) }) + } +} diff --git a/crates/rustack-stepfunctions-http/src/tests.rs b/crates/rustack-stepfunctions-http/src/tests.rs new file mode 100644 index 00000000..9ec7abcf --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/tests.rs @@ -0,0 +1,596 @@ +use std::{ + convert::Infallible, + future::pending, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll}, + time::Duration, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use http::{HeaderValue, StatusCode}; +use http_body::{Body, Frame}; +use http_body_util::{BodyExt, Full}; +use rustack_auth::{ + StaticCredentialProvider, + canonical::build_canonical_request, + hash_payload, + sigv4::{build_string_to_sign, compute_signature, derive_signing_key}, +}; +use rustack_core::http::BodyBudget; +use rustack_stepfunctions_model::{ + StepFunctionsError, StepFunctionsErrorCode, StepFunctionsOperation as Op, output, +}; +use serde_json::{Value, json}; +use tokio::{sync::mpsc, time::Instant}; + +use crate::*; + +#[derive(Debug, Clone, Copy)] +enum Mode { + Normal, + Mismatch, + Error, + Oversize, + Pending, + OversizeError, + SuccessError, +} +struct Handler { + count: AtomicUsize, + sender: mpsc::Sender<(RequestContext, Request)>, + mode: Mode, +} +#[async_trait] +impl StepFunctionsHandler for Handler { + async fn handle( + &self, + context: RequestContext, + request: Request, + ) -> Result { + self.count.fetch_add(1, Ordering::SeqCst); + let op = request.operation(); + self.sender.try_send((context, request)).unwrap(); + match self.mode { + Mode::Pending => pending().await, + Mode::Error => Err(StepFunctionsError::with_message( + StepFunctionsErrorCode::InvalidName, + "domain rejected", + )), + Mode::OversizeError => Err(StepFunctionsError::with_message( + StepFunctionsErrorCode::InvalidName, + "private".repeat(1024), + )), + Mode::SuccessError => { + let mut error = StepFunctionsError::new(StepFunctionsErrorCode::InvalidName); + error.status_code = StatusCode::OK; + Err(error) + } + Mode::Mismatch => Ok(output::DeleteStateMachineOutput::default().into()), + Mode::Oversize => { + let mut output = output::StartExecutionOutput::default(); + output.execution_arn = "\"".repeat(8 * 1024 * 1024); + Ok(output.into()) + } + Mode::Normal => Ok(reply(op)), + } + } +} +fn reply(op: Op) -> Response { + match op { + Op::CreateStateMachine => output::CreateStateMachineOutput::default().into(), + Op::UpdateStateMachine => output::UpdateStateMachineOutput::default().into(), + Op::DeleteStateMachine => output::DeleteStateMachineOutput::default().into(), + Op::DescribeStateMachine => output::DescribeStateMachineOutput::default().into(), + Op::ListStateMachines => output::ListStateMachinesOutput::default().into(), + Op::ValidateStateMachineDefinition => { + output::ValidateStateMachineDefinitionOutput::default().into() + } + Op::StartExecution => { + let mut value = output::StartExecutionOutput::default(); + value.start_date = chrono::DateTime::from_timestamp_millis(-1250).unwrap(); + value.execution_arn = + "arn:aws:states:us-east-1:000000000000:execution:test:wire".into(); + value.into() + } + Op::DescribeExecution => output::DescribeExecutionOutput::default().into(), + Op::ListExecutions => output::ListExecutionsOutput::default().into(), + Op::StopExecution => output::StopExecutionOutput::default().into(), + Op::GetExecutionHistory => output::GetExecutionHistoryOutput::default().into(), + Op::DescribeStateMachineForExecution => { + output::DescribeStateMachineForExecutionOutput::default().into() + } + Op::TagResource => output::TagResourceOutput::default().into(), + Op::UntagResource => output::UntagResourceOutput::default().into(), + Op::ListTagsForResource => output::ListTagsForResourceOutput::default().into(), + Op::SendTaskSuccess => output::SendTaskSuccessOutput::default().into(), + Op::SendTaskFailure => output::SendTaskFailureOutput::default().into(), + Op::SendTaskHeartbeat => output::SendTaskHeartbeatOutput::default().into(), + _ => panic!("unsupported fixture action"), + } +} +fn scope() -> RequestScope { + RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into()).unwrap() +} +fn fixture( + mode: Mode, + strict: bool, +) -> ( + StepFunctionsHttpService, + Arc, + mpsc::Receiver<(RequestContext, Request)>, +) { + let (sender, receiver) = mpsc::channel(64); + let handler = Arc::new(Handler { + count: AtomicUsize::new(0), + sender, + mode, + }); + let credentials = Arc::new(StaticCredentialProvider::new(vec![( + "local-key".into(), + "local-secret".into(), + )])); + let config = StepFunctionsHttpConfig::new(scope(), !strict, Some(credentials)).unwrap(); + ( + StepFunctionsHttpService::new(handler.clone(), config), + handler, + receiver, + ) +} +fn request(action: &str, body: impl Into) -> http::Request> { + http::Request::builder() + .method("POST") + .uri("/") + .header("content-type", CONTENT_TYPE) + .header("x-amz-target", format!("AWSStepFunctions.{action}")) + .body(Full::new(body.into())) + .unwrap() +} +async fn data(response: http::Response) -> Value { + serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap() +} +fn cases() -> Vec<(Op, Value)> { + vec![ + ( + Op::CreateStateMachine, + json!({"name":"machine","roleArn":"role","definition":"{}"}), + ), + ( + Op::UpdateStateMachine, + json!({"stateMachineArn":"machine","definition":"{}"}), + ), + (Op::DeleteStateMachine, json!({"stateMachineArn":"machine"})), + ( + Op::DescribeStateMachine, + json!({"stateMachineArn":"machine"}), + ), + (Op::ListStateMachines, json!({})), + ( + Op::ValidateStateMachineDefinition, + json!({"definition":"not-asl"}), + ), + ( + Op::StartExecution, + json!({"stateMachineArn":"machine","input":"{ \"secret\": 7 }"}), + ), + (Op::DescribeExecution, json!({"executionArn":"execution"})), + (Op::ListExecutions, json!({"stateMachineArn":"machine"})), + (Op::StopExecution, json!({"executionArn":"execution"})), + (Op::GetExecutionHistory, json!({"executionArn":"execution"})), + ( + Op::DescribeStateMachineForExecution, + json!({"executionArn":"execution"}), + ), + ( + Op::TagResource, + json!({"resourceArn":"machine","tags":[{"key":"a","value":"b"}]}), + ), + ( + Op::UntagResource, + json!({"resourceArn":"machine","tagKeys":["a"]}), + ), + (Op::ListTagsForResource, json!({"resourceArn":"machine"})), + ( + Op::SendTaskSuccess, + json!({"taskToken":"secret-token","output":"null"}), + ), + ( + Op::SendTaskFailure, + json!({"taskToken":"secret-token","error":"error","cause":"cause"}), + ), + (Op::SendTaskHeartbeat, json!({"taskToken":"secret-token"})), + ] +} +#[tokio::test] +async fn test_should_dispatch_all_typed_actions_without_enum_wrappers_or_data_in_debug() { + let (service, handler, mut accepted) = fixture(Mode::Normal, false); + for (op, body) in cases() { + let response = service + .handle(request(op.as_str(), serde_json::to_vec(&body).unwrap())) + .await; + assert_eq!(response.status(), StatusCode::OK, "{op:?}"); + assert_eq!(response.headers()["content-type"], CONTENT_TYPE); + assert!(!response.headers().contains_key("x-amzn-errortype")); + let id = response.headers()["x-amzn-requestid"] + .to_str() + .unwrap() + .parse::() + .unwrap(); + let decoded = data(response).await; + if op == Op::StartExecution { + assert_eq!(decoded["startDate"], json!(-1.25)); + assert!(!decoded.as_object().unwrap().contains_key("StartExecution")); + } + let (context, request) = accepted.recv().await.unwrap(); + assert_eq!(context.request_id(), id); + assert_eq!(context.scope(), &scope()); + assert!(!context.authenticated()); + assert_eq!(request.operation(), op); + assert!(!format!("{request:?}").contains("secret")); + if let Request::StartExecution(value) = request { + assert_eq!(value.input.as_deref(), Some("{ \"secret\": 7 }")); + } + } + assert_eq!(handler.count.load(Ordering::SeqCst), 18); +} +#[tokio::test] +async fn test_should_accept_empty_body_as_empty_json_and_reject_unknown_or_unsupported_actions() { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + assert_eq!( + service + .handle(request("ListStateMachines", Bytes::new())) + .await + .status(), + StatusCode::OK + ); + let unknown = service.handle(request("MadeUp", b"{}".as_slice())).await; + assert_eq!(unknown.status(), StatusCode::NOT_FOUND); + assert_eq!(data(unknown).await["__type"], "UnknownOperationException"); + let unsupported = service + .handle(request("StartSyncExecution", b"not-json".as_slice())) + .await; + assert_eq!(unsupported.status(), StatusCode::BAD_REQUEST); + assert_eq!(data(unsupported).await["__type"], "UnsupportedOperation"); + assert_eq!(handler.count.load(Ordering::SeqCst), 1); +} +#[rstest::rstest] +#[case("method")] +#[case("path")] +#[case("query")] +#[case("content_type")] +#[case("duplicate_type")] +#[case("duplicate_target")] +#[case("duplicate_auth")] +#[case("compression")] +#[case("length_mismatch")] +#[case("length_and_transfer")] +#[case("header_count")] +#[case("header_value")] +#[tokio::test] +async fn test_should_reject_ambiguous_envelopes_before_handler(#[case] problem: &str) { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let mut request = request("ListStateMachines", b"{}".as_slice()); + match problem { + "method" => *request.method_mut() = http::Method::GET, + "path" => *request.uri_mut() = "/other".parse().unwrap(), + "query" => *request.uri_mut() = "/?ignored=1".parse().unwrap(), + "content_type" => { + request.headers_mut().insert( + "content-type", + HeaderValue::from_static("application/x-amz-json-1.1"), + ); + } + "duplicate_type" => { + request + .headers_mut() + .append("content-type", HeaderValue::from_static(CONTENT_TYPE)); + } + "duplicate_target" => { + request.headers_mut().append( + "x-amz-target", + HeaderValue::from_static("AWSStepFunctions.DeleteStateMachine"), + ); + } + "duplicate_auth" => { + request + .headers_mut() + .append("authorization", HeaderValue::from_static("one")); + request + .headers_mut() + .append("authorization", HeaderValue::from_static("two")); + } + "compression" => { + request + .headers_mut() + .insert("content-encoding", HeaderValue::from_static("gzip")); + } + "length_mismatch" => { + request + .headers_mut() + .insert("content-length", HeaderValue::from_static("0")); + } + "length_and_transfer" => { + request + .headers_mut() + .insert("content-length", HeaderValue::from_static("2")); + request + .headers_mut() + .insert("transfer-encoding", HeaderValue::from_static("chunked")); + } + "header_count" => { + for index in 0..65 { + request.headers_mut().insert( + format!("x-extra-{index}") + .parse::() + .unwrap(), + HeaderValue::from_static("x"), + ); + } + } + "header_value" => { + request + .headers_mut() + .insert("x-extra", HeaderValue::from_str(&"a".repeat(8193)).unwrap()); + } + _ => panic!("fixture case"), + } + assert!(claims_request(request.headers())); + assert_eq!( + service.handle(request).await.status(), + StatusCode::BAD_REQUEST + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[test] +fn test_should_claim_any_matching_occurrence_but_not_other_services() { + let mut headers = http::HeaderMap::new(); + headers.append( + "x-amz-target", + HeaderValue::from_static("AWSEvents.PutEvents"), + ); + assert!(!claims_request(&headers)); + headers.append( + "x-amz-target", + HeaderValue::from_static("AWSStepFunctions.Unknown"), + ); + assert!(claims_request(&headers)); + assert!(resolve_operation(&headers).is_err()); + headers.clear(); + headers.insert("x-amz-target", HeaderValue::from_static("AWSStepFunctions")); + assert!(claims_request(&headers)); +} +#[rstest::rstest] +#[case(b"{\"maxResults\":1,\"maxResults\":2}".as_slice())] +#[case(b"{\"unknown\":1}".as_slice())] +#[case(b"[]".as_slice())] +#[case(b"{\"maxResults\":-1}".as_slice())] +#[tokio::test] +async fn test_should_reject_bad_shape_duplicates_unknown_fields_and_ranges(#[case] body: &[u8]) { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + assert_eq!( + service + .handle(request("ListStateMachines", Bytes::copy_from_slice(body))) + .await + .status(), + StatusCode::BAD_REQUEST + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[rstest::rstest] +#[case(Mode::Mismatch)] +#[case(Mode::Oversize)] +#[case(Mode::Error)] +#[case(Mode::OversizeError)] +#[case(Mode::SuccessError)] +#[tokio::test] +async fn test_should_bound_and_correlate_replies_and_preserve_domain_errors(#[case] mode: Mode) { + let (service, handler, _accepted) = fixture(mode, false); + let response = service + .handle(request( + "StartExecution", + b"{\"stateMachineArn\":\"machine\"}".as_slice(), + )) + .await; + let code = if matches!(mode, Mode::Error) { + "InvalidName" + } else { + "InternalFailure" + }; + assert_eq!(response.headers()["x-amzn-errortype"], code); + assert_eq!( + response.status(), + if matches!(mode, Mode::Error) { + StatusCode::BAD_REQUEST + } else { + StatusCode::INTERNAL_SERVER_ERROR + } + ); + assert_eq!(data(response).await["__type"], code); + assert_eq!(handler.count.load(Ordering::SeqCst), 1); +} +struct NeverBody { + polled: Arc, +} +impl Body for NeverBody { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Infallible>>> { + self.polled.fetch_add(1, Ordering::SeqCst); + Poll::Pending + } +} +#[tokio::test(start_paused = true)] +async fn test_should_bound_slow_body_and_never_poll_oversized_declared_input() { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let polled = Arc::new(AtomicUsize::new(0)); + let (mut parts, _) = request("ListStateMachines", Bytes::new()).into_parts(); + parts + .headers + .insert("content-length", HeaderValue::from(3 * 1024 * 1024)); + let response = service + .handle(http::Request::from_parts( + parts, + NeverBody { + polled: polled.clone(), + }, + )) + .await; + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(polled.load(Ordering::SeqCst), 0); + let (parts, _) = request("ListStateMachines", Bytes::new()).into_parts(); + let response = service + .handle(http::Request::from_parts(parts, NeverBody { polled })) + .await; + assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[tokio::test] +async fn test_should_cap_actual_body_bytes_without_trusting_length() { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let response = service + .handle(request( + "ListStateMachines", + Bytes::from(vec![b'a'; 2 * 1024 * 1024 + 1]), + )) + .await; + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[tokio::test(start_paused = true)] +async fn test_should_timeout_only_the_waiter_after_a_typed_command_was_accepted() { + let (_original, handler, mut accepted) = fixture(Mode::Pending, false); + let config = StepFunctionsHttpConfig::new(scope(), true, None) + .unwrap() + .with_budgets( + BodyBudget::new( + 2 * 1024 * 1024, + Duration::from_secs(5), + Duration::from_secs(2), + ) + .unwrap(), + Duration::from_secs(1), + ) + .unwrap(); + let service = StepFunctionsHttpService::new(handler.clone(), config); + let before = Instant::now(); + let response = service + .handle(request( + "StartExecution", + b"{\"stateMachineArn\":\"machine\"}".as_slice(), + )) + .await; + assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT); + assert!(Instant::now() >= before + Duration::from_secs(1)); + let (context, request) = accepted.recv().await.unwrap(); + assert_eq!(request.operation(), Op::StartExecution); + assert!(context.deadline() <= Instant::now()); + assert_eq!(handler.count.load(Ordering::SeqCst), 1); +} +#[tokio::test] +async fn test_should_leave_accepted_command_owned_when_http_caller_disappears() { + let (service, _handler, mut accepted) = fixture(Mode::Pending, false); + let task = tokio::spawn(async move { + service + .handle(request( + "StartExecution", + b"{\"stateMachineArn\":\"machine\"}".as_slice(), + )) + .await + }); + let (context, request) = accepted.recv().await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(request.operation(), Op::StartExecution); + assert!(!context.request_id().is_nil()); +} +fn signed( + body: &[u8], + region: &str, + service: &str, + bind_target: bool, +) -> http::Request> { + let mut request = request("ListStateMachines", Bytes::copy_from_slice(body)); + let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + let date = ×tamp[..8]; + request + .headers_mut() + .insert("host", HeaderValue::from_static("localhost")); + request + .headers_mut() + .insert("x-amz-date", HeaderValue::from_str(×tamp).unwrap()); + let mut names = vec!["content-type", "host", "x-amz-date"]; + if bind_target { + names.push("x-amz-target"); + } + let headers: Vec<_> = names + .iter() + .map(|name| (*name, request.headers()[*name].to_str().unwrap())) + .collect(); + let canonical = build_canonical_request("POST", "/", "", &headers, &names, &hash_payload(body)); + let scope = format!("{date}/{region}/{service}/aws4_request"); + let string = build_string_to_sign(×tamp, &scope, &hash_payload(canonical.as_bytes())); + let key = derive_signing_key("local-secret", date, region, service).unwrap(); + let signature = compute_signature(&key, &string).unwrap(); + request.headers_mut().insert( + "authorization", + HeaderValue::from_str(&format!( + "AWS4-HMAC-SHA256 Credential=local-key/{scope}, SignedHeaders={}, \ + Signature={signature}", + names.join(";") + )) + .unwrap(), + ); + request +} +#[tokio::test] +async fn test_should_verify_original_payload_and_bind_service_region_and_action() { + let (service, handler, mut accepted) = fixture(Mode::Normal, true); + assert_eq!( + service + .handle(signed(b"{}", "us-east-1", "states", true)) + .await + .status(), + StatusCode::OK + ); + assert!(accepted.recv().await.unwrap().0.authenticated()); + for (region, name, target) in [ + ("us-west-2", "states", true), + ("us-east-1", "lambda", true), + ("us-east-1", "states", false), + ] { + let response = service.handle(signed(b"{}", region, name, target)).await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(data(response).await["__type"], "AccessDeniedException"); + } + let (mut parts, _) = signed(b"{}", "us-east-1", "states", true).into_parts(); + parts.headers.remove("x-amz-content-sha256"); + let response = service + .handle(http::Request::from_parts( + parts, + Full::new(Bytes::from_static(b"{\"maxResults\":1}")), + )) + .await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(handler.count.load(Ordering::SeqCst), 1); +} +#[test] +fn test_should_fail_closed_config_and_redact_scope_and_auth_sources() { + assert!(StepFunctionsHttpConfig::new(scope(), false, None).is_err()); + assert!(RequestScope::new("other".into(), "us-east-1".into(), "000000000000".into()).is_err()); + assert!(RequestScope::new("aws".into(), "-".into(), "000000000000".into()).is_err()); + assert!(RequestScope::new("aws".into(), "us-east-1".into(), "0".into()).is_err()); + let error = HttpError::Authentication(rustack_auth::AuthError::AccessKeyNotFound( + "secret-access-key".into(), + )); + assert!(!format!("{error:?} {error}").contains("secret")); + assert!(!format!("{:?}", scope()).contains("000000000000")); +} + +mod bounds; diff --git a/crates/rustack-stepfunctions-http/src/tests/bounds.rs b/crates/rustack-stepfunctions-http/src/tests/bounds.rs new file mode 100644 index 00000000..6364c546 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/tests/bounds.rs @@ -0,0 +1,255 @@ +use std::collections::VecDeque; + +use super::*; + +#[rstest::rstest] +#[case("name")] +#[case("name_charset")] +#[case("arn")] +#[case("definition")] +#[case("execution_input")] +#[case("empty_input")] +#[case("output")] +#[case("token")] +#[case("next_token")] +#[case("tags")] +#[case("tag_key")] +#[case("tag_value")] +#[case("tag_keys")] +#[case("cause")] +#[case("error")] +#[case("logging")] +#[case("version")] +#[case("encryption_period")] +#[tokio::test] +async fn test_should_bound_operation_fields_before_handler_entry(#[case] problem: &str) { + let (action, body, code) = match problem { + "name" => ( + "StartExecution", + json!({"stateMachineArn":"machine","name":"a".repeat(81)}), + "InvalidName", + ), + "name_charset" => ( + "StartExecution", + json!({"stateMachineArn":"machine","name":"bad name"}), + "InvalidName", + ), + "arn" => ( + "DescribeExecution", + json!({"executionArn":"a".repeat(257)}), + "InvalidArn", + ), + "definition" => ( + "ValidateStateMachineDefinition", + json!({"definition":"a".repeat(1024*1024+1)}), + "ValidationException", + ), + "execution_input" => ( + "StartExecution", + json!({"stateMachineArn":"machine","input":serde_json::to_string(&"a".repeat(262_144)).unwrap()}), + "InvalidExecutionInput", + ), + "empty_input" => ( + "StartExecution", + json!({"stateMachineArn":"machine","input":""}), + "InvalidExecutionInput", + ), + "output" => ( + "SendTaskSuccess", + json!({"taskToken":"token","output":"not-json"}), + "InvalidOutput", + ), + "token" => ( + "SendTaskHeartbeat", + json!({"taskToken":"é"}), + "InvalidToken", + ), + "next_token" => ( + "ListStateMachines", + json!({"nextToken":"a".repeat(2049)}), + "InvalidToken", + ), + "tags" => ( + "TagResource", + json!({"resourceArn":"machine","tags":vec![json!({"key":"a"});51]}), + "TooManyTags", + ), + "tag_key" => ( + "TagResource", + json!({"resourceArn":"machine","tags":[{"key":"a".repeat(129)}]}), + "ValidationException", + ), + "tag_value" => ( + "TagResource", + json!({"resourceArn":"machine","tags":[{"key":"a","value":"a".repeat(257)}]}), + "ValidationException", + ), + "tag_keys" => ( + "UntagResource", + json!({"resourceArn":"machine","tagKeys":vec!["key";51]}), + "TooManyTags", + ), + "cause" => ( + "StopExecution", + json!({"executionArn":"execution","cause":"a".repeat(32_769)}), + "ValidationException", + ), + "error" => ( + "SendTaskFailure", + json!({"taskToken":"token","error":"a".repeat(257)}), + "ValidationException", + ), + "logging" => ( + "CreateStateMachine", + json!({"name":"machine","roleArn":"role","definition":"{}","loggingConfiguration":{"destinations":[{},{}]}}), + "ValidationException", + ), + "version" => ( + "UpdateStateMachine", + json!({"stateMachineArn":"machine","versionDescription":"a".repeat(257)}), + "ValidationException", + ), + "encryption_period" => ( + "CreateStateMachine", + json!({"name":"machine","roleArn":"role","definition":"{}","encryptionConfiguration":{"type":"AWS_OWNED_KEY","kmsDataKeyReusePeriodSeconds":59}}), + "ValidationException", + ), + _ => panic!("fixture case"), + }; + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let response = service + .handle(request(action, serde_json::to_vec(&body).unwrap())) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(data(response).await["__type"], code); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[tokio::test] +async fn test_should_accept_exact_byte_bounds_unicode_business_data_and_trace_ascii() { + let (service, handler, mut accepted) = fixture(Mode::Normal, false); + for trace in [ + String::new(), + "Root=value; Parent=value".to_owned(), + "t".repeat(256), + ] { + let body = json!({"stateMachineArn":"a".repeat(256),"name":"a".repeat(80),"input":"null","traceHeader":trace}); + assert_eq!( + service + .handle(request( + "StartExecution", + serde_json::to_vec(&body).unwrap() + )) + .await + .status(), + StatusCode::OK + ); + let (_, request) = accepted.recv().await.unwrap(); + let Request::StartExecution(value) = request else { + panic!("start fixture") + }; + assert_eq!(value.trace_header, Some(trace)); + } + let exact = serde_json::to_string(&"é".repeat(131_071)).unwrap(); + assert_eq!(exact.len(), 262_144); + let body = json!({"stateMachineArn":"machine","input":exact}); + assert_eq!( + service + .handle(request( + "StartExecution", + serde_json::to_vec(&body).unwrap() + )) + .await + .status(), + StatusCode::OK + ); + accepted.recv().await.unwrap(); + let body = json!({"definition":"a".repeat(1024*1024)}); + assert_eq!( + service + .handle(request( + "ValidateStateMachineDefinition", + serde_json::to_vec(&body).unwrap() + )) + .await + .status(), + StatusCode::OK + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 5); +} +#[rstest::rstest] +#[case("é".to_owned())] +#[case("a".repeat(257))] +#[tokio::test] +async fn test_should_reject_non_ascii_or_oversized_trace_header(#[case] trace: String) { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let body = json!({"stateMachineArn":"machine","traceHeader":trace}); + assert_eq!( + service + .handle(request( + "StartExecution", + serde_json::to_vec(&body).unwrap() + )) + .await + .status(), + StatusCode::BAD_REQUEST + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +#[rstest::rstest] +#[case("+2")] +#[case("-0")] +#[case("")] +#[tokio::test] +async fn test_should_reject_non_decimal_content_length(#[case] length: &str) { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let mut request = request("ListStateMachines", b"{}".as_slice()); + request + .headers_mut() + .insert("content-length", HeaderValue::from_str(length).unwrap()); + assert_eq!( + service.handle(request).await.status(), + StatusCode::BAD_REQUEST + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} +struct Frames { + frames: VecDeque>, +} +impl Body for Frames { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Infallible>>> { + Poll::Ready(self.get_mut().frames.pop_front().map(Ok)) + } +} +#[rstest::rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn test_should_reject_all_trailers_before_handler_even_when_metadata_is_oversized( + #[case] oversized: bool, +) { + let (service, handler, _accepted) = fixture(Mode::Normal, false); + let mut trailers = http::HeaderMap::new(); + trailers.insert( + "x-trailer-secret", + HeaderValue::from_str(&"s".repeat(if oversized { 9000 } else { 1 })).unwrap(), + ); + let (parts, _) = request("ListStateMachines", Bytes::new()).into_parts(); + let body = Frames { + frames: VecDeque::from([ + Frame::data(Bytes::from_static(b"{}")), + Frame::trailers(trailers), + ]), + }; + let response = service.handle(http::Request::from_parts(parts, body)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + data(response).await["__type"], + "MalformedHttpRequestException" + ); + assert_eq!(handler.count.load(Ordering::SeqCst), 0); +} diff --git a/crates/rustack-stepfunctions-http/src/transport.rs b/crates/rustack-stepfunctions-http/src/transport.rs new file mode 100644 index 00000000..956b9dd7 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/transport.rs @@ -0,0 +1,79 @@ +//! One allowlist defines typed requests, replies and operation correlation. + +use std::{fmt, io::Write}; + +use rustack_core::json::{JsonLimits, from_slice}; +use rustack_stepfunctions_model::{StepFunctionsOperation, input, output}; + +use crate::HttpError; + +macro_rules! operations { + ($($op:ident => $input:ident, $output:ident;)*) => { + /// Typed boundary requests; DTO contents stay out of diagnostics. + #[non_exhaustive] + pub enum Request {$(#[doc=concat!("Typed `",stringify!($op),"` input.")] $op(Box),)*} + /// Typed replies; only the contained DTO is encoded on the wire. + #[non_exhaustive] + pub enum Response {$(#[doc=concat!("Typed `",stringify!($op),"` output.")] $op(Box),)*} + impl Request { + /// Preflight/decode and bound a supported request before handler entry. + /// + /// # Errors + /// Rejects unsupported operations, invalid/ambiguous/oversized JSON and field bounds. + /// + /// ``` + /// let request=rustack_stepfunctions_http::Request::decode(rustack_stepfunctions_model::StepFunctionsOperation::ListStateMachines,b"{}")?; + /// assert_eq!(request.operation(),rustack_stepfunctions_model::StepFunctionsOperation::ListStateMachines); + /// # Ok::<(),rustack_stepfunctions_http::HttpError>(()) + /// ``` + pub fn decode(operation:StepFunctionsOperation, bytes:&[u8])->Result { + let limits=JsonLimits::new(2*1024*1024,16,4096,1024*1024,128).map_err(HttpError::Json)?; + let bytes=if bytes.is_empty(){b"{}".as_slice()}else{bytes}; + let request=match operation {$(StepFunctionsOperation::$op=>Self::$op(Box::new(from_slice::(bytes,limits).map_err(HttpError::Json)?)),)* _=>return Err(HttpError::UnsupportedOperation)}; + crate::validation::validate(&request)?; Ok(request) + } + /// Exact modeled operation for this variant. + #[must_use] pub fn operation(&self)->StepFunctionsOperation {match self {$(Self::$op(_)=>StepFunctionsOperation::$op,)*}} + } + impl Response { + /// Exact modeled operation for correlation before serialization. + #[must_use] pub fn operation(&self)->StepFunctionsOperation {match self {$(Self::$op(_)=>StepFunctionsOperation::$op,)*}} + pub(crate) fn encode(&self,writer:W)->Result<(),serde_json::Error> {match self {$(Self::$op(value)=>serde_json::to_writer(writer,value),)*}} + } + $(impl From for Response {fn from(value:output::$output)->Self {Self::$op(Box::new(value))}})* + } +} +operations! { + CreateStateMachine=>CreateStateMachineInput,CreateStateMachineOutput; + UpdateStateMachine=>UpdateStateMachineInput,UpdateStateMachineOutput; + DeleteStateMachine=>DeleteStateMachineInput,DeleteStateMachineOutput; + DescribeStateMachine=>DescribeStateMachineInput,DescribeStateMachineOutput; + ListStateMachines=>ListStateMachinesInput,ListStateMachinesOutput; + ValidateStateMachineDefinition=>ValidateStateMachineDefinitionInput,ValidateStateMachineDefinitionOutput; + StartExecution=>StartExecutionInput,StartExecutionOutput; + DescribeExecution=>DescribeExecutionInput,DescribeExecutionOutput; + ListExecutions=>ListExecutionsInput,ListExecutionsOutput; + StopExecution=>StopExecutionInput,StopExecutionOutput; + GetExecutionHistory=>GetExecutionHistoryInput,GetExecutionHistoryOutput; + DescribeStateMachineForExecution=>DescribeStateMachineForExecutionInput,DescribeStateMachineForExecutionOutput; + TagResource=>TagResourceInput,TagResourceOutput; + UntagResource=>UntagResourceInput,UntagResourceOutput; + ListTagsForResource=>ListTagsForResourceInput,ListTagsForResourceOutput; + SendTaskSuccess=>SendTaskSuccessInput,SendTaskSuccessOutput; + SendTaskFailure=>SendTaskFailureInput,SendTaskFailureOutput; + SendTaskHeartbeat=>SendTaskHeartbeatInput,SendTaskHeartbeatOutput; +} +impl fmt::Debug for Request { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Request") + .field("operation", &self.operation()) + .finish_non_exhaustive() + } +} +impl fmt::Debug for Response { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Response") + .field("operation", &self.operation()) + .finish_non_exhaustive() + } +} diff --git a/crates/rustack-stepfunctions-http/src/validation.rs b/crates/rustack-stepfunctions-http/src/validation.rs new file mode 100644 index 00000000..c33d9698 --- /dev/null +++ b/crates/rustack-stepfunctions-http/src/validation.rs @@ -0,0 +1,228 @@ +//! Wire byte/count bounds. Core retains ARN, role, ASL and capability semantics. + +use rustack_core::json::{JsonLimits, validate as json}; +use rustack_stepfunctions_model::{ + StepFunctionsErrorCode as Code, + input::{CreateStateMachineInput, UpdateStateMachineInput}, + types::{EncryptionConfiguration, LoggingConfiguration, Tag}, +}; + +use crate::{HttpError, Request}; + +pub(crate) fn validate(request: &Request) -> Result<(), HttpError> { + resource(request)?; + details(request) +} +fn resource(request: &Request) -> Result<(), HttpError> { + let arn = match request { + Request::CreateStateMachine(value) => Some(value.role_arn.as_str()), + Request::UpdateStateMachine(value) => Some(value.state_machine_arn.as_str()), + Request::DeleteStateMachine(value) => Some(value.state_machine_arn.as_str()), + Request::DescribeStateMachine(value) => Some(value.state_machine_arn.as_str()), + Request::StartExecution(value) => Some(value.state_machine_arn.as_str()), + Request::DescribeExecution(value) => Some(value.execution_arn.as_str()), + Request::StopExecution(value) => Some(value.execution_arn.as_str()), + Request::GetExecutionHistory(value) => Some(value.execution_arn.as_str()), + Request::DescribeStateMachineForExecution(value) => Some(value.execution_arn.as_str()), + Request::TagResource(value) => Some(value.resource_arn.as_str()), + Request::UntagResource(value) => Some(value.resource_arn.as_str()), + Request::ListTagsForResource(value) => Some(value.resource_arn.as_str()), + _ => None, + }; + if let Some(arn) = arn { + identifier(arn, 256, Code::InvalidArn)?; + } + Ok(()) +} +fn details(request: &Request) -> Result<(), HttpError> { + match request { + Request::CreateStateMachine(value) => create(value), + Request::UpdateStateMachine(value) => update(value), + Request::ValidateStateMachineDefinition(value) => { + text(&value.definition, 1024 * 1024, Code::ValidationException)?; + page(value.max_results, None, 100) + } + Request::ListStateMachines(value) => { + page(value.max_results, value.next_token.as_deref(), 1000) + } + Request::ListExecutions(value) => { + optional_id(value.state_machine_arn.as_deref(), 256, Code::InvalidArn)?; + optional_id(value.map_run_arn.as_deref(), 256, Code::InvalidArn)?; + page(value.max_results, value.next_token.as_deref(), 1000) + } + Request::GetExecutionHistory(value) => { + page(value.max_results, value.next_token.as_deref(), 1000) + } + Request::StartExecution(value) => { + if let Some(value) = &value.name { + name(value)?; + } + if let Some(value) = &value.input { + payload(value, Code::InvalidExecutionInput)?; + } + ascii_text(value.trace_header.as_deref(), 256) + } + Request::StopExecution(value) => failure(value.error.as_deref(), value.cause.as_deref()), + Request::TagResource(value) => tags(&value.tags), + Request::UntagResource(value) => { + if value.tag_keys.len() > 50 { + return Err(HttpError::Field(Code::TooManyTags)); + } + for key in &value.tag_keys { + nonempty(key, 128, Code::ValidationException)?; + } + Ok(()) + } + Request::SendTaskSuccess(value) => { + identifier(&value.task_token, 2048, Code::InvalidToken)?; + payload(&value.output, Code::InvalidOutput) + } + Request::SendTaskFailure(value) => { + identifier(&value.task_token, 2048, Code::InvalidToken)?; + failure(value.error.as_deref(), value.cause.as_deref()) + } + Request::SendTaskHeartbeat(value) => { + identifier(&value.task_token, 2048, Code::InvalidToken) + } + _ => Ok(()), + } +} +fn create(value: &CreateStateMachineInput) -> Result<(), HttpError> { + text(&value.definition, 1024 * 1024, Code::ValidationException)?; + name(&value.name)?; + tags(&value.tags)?; + metadata( + value.logging_configuration.as_ref(), + value.encryption_configuration.as_ref(), + value.version_description.as_deref(), + ) +} +fn update(value: &UpdateStateMachineInput) -> Result<(), HttpError> { + if let Some(value) = &value.definition { + text(value, 1024 * 1024, Code::ValidationException)?; + } + optional_id(value.role_arn.as_deref(), 256, Code::InvalidArn)?; + metadata( + value.logging_configuration.as_ref(), + value.encryption_configuration.as_ref(), + value.version_description.as_deref(), + ) +} +fn metadata( + logging: Option<&LoggingConfiguration>, + encryption: Option<&EncryptionConfiguration>, + version: Option<&str>, +) -> Result<(), HttpError> { + if let Some(logging) = logging { + if logging.destinations.len() > 1 { + return Err(HttpError::Field(Code::ValidationException)); + } + for destination in &logging.destinations { + if let Some(group) = &destination.cloud_watch_logs_log_group { + optional_id( + group.log_group_arn.as_deref(), + 256, + Code::ValidationException, + )?; + } + } + } + if let Some(encryption) = encryption { + optional_id( + encryption.kms_key_id.as_deref(), + 256, + Code::ValidationException, + )?; + if encryption + .kms_data_key_reuse_period_seconds + .is_some_and(|value| !(60..=900).contains(&value)) + { + return Err(HttpError::Field(Code::ValidationException)); + } + } + if let Some(version) = version { + text(version, 256, Code::ValidationException)?; + } + Ok(()) +} +fn page(maximum: Option, token: Option<&str>, cap: i32) -> Result<(), HttpError> { + if maximum.is_some_and(|value| !(0..=cap).contains(&value)) { + return Err(HttpError::Field(Code::ValidationException)); + } + optional_id(token, 2048, Code::InvalidToken) +} +fn tags(values: &[Tag]) -> Result<(), HttpError> { + if values.len() > 50 { + return Err(HttpError::Field(Code::TooManyTags)); + } + for value in values { + let key = value + .key + .as_deref() + .ok_or(HttpError::Field(Code::ValidationException))?; + nonempty(key, 128, Code::ValidationException)?; + if let Some(value) = &value.value { + text(value, 256, Code::ValidationException)?; + } + } + Ok(()) +} +fn name(value: &str) -> Result<(), HttpError> { + nonempty(value, 80, Code::InvalidName)?; + if !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte)) + { + return Err(HttpError::Field(Code::InvalidName)); + } + Ok(()) +} +fn failure(error: Option<&str>, cause: Option<&str>) -> Result<(), HttpError> { + if let Some(value) = error { + text(value, 256, Code::ValidationException)?; + } + if let Some(value) = cause { + text(value, 32768, Code::ValidationException)?; + } + Ok(()) +} +fn optional_id(value: Option<&str>, max: usize, code: Code) -> Result<(), HttpError> { + if let Some(value) = value { + identifier(value, max, code)?; + } + Ok(()) +} +fn identifier(value: &str, max: usize, code: Code) -> Result<(), HttpError> { + nonempty(value, max, code)?; + if !value.bytes().all(|byte| byte.is_ascii_graphic()) { + return Err(HttpError::Field(code)); + } + Ok(()) +} +fn nonempty(value: &str, max: usize, code: Code) -> Result<(), HttpError> { + if value.is_empty() { + Err(HttpError::Field(code)) + } else { + text(value, max, code) + } +} +fn text(value: &str, max: usize, code: Code) -> Result<(), HttpError> { + if value.len() > max { + Err(HttpError::Field(code)) + } else { + Ok(()) + } +} +fn payload(value: &str, code: Code) -> Result<(), HttpError> { + nonempty(value, 262_144, code)?; + let limits = JsonLimits::new(262_144, 64, 65_536, 262_144, 262_144).map_err(HttpError::Json)?; + json(value.as_bytes(), limits).map_err(|_| HttpError::Field(code)) +} + +fn ascii_text(value: Option<&str>, maximum: usize) -> Result<(), HttpError> { + if value.is_some_and(|value| value.len() > maximum || !value.is_ascii()) { + Err(HttpError::Field(Code::ValidationException)) + } else { + Ok(()) + } +} diff --git a/crates/rustack-stepfunctions-http/tests/network.rs b/crates/rustack-stepfunctions-http/tests/network.rs new file mode 100644 index 00000000..d694a0e3 --- /dev/null +++ b/crates/rustack-stepfunctions-http/tests/network.rs @@ -0,0 +1,281 @@ +//! Actual HTTP framing and ordinary SDK qualification of the transport only. +#![forbid(unsafe_code)] + +use std::{ + net::SocketAddr, + path::Path, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use async_trait::async_trait; +use hyper::server::conn::http1; +use hyper_util::rt::TokioIo; +use rustack_auth::StaticCredentialProvider; +use rustack_stepfunctions_http::{ + Request, RequestContext, RequestScope, Response, StepFunctionsHandler, StepFunctionsHttpConfig, + StepFunctionsHttpService, +}; +use rustack_stepfunctions_model::{StepFunctionsError, StepFunctionsErrorCode, output}; +use serde_json::Value; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + process::Command, + sync::watch, + task::{JoinHandle, JoinSet}, + time::timeout, +}; + +#[derive(Default)] +struct Fixture { + calls: AtomicUsize, + authenticated: AtomicUsize, +} +#[async_trait] +impl StepFunctionsHandler for Fixture { + async fn handle( + &self, + context: RequestContext, + request: Request, + ) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + if context.authenticated() { + self.authenticated.fetch_add(1, Ordering::SeqCst); + } + match request { + Request::ListStateMachines(_) => Ok(output::ListStateMachinesOutput::default().into()), + Request::StartExecution(value) => { + assert_eq!(value.input.as_deref(), Some(" {\"n\":7}")); + assert_eq!(value.trace_header.as_deref(), Some("")); + let mut response = output::StartExecutionOutput::default(); + response.execution_arn = + "arn:aws:states:us-east-1:000000000000:execution:wire:execution".into(); + response.start_date = chrono::DateTime::from_timestamp_millis(-1250).unwrap(); + Ok(response.into()) + } + Request::DescribeExecution(_) => Err(StepFunctionsError::with_message( + StepFunctionsErrorCode::ExecutionDoesNotExist, + "fixture missing", + )), + _ => Err(StepFunctionsError::new( + StepFunctionsErrorCode::UnsupportedOperation, + )), + } + } +} +struct Server { + address: SocketAddr, + stop: watch::Sender, + task: Option>>, +} +impl Server { + async fn start(handler: Arc, strict: bool) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let scope = RequestScope::new("aws".into(), "us-east-1".into(), "000000000000".into())?; + let credentials = Arc::new(StaticCredentialProvider::new(vec![( + "local-key".into(), + "local-secret".into(), + )])); + let service = StepFunctionsHttpService::new( + handler, + StepFunctionsHttpConfig::new(scope, !strict, Some(credentials))?, + ); + let (stop, mut stopped) = watch::channel(false); + let task = tokio::spawn(async move { + let mut connections = JoinSet::new(); + let primary = loop { + tokio::select! { + biased; + _=stopped.changed()=>break Ok(()), + joined=connections.join_next(),if !connections.is_empty()=>match joined { + Some(Ok(Ok(()))) | None=>{},Some(Ok(Err(error)))=>break Err(error),Some(Err(error))=>break Err(error.into()), + }, + accepted=listener.accept(),if connections.len()<8=>match accepted { + Ok((socket,_))=>{let service=service.clone();connections.spawn(async move { + timeout(Duration::from_secs(15),http1::Builder::new().serve_connection(TokioIo::new(socket),service)).await.context("connection deadline")?.context("HTTP connection") + });}, + Err(error)=>break Err(error.into()), + }, + } + }; + connections.abort_all(); + let mut cleanup = Ok(()); + while let Some(joined) = connections.join_next().await { + match joined { + Ok(Ok(())) => {} + Ok(Err(error)) => cleanup = Err(error), + Err(error) if error.is_cancelled() => {} + Err(error) => cleanup = Err(error.into()), + } + } + primary.and(cleanup) + }); + Ok(Self { + address, + stop, + task: Some(task), + }) + } + async fn shutdown(&mut self) -> Result<()> { + self.stop.send_replace(true); + let Some(mut task) = self.task.take() else { + return Ok(()); + }; + match timeout(Duration::from_secs(10), &mut task).await { + Ok(joined) => joined?, + Err(error) => { + // Keep the signaled root and its connection JoinSet alive. + // A waiter deadline is not permission to lose drain ownership. + self.task = Some(task); + Err(error.into()) + } + } + } +} +impl Drop for Server { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +#[tokio::test(start_paused = true)] +async fn test_should_retain_server_join_ownership_after_cleanup_waiter_expiry() -> Result<()> { + let (stop, mut stopped) = watch::channel(false); + let (release, held) = tokio::sync::oneshot::channel(); + let joined = Arc::new(AtomicUsize::new(0)); + let completed = joined.clone(); + let task = tokio::spawn(async move { + let mut connections = JoinSet::new(); + connections.spawn(std::future::pending::<()>()); + let _ = stopped.changed().await; + held.await.context("release drain")?; + connections.abort_all(); + while connections.join_next().await.is_some() {} + completed.store(1, Ordering::SeqCst); + Ok(()) + }); + let id = task.id(); + let mut server = Server { + address: "127.0.0.1:1".parse()?, + stop, + task: Some(task), + }; + ensure!( + server.shutdown().await.is_err(), + "held drain did not expire" + ); + ensure!( + server.task.as_ref().context("retained root")?.id() == id, + "root changed after timeout" + ); + ensure!( + joined.load(Ordering::SeqCst) == 0, + "drain completed before release" + ); + release + .send(()) + .map_err(|()| anyhow::anyhow!("drain receiver lost"))?; + server.shutdown().await?; + ensure!( + joined.load(Ordering::SeqCst) == 1, + "connection handles were not joined" + ); + Ok(()) +} + +#[tokio::test] +async fn test_should_reject_actual_chunked_trailers_without_dispatching() -> Result<()> { + let handler = Arc::new(Fixture::default()); + let mut server = Server::start(handler.clone(), false).await?; + let result = async { + let mut socket = TcpStream::connect(server.address).await?; + let request = format!( + "POST / HTTP/1.1\r\nHost: localhost\r\nContent-Type: \ + application/x-amz-json-1.0\r\nX-Amz-Target: \ + AWSStepFunctions.ListStateMachines\r\nTransfer-Encoding: chunked\r\nTrailer: \ + x-opaque\r\nConnection: close\r\n\r\n2\r\n{{}}\r\n0\r\nx-opaque: {}\r\n\r\n", + "s".repeat(9000) + ); + timeout(Duration::from_secs(5), socket.write_all(request.as_bytes())).await??; + let mut response = Vec::new(); + timeout( + Duration::from_secs(5), + socket.take(65536).read_to_end(&mut response), + ) + .await??; + let split = response + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .context("HTTP response headers")?; + ensure!( + response.starts_with(b"HTTP/1.1 400"), + "trailer did not receive400" + ); + let value: Value = serde_json::from_slice(response.get(split + 4..).context("body")?)?; + ensure!( + value["__type"] == "MalformedHttpRequestException", + "wrong trailer error" + ); + ensure!( + handler.calls.load(Ordering::SeqCst) == 0, + "trailers reached handler" + ); + anyhow::Ok(()) + } + .await; + let cleanup = server.shutdown().await; + result.and(cleanup) +} +#[tokio::test] +#[ignore = "requires uv and pinned ordinary boto3 fixture"] +async fn test_should_exchange_signed_boto3_requests_with_typed_transport() -> Result<()> { + let handler = Arc::new(Fixture::default()); + let mut server = Server::start(handler.clone(), true).await?; + let result = async { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut command = Command::new("uv"); + command + .args([ + "run", + "--no-project", + "--python", + "3.13", + "--with-requirements", + ]) + .arg(root.join("tests/workflows/requirements.txt")) + .arg("python") + .arg(root.join("tests/workflows/stepfunctions_http_sdk.py")) + .arg(format!("http://{}", server.address)) + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("NO_PROXY", "127.0.0.1,localhost") + .kill_on_drop(true); + let mut child = command.spawn().context("start pinned SDK")?; + let status = match timeout(Duration::from_secs(60), child.wait()).await { + Ok(status) => status?, + Err(error) => { + let _ = child.kill().await; + let _ = child.wait().await; + return Err(error.into()); + } + }; + ensure!(status.success(), "SDK fixture failed: {status}"); + ensure!( + handler.calls.load(Ordering::SeqCst) == 3, + "wrong dispatch count" + ); + ensure!( + handler.authenticated.load(Ordering::SeqCst) == 3, + "SDK calls not authenticated" + ); + anyhow::Ok(()) + } + .await; + let cleanup = server.shutdown().await; + result.and(cleanup) +} diff --git a/crates/rustack-stepfunctions-model/src/error.rs b/crates/rustack-stepfunctions-model/src/error.rs index 314e9117..55713064 100644 --- a/crates/rustack-stepfunctions-model/src/error.rs +++ b/crates/rustack-stepfunctions-model/src/error.rs @@ -164,11 +164,10 @@ impl StepFunctionsErrorCode { | Self::TaskDoesNotExist | Self::TaskTimedOut | Self::TooManyTags - | Self::UnknownOperationException | Self::UnsupportedOperation | Self::ValidationException => http::StatusCode::BAD_REQUEST, Self::ServiceQuotaExceededException => http::StatusCode::INTERNAL_SERVER_ERROR, - Self::ResourceNotFound => http::StatusCode::NOT_FOUND, + Self::ResourceNotFound | Self::UnknownOperationException => http::StatusCode::NOT_FOUND, Self::ConflictException => http::StatusCode::CONFLICT, } } diff --git a/specs/index.md b/specs/index.md index e5e29225..e0f0b6ea 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. HTTP routing, registry, ASL engine and the remaining P2–P5 requirements remain pending. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Main-service registration, registry, ASL engine and the remaining P2–P5 requirements remain pending. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-10-api-design.md b/specs/rustack-workflows-10-api-design.md index aff64460..5874e7d1 100644 --- a/specs/rustack-workflows-10-api-design.md +++ b/specs/rustack-workflows-10-api-design.md @@ -48,6 +48,20 @@ The JSON adapter uses serde_json raw_value to preserve decimal numeric lexemes w └───────────────────────────────┘ └────────────────────────────────┘ ``` +### 1.2 Typed HTTP boundary before registry wiring + +`rustack-stepfunctions-http` owns a concrete Hyper service, an object-safe async StepFunctionsHandler port, typed Request/Response enums for the15 M1 actions and three callback actions, immutable RequestScope and per-call RequestContext. Native async syntax with async-trait is justified by Arc; no default/echo/unimplemented production handler is supplied. Model presence and callback transport types do not advertise core support. Register the service before S3 only when the real core adapter lands; keep feature/capability publication coupled to that registration. + +Claim any exact AWSStepFunctions target family, including unknown actions, duplicate occurrences and a missing action suffix, independent of method/path; never forward those to S3. Require POST at root without query, one bounded ASCII target/content-type, JSON1.0 media type (optionally UTF-8 charset), no compression, and unambiguous Content-Length/Transfer-Encoding. Bound the whole header map to64 entries/32KiB, each value≤8KiB; target/content-type≤256 bytes. Unknown actions return UnknownOperationException/404 per the current [SFN common-error contract](https://docs.aws.amazon.com/step-functions/latest/apireference/CommonErrors.html); known actions outside the18-action transport profile return UnsupportedOperation/400. This updates the initial custom unknown-operation status in generator configuration. These checks are local rejection policy, not a measured full AWS error-precedence oracle. + +Collect through the shared source-level BodyBudget capped at2MiB, with a configured total request deadline covering collection/authentication/dispatch/encoding. Reject every trailer frame before dispatch, including unknown and oversized trailer fields; trailers must not bypass the initial header limits. Authenticate the original bytes before treating a zero-byte body as `{}`. Explicit development mode skips verification; strict construction without a provider fails. Strict requests reuse ordinary SigV4 payload verification, require signing service states and the configured region, and bind x-amz-target in SignedHeaders. Scope partition/region/account comes only from validated operator configuration, never unsigned request fields. Authentication failures emit a static AccessDeniedException/403 while preserving typed sources in redacted diagnostics. This remains the gateway's local trust profile, not an IAM evaluator. + +Preflight JSON (depth16,4,096 aggregate members,128-byte keys and1MiB maximum strings) before typed DTO construction. Strict generated DTOs reject unknown/duplicate fields. Validate operation-specific string/collection byte bounds from §4 before calling the handler; traceHeader accepts0–256 ASCII bytes, including empty text and spaces, rather than using the nonempty identifier grammar. Business JSON input/output text is independently bounded/validated, while definition strings remain compiler-owned so Validate can return FAIL diagnostics. Domain ARN/role/profile/ASL semantics remain core-owned and must validate before actor commands. Request/response/context/body/config Debug never expose definitions, input/output, task tokens, headers or credentials. + +Handler response variants must match the request operation. Encode only the selected DTO, preserving numeric timestamps, with a bounded8MiB writer before returning200; failure/oversize/mismatch returns static InternalFailure/500 without replaying a mutation. History pagination later caps each encoded page at4MiB and may return fewer than maxResults with a consistent continuation token; individual events are never split/truncated. Success/error responses carry JSON1.0 content type, x-amzn-requestid and server headers; errors use short __type/message plus x-amzn-errortype. Handler messages are bounded to1KiB; invalid error status/message becomes InternalFailure. IO/JSON/auth sources are retained but never copied to public diagnostics. Final monotonic deadline checks prevent a late result becoming success. Dropping/timing out the HTTP waiter does not authorize rollback/cancellation of accepted core commands; the actor owns execution lifetime. + +The embedding gateway owns bounded connection/request admission; this service spawns no per-request work. Tests cover all typed action mappings, empty/normal JSON, header ambiguity and nonfallthrough claims, body limits/slow progress, duplicate/unknown keys, operation-specific bounds, signature/service/region/target binding, static error envelopes, response correlation/byte limits, and deadline/abandoned caller behavior. The network fixture owns all connection tasks in a JoinSet, signals stop on shutdown, and retains its root JoinHandle after a cleanup-waiter timeout so the same join can resume. An ordinary SDK against a controlled typed handler can qualify this transport only; it cannot satisfy registry, ASL or Lambda-execution acceptance. + ## 2. API coverage | Milestone | Operations | Required behavior | @@ -120,6 +134,8 @@ Append history, update read views and settle terminal state atomically within th Use random opaque pagination handles backed by a bounded server-side registry. Bind scope, operation, filters, ordering, includeExecutionData and a read high-water mark. TTL is 24h. Stable execution sorting uses start time and ARN; history reversal uses event ID. Newly appended records do not change the cursor's captured boundary. History maxResults defaults to 100 and caps at 1000; 0 selects the default. Other list operations follow their Smithy limits. Expired, modified, cross-scope or argument-mismatched tokens return InvalidToken. Reject capacity exhaustion rather than evict live tokens. +Cap each complete encoded history page at4MiB, including metadata and escaped JSON string contents. Return fewer than maxResults when needed, with a continuation token at the next event; never split or truncate an event. This leaves headroom below the transport's8MiB response cap. + Keep at most 25,000 events, also constrained by per-execution/global byte budgets. Reserve one event slot and byte allowance for termination. Imminent exhaustion fails the execution with local `States.HistoryLimitExceeded`; never truncate history and still report success. This locally identified error is advertised as a limitation. Clean up expired terminal histories only; preserve independent name tombstones. ## 7. Rust engineering contract diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 189f5fd9..3fc972d2 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -168,3 +168,9 @@ Scripted restart cases require unknown404 plus blocked relay, later positive com `make workflows-model-check` verifies the pinned312-shape/37-operation source. `make test-stepfunctions-model` compares formatted regeneration byte-for-byte, exercises five generator policy cases and34 actual model cases plus a runnable macro doctest, then applies ordinary tool and strict production-model Clippy. Cover required/optional/omitted/null, negative and fractional epochs, scientific notation including huge zero/underflow exponents, positive overflow, chrono boundaries, submillisecond truncation, hostile lexical size, required collections, unknown fields and diagnostic redaction of task tokens/nested data/error sources. Verify format precedence and unchanged other-protocol timestamp bindings, both serde-disabled JSON protocols and strict no-serde Debug ownership. No generated lint suppression or ignored doctest may satisfy a check. Run `make audit-codegen deny-codegen` for the excluded developer-tool graph as well as the workspace gates. Model presence and known operation names do not establish endpoint, execution, authorization, ASL or Lambda integration acceptance. The user-prioritized API/compiler/engine work proceeds before final Docker-image qualification, whose original acceptance requirements remain required. + +## Step Functions typed HTTP qualification + +`make test-stepfunctions-http` exercises all18 typed action mappings, operation-correlated DTO encoding, numeric dates, source-level2MiB body limits, total deadlines, redaction and strict production-library Clippy. Reject duplicate/unknown JSON fields, header ambiguity, signed service/region/action mismatches, nondecimal Content-Length and every trailer frame. Cover exact byte bounds, Unicode business JSON, empty/space-containing ASCII trace headers, oversized/invalid handler errors and8MiB response overflow without mutation replay. Cancellation cases verify accepted commands survive HTTP waiter loss; these controlled-handler cases do not establish execution-actor ownership. + +The actual Hyper network fixture rejects a chunked request carrying an oversized trailer without dispatch. A paused regression expires its cleanup waiter and then resumes the same root task's join, proving connection-task ownership is retained. `make test-stepfunctions-http-sdk` runs pinned ordinary boto3 with strict signed local requests against the controlled typed handler: StartExecution preserves raw input and a numeric negative fractional date, ListStateMachines decodes normally, DescribeExecution returns a modeled exception and CreateActivity reports unsupported. The fixture bounds connection admission and joins its server/connection tasks. These are transport-only proofs; the main program does not advertise/register Step Functions until a real core adapter exists. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 1af47b66..d8ede8ef 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -158,8 +158,9 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3/1.4 startup probes | Implemented/reviewed; no readiness grant | §4.7/D37 validate exact user bridge/none selections, bind full IDs, and execute the pinned Rust probe under retained EnvironmentManager ownership. Private cgroup namespaces and process root-membership checks cover v1/v2 parser semantics; exact kernel limits/flags, scoped proof matching, caller abandonment, independent cleanup and final deadline/live-input checks are enforced. W92–W94 closed;57 focused cases pass. Workspace2,370/411 gated, boundary954 units+217 docs (1,171 total)/17 gated; standard/strict/audit/deny pass. Each arm64 VZ and amd64/QEMU cgroup-v2 daemon passes17 distinct actual probe/SDK/pool/wire/runtime cases; scoped resources and the two dedicated probe bridges are removed. Actual v1/native-x86, historical effects, pulls, endpoint/config/capability coordinator acceptance remain open. | | 1.3/1.4 historical container effects | Implemented/reviewed; other mutation classes remain | §4.8/D38 add bounded durable per-container intents before creation and known IDs before deletion. The restart pass preserves unknown absence, blocks unresolved relays, validates exact identities and resumes after cancellation. W95–W96 are fixed and independently rechecked.24 new scripted/unit cases pass, including create-before-effect and durable-erase reservation barriers. Workspace2,394/412 gated; boundary978 units+218 docs (1,196 total)/18 gated; standard/strict/audit/deny gates pass. Each arm64 VZ and amd64/QEMU daemon passes11 distinct actual recovery/pool/SDK/wire cases; scoped container/private-alias queries are empty afterward. Actual directory reacquisition and historical pair retirement pass on both. This does not cover historical image aliases or enable production startup. | | 2.1 generated model/timestamps | Implemented/reviewed; HTTP/registry still pending | API§1.1/D40 generates37 official operations from312 pinned shapes with exact numeric required/optional epoch dates and strict DTO policy.5 generator cases,34 model cases,1 doctest, regeneration consistency and strict model/tool checks pass. W97–W100 are fixed/rechecked. Final current-worktree workspace2,447/414 gated, including the retained deferred image checkpoint; build/test/fmt/Clippy/rustdoc, root475/tool29-crate audit and both deny graphs pass. This is protocol/model evidence, not API/engine or real-image workflow acceptance. | +| 2.1 typed HTTP transport | Implemented/reviewed; real core/registration pending | API§1.2/D41 implements18 typed ports, bounded envelopes/body/response, strict SigV4 service/region/target binding, exact DTO dates and accepted-command waiter isolation.56 unit tests, two nongated network tests, eight doctests, strict HTTP Clippy and four real signed boto3 calls pass. W101–W104 are fixed and independently rechecked. Current-worktree workspace2,505/415 gated, standard build/test/fmt/Clippy/rustdoc, audit and deny pass. No production fallback handler, main registration or engine acceptance is claimed. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. -Current execution priority: complete the Step Functions HTTP boundary, service registration, typed registry and ASL compiler/engine under the user's explicit reorder above. Keep Docker-image work deferred until the subsequent workflow implementation is complete; preserve its current worktree checkpoint and all original final image/platform gates. +Current execution priority: build the typed registry and ASL compiler/engine, then couple service registration to the real core adapter under the user's explicit reorder above. Keep Docker-image work deferred until the subsequent workflow implementation is complete; preserve its current worktree checkpoint and all original final image/platform gates. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index fc6c2a43..0fbf7ddd 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -477,6 +477,19 @@ API§1.1/D40 lands the official model and protocol-aware numeric serde generatio Final focused evidence:5 generator tests,34 generated-model cases and1 executed doctest; formatted regeneration and strict tool/model checks pass. Final workspace build/test/fmt/Clippy/rustdoc pass with2,447 tests/414 gated, including retained deferred Docker changes. Audit scans475 workspace/29 standalone-tool dependencies without vulnerabilities; both deny graphs pass with existing unmatched/duplicate-policy notices and no new ignores. The developer tool uses bounded ordinary-file Tokio IO, validates Smithy2.0 and no longer retains unused trait/phase data. This does not close P2, M1 or final real-image/platform requirements. +## P2 typed HTTP independent review + +The production transport implements API§1.2/D41 without a default handler or premature gateway registration. The independent read-only reviewer rechecked all four findings and reported no remaining in-slice findings. + +| ID | Priority / source | Closed correction | Verification | +|---|---|---|---| +| W101 | P2; `crates/rustack-stepfunctions-http/src/body.rs` | Reject every trailer frame through a local body adapter before shared collection/typed dispatch | Small/oversized generic trailer frames and an actual chunked Hyper request | +| W102 | P2; `crates/rustack-stepfunctions-http/src/validation.rs` | Give traceHeader its0–256 ASCII text contract, including empty/spaces | Exact limits plus Unicode/oversize rejection; ordinary SDK sends empty trace | +| W103 | P3; `crates/rustack-stepfunctions-http/src/protocol.rs` | Validate nonempty ASCII decimal Content-Length before numeric conversion | `+2`, `-0` and empty values reject before handler | +| W104 | P3; `crates/rustack-stepfunctions-http/tests/network.rs` | Retain the same server root JoinHandle after a cleanup waiter expires; resume joining rather than aborting ownership | Paused expiry/resume test verifies task identity and joined child | + +Focused evidence:56 unit tests, two nongated network tests and eight doctests pass; strict production HTTP Clippy passes. The separately gated real signed boto3 fixture passes four calls against a controlled typed handler. This qualifies transport only; registry, compiler, execution engine and real Lambda/image workflow acceptance remain required. Final current-worktree standard build/test/fmt/Clippy/rustdoc pass with2,505 tests/415 gated (including the deferred Docker checkpoint). Workspace476/tool29 dependency audits and both deny graphs pass. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 0e2e6b46..74a476dd 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -357,3 +357,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: SDK wire dates are numeric and exact at the promised precision, including negative/exponent/range edges, while other protocols retain their bindings. Oversized exponents are classified with bounded work. Serde-disabled output gets neither duplicate Debug nor unused codec modules. - **Pinned by**: [API §1.1](./rustack-workflows-10-api-design.md#11-executable-timestampmodel-generation-contract), [verification](./rustack-workflows-70-verification-plan.md), [implementation plan](./rustack-workflows-90-impl-plan.md). - **Date**: 2026-10-26. The model reuses workspace serde/chrono/typed-builder; the excluded developer generator adds the already-workspace [Tokio1.54.1](https://docs.rs/tokio/1.54.1/tokio/attr.main.html) with explicit local-IO features and independent audit/deny. No Step Functions endpoint or engine is advertised by the model alone. + +## D41 — Terminate HTTP ambiguity before typed registry dispatch + +- **Context**: the new workflow service needs AWS JSON1.0 routing, scope/authentication and bounded typed IO before the registry/ASL engine can consume it. +- **Alternatives considered**: raw-JSON handler ports repeat parsing/limits; a default handler creates misleading service availability; suffix-based routing can leak unknown actions to S3; serializing an unchecked handler response can return a wrong operation or unbounded history. +- **Decision**: expose typed18-action transport ports with explicit validated scope/auth configuration, source-level budgets and operation-correlated bounded responses. Couple main-program registration to the real core adapter. Preserve accepted command ownership beyond HTTP cancellation. +- **Why**: protocol qualification is independently testable without claiming workflow execution; core keeps domain/compiler semantics and cleanup ownership. Encoded page budgets prevent successful responses from relying on unbounded allocation. +- **Pinned by**: [API §1.2](./rustack-workflows-10-api-design.md#12-typed-http-boundary-before-registry-wiring), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Reuse existing HTTP/auth/JSON/Tokio/async-trait dependencies. Unknown-action404 follows the current official common-error reference; no AWS error-precedence execution is claimed. diff --git a/tests/workflows/stepfunctions_http_sdk.py b/tests/workflows/stepfunctions_http_sdk.py new file mode 100644 index 00000000..63691b20 --- /dev/null +++ b/tests/workflows/stepfunctions_http_sdk.py @@ -0,0 +1,45 @@ +"""Ordinary boto3 against a controlled typed HTTP handler, not an ASL engine.""" + +import json +import sys +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +client = boto3.client( + "stepfunctions", + endpoint_url=sys.argv[1], + region_name="us-east-1", + aws_access_key_id="local-key", + aws_secret_access_key="local-secret", + config=Config(signature_version="v4", retries={"max_attempts": 0}, connect_timeout=3, read_timeout=5), +) +try: + started = client.start_execution( + stateMachineArn="arn:aws:states:us-east-1:000000000000:stateMachine:wire", + name="wire", + input=' {"n":7}', + traceHeader="", + ) + assert started["executionArn"].endswith(":execution:wire:execution") + assert started["startDate"].timestamp() == -1.25 + uuid.UUID(started["ResponseMetadata"]["RequestId"]) + listed = client.list_state_machines() + assert listed["stateMachines"] == [] + try: + client.describe_execution(executionArn=started["executionArn"]) + raise AssertionError("expected typed fixture rejection") + except ClientError as error: + assert error.response["Error"]["Code"] == "ExecutionDoesNotExist" + assert error.response["Error"]["Message"] == "fixture missing" + try: + client.create_activity(name="outside-profile") + raise AssertionError("expected unsupported-operation rejection") + except ClientError as error: + assert error.response["Error"]["Code"] == "UnsupportedOperation" + assert error.response["ResponseMetadata"]["HTTPStatusCode"] == 400 + print(json.dumps({"transportOnly": True, "signedCalls": 4, "epochSeconds": -1.25})) +finally: + client.close() From 460dc8947e638c21963b8a41eec1f87a6403aa8e Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 06:52:06 -0700 Subject: [PATCH 034/106] phase 2: compile bounded ASL paths and immutable payload semantics Implement ASL section 2.1.1 and D42: winnow path IR, missing/single/selection results, bounded materialization, separate internal context and transactional result writes. Independent review closes W105-W107, including borrowed deep-tree rejection and numeric normalization headroom. 75 unit tests, 12 doctests, strict core checks and standard workspace gates pass (2580 tests, 415 gated in the current worktree). Audit/deny currently fail newly refreshed existing zstd/Squib advisories, explicitly tracked as B06 without ignores. Compiler, registry, engine and full M1 acceptance remain required; deferred Docker changes remain outside this commit. --- Cargo.lock | 14 + Cargo.toml | 1 + Makefile | 7 + crates/rustack-stepfunctions-core/Cargo.toml | 23 + .../rustack-stepfunctions-core/src/error.rs | 45 ++ crates/rustack-stepfunctions-core/src/lib.rs | 15 + crates/rustack-stepfunctions-core/src/path.rs | 479 ++++++++++++++++++ .../src/path/tests.rs | 342 +++++++++++++ .../rustack-stepfunctions-core/src/payload.rs | 292 +++++++++++ .../src/payload/tests.rs | 157 ++++++ specs/index.md | 2 +- specs/rustack-workflows-30-asl-design.md | 30 ++ .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 1 + specs/rustack-workflows-93-review.md | 15 +- specs/rustack-workflows-99-decisions.md | 9 + 16 files changed, 1436 insertions(+), 2 deletions(-) create mode 100644 crates/rustack-stepfunctions-core/Cargo.toml create mode 100644 crates/rustack-stepfunctions-core/src/error.rs create mode 100644 crates/rustack-stepfunctions-core/src/lib.rs create mode 100644 crates/rustack-stepfunctions-core/src/path.rs create mode 100644 crates/rustack-stepfunctions-core/src/path/tests.rs create mode 100644 crates/rustack-stepfunctions-core/src/payload.rs create mode 100644 crates/rustack-stepfunctions-core/src/payload/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 2668736e..ca410fd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4190,6 +4190,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustack-stepfunctions-core" +version = "0.9.0" +dependencies = [ + "bytes", + "proptest", + "rstest", + "rustack-core", + "serde", + "serde_json", + "thiserror", + "winnow", +] + [[package]] name = "rustack-stepfunctions-http" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index fee7477d..1f948cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ rustack-lambda-core = { version = "0.11.0", path = "crates/rustack-lambda-core" rustack-lambda-runtime = { version = "0.9.0", path = "crates/rustack-lambda-runtime" } rustack-stepfunctions-model = { version = "0.9.0", path = "crates/rustack-stepfunctions-model" } rustack-stepfunctions-http = { version = "0.9.0", path = "crates/rustack-stepfunctions-http" } +rustack-stepfunctions-core = { version = "0.9.0", path = "crates/rustack-stepfunctions-core" } rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } rustack-logs-http = { version = "0.11.0", path = "crates/rustack-logs-http" } diff --git a/Makefile b/Makefile index d0aab46d..ce135564 100644 --- a/Makefile +++ b/Makefile @@ -153,6 +153,13 @@ test-stepfunctions-http-sdk: .PHONY: test-stepfunctions-http test-stepfunctions-http-sdk +test-stepfunctions-core: + @cargo test -p rustack-stepfunctions-core + @cargo clippy -p rustack-stepfunctions-core --all-targets -- -D warnings + @cargo clippy -p rustack-stepfunctions-core --lib --no-deps -- -D warnings -W clippy::pedantic -W clippy::missing_errors_doc -W clippy::doc_markdown -W clippy::unwrap_used -W clippy::expect_used -W clippy::indexing_slicing -W clippy::panic + +.PHONY: test-stepfunctions-core + audit-codegen: @cargo audit --file codegen/Cargo.lock diff --git a/crates/rustack-stepfunctions-core/Cargo.toml b/crates/rustack-stepfunctions-core/Cargo.toml new file mode 100644 index 00000000..b3618a76 --- /dev/null +++ b/crates/rustack-stepfunctions-core/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rustack-stepfunctions-core" +description = "Bounded pure ASL data semantics for the Rustack workflow engine" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +rustack-core = { workspace = true } +bytes = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +winnow = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +proptest = { workspace = true } diff --git a/crates/rustack-stepfunctions-core/src/error.rs b/crates/rustack-stepfunctions-core/src/error.rs new file mode 100644 index 00000000..94b134b0 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/error.rs @@ -0,0 +1,45 @@ +//! Redacted semantic failures with typed source ownership. + +use std::fmt; + +use rustack_core::json::JsonError; +use thiserror::Error; + +/// Bounded ASL data failures; messages never contain input or path text. +#[derive(Error)] +#[non_exhaustive] +pub enum AslError { + /// The path is invalid or exceeds the compiled-profile limits. + #[error("invalid ASL path")] + InvalidPath, + /// A selection operator is not a reference path. + #[error("invalid ASL reference path")] + InvalidReference, + /// Result paths cannot write into the context object. + #[error("invalid ASL result target")] + InvalidResultTarget, + /// A singular path could not select a value. + #[error("ASL path selected no value")] + Missing, + /// A merge traversed a scalar or an absent/out-of-range array element. + #[error("ASL result path cannot be applied")] + Merge, + /// Expansion exceeds the allowed bounded evaluation work. + #[error("ASL path evaluation work limit exceeded")] + WorkLimit, + /// Materialized data exceeds the payload byte allowance. + #[error("ASL payload byte limit exceeded")] + DataLimit, + /// Input or constructed data violates bounded JSON validation. + #[error("invalid bounded ASL JSON")] + Json(#[from] JsonError), + /// A bounded serializer failed; retain its source without projecting it. + #[error("ASL payload encoding failed")] + Encoding(#[source] serde_json::Error), +} + +impl fmt::Debug for AslError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} diff --git a/crates/rustack-stepfunctions-core/src/lib.rs b/crates/rustack-stepfunctions-core/src/lib.rs new file mode 100644 index 00000000..3defe0f9 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/lib.rs @@ -0,0 +1,15 @@ +//! Pure, bounded ASL payload and compiled-path semantics. +//! +//! This crate has no network, SDK, Lambda or HTTP dependency. Immutable payloads +//! preserve source bytes and compiled paths distinguish missing values from null. +//! No registry or execution-service capability is implied by these data APIs. + +#![forbid(unsafe_code)] + +mod error; +mod path; +mod payload; + +pub use error::AslError; +pub use path::{MatchKind, Path, PathMatches, ReferencePath, ResultPath}; +pub use payload::{EvaluationContext, MAX_PAYLOAD_BYTES, Payload}; diff --git a/crates/rustack-stepfunctions-core/src/path.rs b/crates/rustack-stepfunctions-core/src/path.rs new file mode 100644 index 00000000..e65411dd --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/path.rs @@ -0,0 +1,479 @@ +//! Compiled bounded `JSONPath` subset, borrowed selection and result merging. + +use std::{fmt, str::FromStr}; + +use serde_json::{Map, Value}; +use winnow::{ + ModalResult, Parser, + token::{any, take_while}, +}; + +use crate::{AslError, EvaluationContext, Payload, payload::encode}; + +const MAX_PATH_BYTES: usize = 1024; +const MAX_SEGMENTS: usize = 128; +const MAX_WORK: usize = 10_000; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Root { + Data, + Context, +} + +#[derive(Clone)] +enum Segment { + Member(String), + Index(usize), + Wildcard, + Slice { start: usize, end: Option }, +} + +/// Immutable compiled path, with no runtime reparsing or unsupported operators. +#[derive(Clone)] +#[non_exhaustive] +pub struct Path { + root: Root, + segments: Vec, + selection: bool, +} + +impl fmt::Debug for Path { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Path()") + } +} + +impl FromStr for Path { + type Err = AslError; + + fn from_str(text: &str) -> Result { + if text.is_empty() || text.len() > MAX_PATH_BYTES { + return Err(AslError::InvalidPath); + } + let mut input = text; + literal(&mut input, "$")?; + let root = if input.starts_with('$') { + literal(&mut input, "$")?; + Root::Context + } else { + Root::Data + }; + let mut segments = Vec::new(); + let mut selection = false; + while !input.is_empty() { + if segments.len() >= MAX_SEGMENTS { + return Err(AslError::InvalidPath); + } + let segment = if input.starts_with('.') { + literal(&mut input, ".")?; + let first: ModalResult<&str> = + take_while(1, identifier_first).parse_next(&mut input); + let first = first.map_err(|_| AslError::InvalidPath)?; + let rest: ModalResult<&str> = + take_while(0.., identifier_rest).parse_next(&mut input); + let rest = rest.map_err(|_| AslError::InvalidPath)?; + Segment::Member(format!("{first}{rest}")) + } else { + bracket(&mut input)? + }; + selection |= matches!(segment, Segment::Wildcard | Segment::Slice { .. }); + segments.push(segment); + } + Ok(Self { + root, + segments, + selection, + }) + } +} + +fn identifier_first(value: char) -> bool { + value == '_' || value.is_ascii_alphabetic() || (!value.is_ascii() && !value.is_control()) +} + +fn identifier_rest(value: char) -> bool { + identifier_first(value) || value.is_ascii_digit() +} + +fn literal(input: &mut &str, mut token: &'static str) -> Result<(), AslError> { + let parsed: ModalResult<&str> = token.parse_next(input); + parsed.map(|_| ()).map_err(|_| AslError::InvalidPath) +} + +fn number(input: &mut &str) -> Result, AslError> { + let parsed: ModalResult<&str> = take_while(0.., |c: char| c.is_ascii_digit()).parse_next(input); + let text = parsed.map_err(|_| AslError::InvalidPath)?; + if text.is_empty() { + Ok(None) + } else { + text.parse().map(Some).map_err(|_| AslError::InvalidPath) + } +} + +fn quoted(input: &mut &str) -> Result { + literal(input, "'")?; + let mut key = String::new(); + loop { + let parsed: ModalResult = any.parse_next(input); + match parsed.map_err(|_| AslError::InvalidPath)? { + '\'' => return Ok(key), + '\\' => { + let escaped: ModalResult = any.parse_next(input); + match escaped.map_err(|_| AslError::InvalidPath)? { + value @ ('\'' | '\\') => key.push(value), + _ => return Err(AslError::InvalidPath), + } + } + value if value.is_control() => return Err(AslError::InvalidPath), + value => key.push(value), + } + } +} + +fn bracket(input: &mut &str) -> Result { + literal(input, "[")?; + let segment = if input.starts_with('\'') { + Segment::Member(quoted(input)?) + } else if input.starts_with('*') { + literal(input, "*")?; + Segment::Wildcard + } else { + let lower = number(input)?; + if input.starts_with(':') { + literal(input, ":")?; + Segment::Slice { + start: lower.unwrap_or(0), + end: number(input)?, + } + } else { + Segment::Index(lower.ok_or(AslError::InvalidPath)?) + } + }; + literal(input, "]")?; + Ok(segment) +} + +/// Shape of a path result; JSON null is still a single selected value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum MatchKind { + /// No value matched a singular traversal. + Missing, + /// Exactly one value was selected without selection operators. + Single, + /// Wildcard/slice semantics, including zero or one selected value. + Selection, +} + +/// Borrowed immutable matches, constructible only through a bounded path. +#[non_exhaustive] +pub struct PathMatches<'a> { + kind: MatchKind, + values: Vec<&'a Value>, +} + +impl fmt::Debug for PathMatches<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PathMatches") + .field("kind", &self.kind) + .finish_non_exhaustive() + } +} + +impl PathMatches<'_> { + /// Inspect cardinality semantics without exposing business data. + /// + /// ``` + /// use rustack_stepfunctions_core::{Path, Payload, MatchKind}; + /// let data = Payload::parse(b"null")?; + /// let context = rustack_stepfunctions_core::EvaluationContext::from(&data); + /// let result = "$".parse::()?.select(&data, &context)?; + /// assert_eq!(result.kind(), MatchKind::Single); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn kind(&self) -> MatchKind { + self.kind + } + + /// Read a singular match; missing and selection results return None. + /// + /// ``` + /// use rustack_stepfunctions_core::{Path, Payload}; + /// let data = Payload::parse(b"null")?; + /// assert!("$".parse::()?.select(&data, &rustack_stepfunctions_core::EvaluationContext::from(&data))?.single().is_some()); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn single(&self) -> Option<&Value> { + if self.kind == MatchKind::Single { + self.values.first().copied() + } else { + None + } + } + + /// Materialize a complete bounded result; selections always become arrays. + /// + /// # Errors + /// Rejects missing matches, excessive encoded data or constructed depth. + /// + /// ``` + /// use rustack_stepfunctions_core::{Path, Payload}; + /// let data = Payload::parse(b"[7]")?; + /// let result = "$[*]".parse::()?.select(&data, &rustack_stepfunctions_core::EvaluationContext::from(&data))?.to_payload()?; + /// assert_eq!(result.original_bytes(), b"[7]"); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn to_payload(&self) -> Result { + let bytes = match self.kind { + MatchKind::Missing => return Err(AslError::Missing), + MatchKind::Single => encode(self.values.first().ok_or(AslError::Missing)?)?, + MatchKind::Selection => encode(&self.values)?, + }; + Payload::from_bytes(bytes) + } +} + +impl Path { + /// Evaluate compiled segments against immutable data and context roots. + /// + /// # Errors + /// Rejects evaluation beyond10,000 visited candidates/selected members. + /// + /// ``` + /// use rustack_stepfunctions_core::{Path, Payload}; + /// let data = Payload::parse(br#"{"items":[{"n":3}]}"#)?; + /// let path: Path = "$.items[*].n".parse()?; + /// assert_eq!(path.select(&data, &rustack_stepfunctions_core::EvaluationContext::from(&data))?.to_payload()?.value(), &serde_json::json!([3])); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn select<'a>( + &self, + data: &'a Payload, + context: &'a EvaluationContext, + ) -> Result, AslError> { + let root = match self.root { + Root::Data => data.value(), + Root::Context => context.value(), + }; + let mut values = vec![root]; + let mut remaining = MAX_WORK; + for segment in &self.segments { + let mut next = Vec::new(); + for value in values { + spend(&mut remaining, 1)?; + select_segment(segment, value, &mut next, &mut remaining)?; + } + values = next; + } + let kind = if self.selection { + MatchKind::Selection + } else if values.is_empty() { + MatchKind::Missing + } else { + MatchKind::Single + }; + Ok(PathMatches { kind, values }) + } +} + +fn spend(remaining: &mut usize, amount: usize) -> Result<(), AslError> { + *remaining = remaining.checked_sub(amount).ok_or(AslError::WorkLimit)?; + Ok(()) +} + +fn select_segment<'a>( + segment: &Segment, + value: &'a Value, + next: &mut Vec<&'a Value>, + remaining: &mut usize, +) -> Result<(), AslError> { + match segment { + Segment::Member(key) => { + if let Some(value) = value.as_object().and_then(|object| object.get(key)) { + next.push(value); + } + } + Segment::Index(index) => { + if let Some(value) = value.as_array().and_then(|array| array.get(*index)) { + next.push(value); + } + } + Segment::Wildcard => match value { + Value::Array(values) => { + spend(remaining, values.len())?; + next.extend(values); + } + Value::Object(values) => { + spend(remaining, values.len())?; + let mut ordered: Vec<_> = values.iter().collect(); + ordered.sort_unstable_by_key(|(key, _)| *key); + next.extend(ordered.into_iter().map(|(_, value)| value)); + } + _ => {} + }, + Segment::Slice { start, end } => { + if let Some(values) = value.as_array() { + let lower = (*start).min(values.len()); + let upper = end.unwrap_or(values.len()).min(values.len()); + if upper > lower { + spend(remaining, upper.saturating_sub(lower))?; + next.extend(values.get(lower..upper).ok_or(AslError::InvalidPath)?); + } + } + } + } + Ok(()) +} + +/// Compiled singular reference into either immutable root. +#[derive(Clone)] +#[non_exhaustive] +pub struct ReferencePath(Path); + +impl fmt::Debug for ReferencePath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ReferencePath()") + } +} + +impl TryFrom for ReferencePath { + type Error = AslError; + fn try_from(path: Path) -> Result { + if path.selection { + Err(AslError::InvalidReference) + } else { + Ok(Self(path)) + } + } +} + +impl FromStr for ReferencePath { + type Err = AslError; + fn from_str(text: &str) -> Result { + Self::try_from(text.parse::()?) + } +} + +impl ReferencePath { + /// Look up a singular reference; missing/container mismatch returns None. + /// + /// ``` + /// use rustack_stepfunctions_core::{ReferencePath, Payload}; + /// let data = Payload::parse(br#"{"n":null}"#)?; + /// assert!("$.n".parse::()?.lookup(&data, &rustack_stepfunctions_core::EvaluationContext::from(&data)).is_some()); + /// assert!("$.absent".parse::()?.lookup(&data, &rustack_stepfunctions_core::EvaluationContext::from(&data)).is_none()); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn lookup<'a>( + &self, + data: &'a Payload, + context: &'a EvaluationContext, + ) -> Option<&'a Value> { + let mut value = match self.0.root { + Root::Data => data.value(), + Root::Context => context.value(), + }; + for segment in &self.0.segments { + value = match segment { + Segment::Member(key) => value.as_object()?.get(key)?, + Segment::Index(index) => value.as_array()?.get(*index)?, + Segment::Wildcard | Segment::Slice { .. } => return None, + }; + } + Some(value) + } +} + +/// Data-root reference validated for result writes; context writes are rejected. +#[derive(Clone)] +#[non_exhaustive] +pub struct ResultPath(ReferencePath); + +impl fmt::Debug for ResultPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ResultPath()") + } +} + +impl TryFrom for ResultPath { + type Error = AslError; + fn try_from(path: ReferencePath) -> Result { + if path.0.root == Root::Context { + Err(AslError::InvalidResultTarget) + } else { + Ok(Self(path)) + } + } +} + +impl FromStr for ResultPath { + type Err = AslError; + fn from_str(text: &str) -> Result { + Self::try_from(text.parse::()?) + } +} + +impl ResultPath { + /// Merge into original state input without mutating either input on failure. + /// + /// # Errors + /// Rejects scalar traversal, absent/out-of-range array slots and output limits. + /// + /// ``` + /// use rustack_stepfunctions_core::{ResultPath, Payload}; + /// let state = Payload::parse(br#"{"n":1}"#)?; + /// let result = Payload::parse(b"2")?; + /// let output = "$.nested.value".parse::()?.merge(&state, &result)?; + /// assert_eq!(output.value()["nested"]["value"], 2); + /// assert_eq!(state.value(), &serde_json::json!({"n":1})); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn merge(&self, original: &Payload, result: &Payload) -> Result { + if self.0.0.segments.is_empty() { + return Ok(result.clone()); + } + let mut output = original.value().clone(); + let mut destination = &mut output; + let mut segments = self.0.0.segments.iter().peekable(); + while let Some(segment) = segments.next() { + let last = segments.peek().is_none(); + destination = match segment { + Segment::Member(key) => { + let object = destination.as_object_mut().ok_or(AslError::Merge)?; + if last { + object.insert(key.clone(), result.value().clone()); + break; + } + if !object.contains_key(key) { + if !matches!(segments.peek(), Some(Segment::Member(_))) { + return Err(AslError::Merge); + } + object.insert(key.clone(), Value::Object(Map::new())); + } + object.get_mut(key).ok_or(AslError::Merge)? + } + Segment::Index(index) => { + let slot = destination + .as_array_mut() + .and_then(|values| values.get_mut(*index)) + .ok_or(AslError::Merge)?; + if last { + *slot = result.value().clone(); + break; + } + slot + } + Segment::Wildcard | Segment::Slice { .. } => { + return Err(AslError::InvalidResultTarget); + } + }; + } + Payload::from_value(&output) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-core/src/path/tests.rs b/crates/rustack-stepfunctions-core/src/path/tests.rs new file mode 100644 index 00000000..ffc8cd57 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/path/tests.rs @@ -0,0 +1,342 @@ +use std::{error::Error, str::FromStr}; + +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::json::JsonError; +use serde_json::{Value, json}; + +use crate::{ + AslError, EvaluationContext, MAX_PAYLOAD_BYTES, MatchKind, Path, Payload, ReferencePath, + ResultPath, +}; + +type TestResult = Result<(), Box>; + +#[test] +fn test_should_execute_independent_catalog_path_cases() -> TestResult { + let catalog: Value = serde_json::from_str(include_str!( + "../../../../tests/workflows/semantic-cases.json" + ))?; + let mut count = 0; + for case in catalog["cases"].as_array().ok_or("missing cases")? { + if case["kind"] != "path" { + continue; + } + let input = Payload::from_value(&case["input"])?; + let context = + EvaluationContext::from_value(&case.get("context").cloned().unwrap_or(json!({})))?; + let path: Path = case["path"].as_str().ok_or("missing path")?.parse()?; + let matches = path.select(&input, &context)?; + if case.get("expectedError").is_some() { + assert_eq!(matches.kind(), MatchKind::Missing); + assert!(matches!(matches.to_payload(), Err(AslError::Missing))); + } else { + assert_eq!( + matches.to_payload()?.value(), + &case["expected"], + "{}", + case["id"] + ); + } + count += 1; + } + assert_eq!(count, 7); + Ok(()) +} + +#[rstest] +#[case("$")] +#[case("$$")] +#[case("$.a_b0")] +#[case("$.商品.价格")] +#[case("$['']")] +#[case("$['a\\'b']")] +#[case("$['a\\\\b']")] +#[case("$[0001]")] +#[case("$[:]")] +#[case("$[:2]")] +#[case("$[2:]")] +#[case("$[2:0]")] +#[case("$[*][*]")] +fn test_should_compile_the_explicit_path_subset(#[case] text: &str) -> TestResult { + let path: Path = text.parse()?; + assert_eq!(format!("{path:?}"), "Path()"); + Ok(()) +} + +#[rstest] +#[case("")] +#[case(" ")] +#[case("x")] +#[case("$$$")] +#[case("$.0a")] +#[case("$.a-b")] +#[case("$.a.")] +#[case("$..a")] +#[case("$.*")] +#[case("$[]")] +#[case("$[-1]")] +#[case("$[+1]")] +#[case("$[1,2]")] +#[case("$[1:2:1]")] +#[case("$[ 1]")] +#[case("$[?(@.x)]")] +#[case("$[\"a\"]")] +#[case("$['unterminated]")] +#[case("$['bad\\q']")] +#[case("$['bad\n']")] +#[case("$['a']junk")] +#[case("$[999999999999999999999999999999999]")] +fn test_should_reject_unsupported_or_hostile_path_syntax(#[case] text: &str) { + assert!(matches!(Path::from_str(text), Err(AslError::InvalidPath))); +} + +#[test] +fn test_should_enforce_exact_path_bytes_and_segment_counts() -> TestResult { + let maximum = format!("$['{}']", "k".repeat(1019)); + assert_eq!(maximum.len(), 1024); + maximum.parse::()?; + assert!(matches!( + format!("{maximum}x").parse::(), + Err(AslError::InvalidPath) + )); + format!("${}", ".a".repeat(128)).parse::()?; + assert!(matches!( + format!("${}", ".a".repeat(129)).parse::(), + Err(AslError::InvalidPath) + )); + Ok(()) +} + +#[rstest] +#[case("$[*][*]", json!([[1,2],[],[3]]), json!([1,2,3]))] +#[case("$[*].n", json!([{"n":1},{},{"n":null}]), json!([1,null]))] +#[case("$[*]", json!({"z":3,"a":1,"m":2}), json!([1,2,3]))] +#[case("$[1:]", json!([0,1,2]), json!([1,2]))] +#[case("$[:2]", json!([0,1,2]), json!([0,1]))] +#[case("$[1:999]", json!([0,1,2]), json!([1,2]))] +#[case("$[999:]", json!([0,1,2]), json!([]))] +#[case("$[2:1]", json!([0,1,2]), json!([]))] +#[case("$[*]", json!(null), json!([]))] +#[case("$.missing[*]", json!({}), json!([]))] +fn test_should_preserve_deterministic_selection_semantics( + #[case] text: &str, + #[case] input: Value, + #[case] expected: Value, +) -> TestResult { + let data = Payload::from_value(&input)?; + let context = EvaluationContext::from(&data); + let result = text.parse::()?.select(&data, &context)?; + assert_eq!(result.kind(), MatchKind::Selection); + assert!(result.single().is_none()); + assert_eq!(result.to_payload()?.value(), &expected); + Ok(()) +} + +#[test] +fn test_should_distinguish_missing_null_context_and_wrong_container() -> TestResult { + let data = Payload::parse(br#"{"n":null,"a":[],"x":3}"#)?; + let context = EvaluationContext::from_value(&json!({"n":"context"}))?; + let null = "$.n".parse::()?.select(&data, &context)?; + assert_eq!(null.kind(), MatchKind::Single); + assert_eq!(null.single(), Some(&Value::Null)); + assert_eq!( + "$$.n".parse::()?.lookup(&data, &context), + Some(&json!("context")) + ); + for text in ["$.absent", "$.a[0]", "$.x.n", "$.a.n", "$.n.n", "$[0]"] { + assert_eq!( + text.parse::()?.select(&data, &context)?.kind(), + MatchKind::Missing + ); + assert!( + text.parse::()? + .lookup(&data, &context) + .is_none() + ); + } + Ok(()) +} + +#[test] +fn test_should_prove_reference_and_result_root_invariants() -> TestResult { + for path in ["$[*]", "$[0:2]", "$$.x[*]"] { + assert!(matches!( + path.parse::(), + Err(AslError::InvalidReference) + )); + } + "$$.x".parse::()?; + assert!(matches!( + "$$.x".parse::(), + Err(AslError::InvalidResultTarget) + )); + assert_eq!( + format!("{:?}", "$.secret".parse::()?), + "ReferencePath()" + ); + assert_eq!( + format!("{:?}", "$.secret".parse::()?), + "ResultPath()" + ); + Ok(()) +} + +#[rstest] +#[case("$", json!({"a":1}), json!(7))] +#[case("$.a.b", json!({}), json!({"a":{"b":7}}))] +#[case("$.a", json!({"a":null}), json!({"a":7}))] +#[case("$[1].a", json!([0,{}]), json!([0,{"a":7}]))] +#[case("$[0]", json!([1]), json!([7]))] +#[case("$['odd.key']", json!({"old":1}), json!({"old":1,"odd.key":7}))] +fn test_should_merge_only_valid_object_and_existing_array_targets( + #[case] path: &str, + #[case] input: Value, + #[case] expected: Value, +) -> TestResult { + let original = Payload::from_value(&input)?; + let result = Payload::parse(b"7")?; + let output = path.parse::()?.merge(&original, &result)?; + assert_eq!(output.value(), &expected); + assert_eq!(original.value(), &input); + Ok(()) +} + +#[rstest] +#[case("$.a.b", json!({"a":null}))] +#[case("$.a.b", json!({"a":1}))] +#[case("$.a[0]", json!({}))] +#[case("$[0]", json!([]))] +#[case("$[1]", json!([0]))] +#[case("$.x", json!(null))] +#[case("$.x", json!([]))] +#[case("$[0]", json!({"0":1}))] +fn test_should_reject_invalid_merges_without_mutating_original( + #[case] path: &str, + #[case] input: Value, +) -> TestResult { + let original = Payload::from_value(&input)?; + let result = Payload::parse(b"7")?; + assert!(matches!( + path.parse::()?.merge(&original, &result), + Err(AslError::Merge) + )); + assert_eq!(original.value(), &input); + Ok(()) +} + +#[test] +fn test_should_bound_path_work_and_merged_output() -> TestResult { + let exact = Payload::from_value(&json!(vec![0; 9999]))?; + let path: Path = "$[*]".parse()?; + path.select(&exact, &EvaluationContext::from(&exact))? + .to_payload()?; + let excess = Payload::from_value(&json!(vec![0; 10_000]))?; + assert!(matches!( + path.select(&excess, &EvaluationContext::from(&excess)), + Err(AslError::WorkLimit) + )); + let large = Payload::from_value(&json!({"large":"v".repeat(150_000)}))?; + let result = Payload::from_value(&json!("x".repeat(150_000)))?; + assert!(matches!( + "$.result".parse::()?.merge(&large, &result), + Err(AslError::DataLimit) + )); + assert!(large.value().get("result").is_none()); + Ok(()) +} + +#[test] +fn test_should_bound_selection_materialization_before_cloning_or_encoding_large_data() -> TestResult +{ + let source = Payload::from_value(&json!([[["x".repeat(262_136)]]]))?; + assert_eq!(source.original_bytes().len(), MAX_PAYLOAD_BYTES); + let selected = "$[*]" + .parse::()? + .select(&source, &EvaluationContext::from(&source))? + .to_payload()?; + assert_eq!(selected.original_bytes().len(), MAX_PAYLOAD_BYTES); + assert_eq!(selected.value(), source.value()); + let flattened = "$[*][*]" + .parse::()? + .select(&source, &EvaluationContext::from(&source))? + .to_payload()?; + assert_eq!(flattened.original_bytes().len(), MAX_PAYLOAD_BYTES - 2); + Ok(()) +} + +#[test] +fn test_should_bound_payload_bytes_depth_members_duplicates_and_encoding() -> TestResult { + let raw = format!("\"{}\"", "x".repeat(MAX_PAYLOAD_BYTES - 2)); + let data = Payload::parse(raw.as_bytes())?; + assert_eq!(data.original_bytes().len(), MAX_PAYLOAD_BYTES); + assert!(matches!( + Payload::parse(format!("{raw} ").as_bytes()), + Err(AslError::Json(JsonError::Bytes)) + )); + assert!(matches!( + Payload::parse(br#"{"a":1,"\u0061":2}"#), + Err(AslError::Json(JsonError::Duplicate)) + )); + assert!(Payload::parse(b"").is_err()); + let depth64 = format!("{}0{}", "[".repeat(64), "]".repeat(64)); + Payload::parse(depth64.as_bytes())?; + assert!(matches!( + Payload::parse(format!("[{depth64}]").as_bytes()), + Err(AslError::Json(JsonError::Depth)) + )); + let mut nested = json!(0); + for _ in 0..65 { + nested = json!([nested]); + } + assert!(matches!( + Payload::from_value(&nested), + Err(AslError::Json(JsonError::Depth)) + )); + assert!(matches!( + Payload::from_value(&json!(vec![0; 65_537])), + Err(AslError::Json(JsonError::Members)) + )); + assert!(matches!( + Payload::from_value(&json!("\u{0}".repeat(50_000))), + Err(AslError::DataLimit) + )); + assert!(matches!( + Payload::from_value(&json!("x".repeat(MAX_PAYLOAD_BYTES + 1))), + Err(AslError::Json(JsonError::String)) + )); + assert_eq!(format!("{data:?}"), "Payload()"); + let source = Payload::parse(b" null\n")?; + assert_eq!(source.clone().original_bytes(), b" null\n"); + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + #[test] + fn test_should_parse_arbitrary_utf8_without_panics(text in ".{0,1200}") { + let _ = text.parse::(); + let _ = text.parse::(); + let _ = text.parse::(); + } + + #[test] + fn test_should_round_trip_quoted_keys(key in "[^\\p{Cc}]{0,80}", number in any::()) { + let escaped = key.replace('\\', "\\\\").replace('\'', "\\'"); + let text = format!("$['{escaped}']"); + let path = text.parse::().map_err(|e| TestCaseError::fail(e.to_string()))?; + let data = Payload::from_value(&json!({key:number})).map_err(|e| TestCaseError::fail(e.to_string()))?; + let context = EvaluationContext::from(&data); + prop_assert_eq!(path.lookup(&data, &context), Some(&json!(number))); + } + + #[test] + fn test_should_preserve_original_when_merging_object_member(input in any::(), result in any::()) { + let original = Payload::from_value(&json!({"input":input})).map_err(|e| TestCaseError::fail(e.to_string()))?; + let value = Payload::from_value(&json!(result)).map_err(|e| TestCaseError::fail(e.to_string()))?; + let path = "$.result".parse::().map_err(|e| TestCaseError::fail(e.to_string()))?; + let output = path.merge(&original, &value).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(original.value(), &json!({"input":input})); + prop_assert_eq!(output.value(), &json!({"input":input,"result":result})); + } +} diff --git a/crates/rustack-stepfunctions-core/src/payload.rs b/crates/rustack-stepfunctions-core/src/payload.rs new file mode 100644 index 00000000..197a2da2 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/payload.rs @@ -0,0 +1,292 @@ +//! Validated immutable JSON and bounded output materialization. + +use std::{fmt, io, sync::Arc}; + +use bytes::Bytes; +use rustack_core::json::{JsonError, JsonLimits, from_slice}; +use serde::Serialize; +use serde_json::Value; + +use crate::AslError; + +/// Maximum original or materialized UTF-8 JSON payload size. +pub const MAX_PAYLOAD_BYTES: usize = 262_144; +const MAX_DEPTH: usize = 64; +const MAX_MEMBERS: usize = 65_536; + +const PAYLOAD_LIMITS: TreeLimits = TreeLimits { + bytes: MAX_PAYLOAD_BYTES, + depth: MAX_DEPTH, + members: MAX_MEMBERS, +}; + +const CONTEXT_LIMITS: TreeLimits = TreeLimits { + bytes: 4 * 1024 * 1024, + depth: 68, + members: 2 * MAX_MEMBERS + 512, +}; + +#[derive(Clone, Copy)] +struct TreeLimits { + bytes: usize, + depth: usize, + members: usize, +} + +/// Immutable bounded JSON, retaining original bytes for execution idempotency. +#[derive(Clone)] +#[non_exhaustive] +pub struct Payload { + value: Arc, + bytes: Bytes, +} + +impl fmt::Debug for Payload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Payload()") + } +} + +impl Payload { + /// Validate JSON before constructing an immutable parsed tree. + /// + /// # Errors + /// Rejects duplicate keys, invalid JSON and byte/depth/member/string limits. + /// + /// ``` + /// use rustack_stepfunctions_core::Payload; + /// let input = Payload::parse(br#" {"n":null} "#)?; + /// assert_eq!(input.original_bytes(), br#" {"n":null} "#); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn parse(input: &[u8]) -> Result { + if input.len() > MAX_PAYLOAD_BYTES { + return Err(JsonError::Bytes.into()); + } + Self::from_bytes(Bytes::copy_from_slice(input)) + } + + /// Validate a constructed tree before bounded compact serialization. + /// + /// # Errors + /// Rejects excessive structure, strings or serialized byte size. + /// + /// ``` + /// use rustack_stepfunctions_core::Payload; + /// let output = Payload::from_value(&serde_json::json!({"ok": true}))?; + /// assert_eq!(output.value()["ok"], true); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn from_value(value: &Value) -> Result { + validate_value(value, PAYLOAD_LIMITS)?; + let bytes = encode(value)?; + Ok(Self { + value: Arc::new(value.clone()), + bytes, + }) + } + + /// Inspect the immutable parsed value; mutations require a new validation. + /// + /// ``` + /// let input = rustack_stepfunctions_core::Payload::parse(b"null")?; + /// assert!(input.value().is_null()); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn value(&self) -> &Value { + &self.value + } + + /// Read the exact source bytes, including whitespace when parsed from text. + /// + /// ``` + /// let input = rustack_stepfunctions_core::Payload::parse(b" 1 ")?; + /// assert_eq!(input.original_bytes(), b" 1 "); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn original_bytes(&self) -> &[u8] { + &self.bytes + } + + pub(crate) fn from_bytes(bytes: Bytes) -> Result { + let limits = JsonLimits::new( + MAX_PAYLOAD_BYTES, + MAX_DEPTH, + MAX_MEMBERS, + MAX_PAYLOAD_BYTES, + MAX_PAYLOAD_BYTES, + )?; + let value = from_slice(&bytes, limits)?; + Ok(Self { + value: Arc::new(value), + bytes, + }) + } +} + +/// Immutable internal ASL context with headroom for embedded business payloads. +/// +/// Context is bounded to4MiB,68 container levels and131,584 members. Selected +/// output still passes through the independent256KiB/depth64 payload boundary. +#[derive(Clone)] +#[non_exhaustive] +pub struct EvaluationContext { + value: Arc, +} + +impl fmt::Debug for EvaluationContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("EvaluationContext()") + } +} + +impl From<&Payload> for EvaluationContext { + fn from(payload: &Payload) -> Self { + Self { + value: payload.value.clone(), + } + } +} + +impl EvaluationContext { + /// Validate an internal context before cloning its tree. + /// + /// Borrowing prevents recursive destruction of rejected caller-owned trees. + /// Schema/scoping and Task.Token access are owned by the future typed engine. + /// + /// # Errors + /// Rejects the internal context's independent byte/depth/member/string limits. + /// + /// ``` + /// use rustack_stepfunctions_core::EvaluationContext; + /// let context = EvaluationContext::from_value(&serde_json::json!({ + /// "Execution": {"Input": {"message":"hello"}} + /// }))?; + /// assert!(context.value().get("Execution").is_some()); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn from_value(value: &Value) -> Result { + validate_value(value, CONTEXT_LIMITS)?; + encode_with_limit(value, CONTEXT_LIMITS.bytes)?; + Ok(Self { + value: Arc::new(value.clone()), + }) + } + + /// Inspect the immutable context; Debug remains redacted. + /// + /// ``` + /// use rustack_stepfunctions_core::{EvaluationContext, Payload}; + /// let input = Payload::parse(b"{}")?; + /// assert!(EvaluationContext::from(&input).value().is_object()); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + #[must_use] + pub fn value(&self) -> &Value { + &self.value + } +} + +fn validate_value(value: &Value, limits: TreeLimits) -> Result<(), AslError> { + let mut stack = vec![(value, 0_usize)]; + let mut remaining = limits.members; + while let Some((value, depth)) = stack.pop() { + let children = match value { + Value::Array(values) => { + check_container(depth, values.len(), &mut remaining, limits.depth)?; + stack.extend(values.iter().map(|value| (value, depth.saturating_add(1)))); + continue; + } + Value::Object(values) => { + check_container(depth, values.len(), &mut remaining, limits.depth)?; + for key in values.keys() { + check_string(key)?; + } + values.values() + } + Value::String(value) => { + check_string(value)?; + continue; + } + _ => continue, + }; + stack.extend(children.map(|value| (value, depth.saturating_add(1)))); + } + Ok(()) +} + +fn check_container( + depth: usize, + length: usize, + remaining: &mut usize, + max_depth: usize, +) -> Result<(), AslError> { + if depth >= max_depth { + return Err(JsonError::Depth.into()); + } + *remaining = remaining.checked_sub(length).ok_or(JsonError::Members)?; + Ok(()) +} + +fn check_string(value: &str) -> Result<(), AslError> { + if value.len() > MAX_PAYLOAD_BYTES { + Err(JsonError::String.into()) + } else { + Ok(()) + } +} + +pub(crate) fn encode(value: &T) -> Result { + encode_with_limit(value, MAX_PAYLOAD_BYTES) +} + +fn encode_with_limit(value: &T, limit: usize) -> Result { + let mut writer = LimitedWriter { + bytes: Vec::new(), + full: false, + limit, + }; + if let Err(error) = serde_json::to_writer(&mut writer, value) { + return if writer.full { + Err(AslError::DataLimit) + } else { + Err(AslError::Encoding(error)) + }; + } + Ok(Bytes::from(writer.bytes)) +} + +#[cfg(test)] +mod tests; + +struct LimitedWriter { + bytes: Vec, + full: bool, + limit: usize, +} + +impl io::Write for LimitedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let needed = self.bytes.len().saturating_add(bytes.len()); + if needed > self.limit { + self.full = true; + return Err(io::Error::other("bounded ASL output exhausted")); + } + if needed > self.bytes.capacity() { + let capacity = needed + .max(self.bytes.capacity().saturating_mul(2)) + .clamp(4096, self.limit); + self.bytes + .try_reserve_exact(capacity.saturating_sub(self.bytes.len())) + .map_err(|_| io::Error::other("bounded ASL allocation failed"))?; + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crates/rustack-stepfunctions-core/src/payload/tests.rs b/crates/rustack-stepfunctions-core/src/payload/tests.rs new file mode 100644 index 00000000..7e853384 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/payload/tests.rs @@ -0,0 +1,157 @@ +use std::error::Error; + +use rustack_core::json::JsonError; +use serde_json::{Value, json}; + +use crate::{AslError, EvaluationContext, MAX_PAYLOAD_BYTES, Path, Payload}; + +type TestResult = Result<(), Box>; + +#[test] +fn test_should_reject_borrowed_extreme_depth_without_recursive_owned_teardown() { + let mut value = Value::Null; + for _ in 0..50_000 { + value = Value::Array(vec![value]); + } + let payload = Payload::from_value(&value); + let context = EvaluationContext::from_value(&value); + // The caller retains and safely disposes its own rejected tree. + let mut owned = vec![value]; + while let Some(value) = owned.pop() { + if let Value::Array(mut children) = value { + owned.append(&mut children); + } + } + assert!(matches!(payload, Err(AslError::Json(JsonError::Depth)))); + assert!(matches!(context, Err(AslError::Json(JsonError::Depth)))); +} + +#[test] +fn test_should_embed_maximum_payload_bytes_without_reducing_output_limit() -> TestResult { + let payload = Payload::parse(format!("\"{}\"", "x".repeat(MAX_PAYLOAD_BYTES - 2)).as_bytes())?; + let context = EvaluationContext::from_value(&json!({ + "Execution":{"Input":payload.value(),"Name":"run","Id":"execution-id"}, + "StateMachine":{"Name":"machine"},"State":{"Name":"current"} + }))?; + let path: Path = "$$.Execution.Input".parse()?; + let result = path.select(&payload, &context)?.to_payload()?; + assert_eq!(result.original_bytes().len(), MAX_PAYLOAD_BYTES); + assert_eq!(result.value(), payload.value()); + assert!(matches!( + "$$".parse::()? + .select(&payload, &context)? + .to_payload(), + Err(AslError::DataLimit) + )); + assert_eq!(format!("{context:?}"), "EvaluationContext()"); + Ok(()) +} + +#[test] +fn test_should_embed_maximum_payload_depth_and_member_count_with_metadata() -> TestResult { + let raw = format!("{}0{}", "[".repeat(64), "]".repeat(64)); + let deep = Payload::parse(raw.as_bytes())?; + let context = EvaluationContext::from_value(&json!({"Execution":{"Input":deep.value()}}))?; + let selected = "$$.Execution.Input" + .parse::()? + .select(&deep, &context)? + .to_payload()?; + assert_eq!(selected.value(), deep.value()); + assert!(matches!( + "$$".parse::()?.select(&deep, &context)?.to_payload(), + Err(AslError::Json(JsonError::Depth)) + )); + let wide = Payload::from_value(&json!(vec![0; 65_536]))?; + let context = EvaluationContext::from_value(&json!({ + "Execution":{"Input":wide.value()}, + "Map":{"Item":{"Value":wide.value(),"Index":0,"Source":"STATE_DATA"}} + }))?; + assert_eq!( + "$$.Execution.Input" + .parse::()? + .select(&wide, &context)? + .to_payload()? + .value(), + wide.value() + ); + Ok(()) +} + +#[test] +fn test_should_enforce_independent_context_byte_depth_member_and_string_limits() { + let value = json!(vec!["x".repeat(MAX_PAYLOAD_BYTES); 16]); + assert!(matches!( + EvaluationContext::from_value(&value), + Err(AslError::DataLimit) + )); + let mut value = Value::Null; + for _ in 0..68 { + value = json!([value]); + } + assert!(EvaluationContext::from_value(&value).is_ok()); + assert!(matches!( + EvaluationContext::from_value(&json!([value])), + Err(AslError::Json(JsonError::Depth)) + )); + assert!(matches!( + EvaluationContext::from_value(&json!(vec![0; 131_585])), + Err(AslError::Json(JsonError::Members)) + )); + assert!(matches!( + EvaluationContext::from_value(&json!("x".repeat(MAX_PAYLOAD_BYTES + 1))), + Err(AslError::Json(JsonError::String)) + )); +} + +#[test] +fn test_should_allow_numeric_normalization_expansion_inside_internal_context() -> TestResult { + let input = format!("[{}]", vec!["1e9"; 65_535].join(",")); + assert_eq!(input.len(), 262_141); + let input = Payload::parse(input.as_bytes())?; + let item = Payload::parse(format!("\"{}\"", "x".repeat(MAX_PAYLOAD_BYTES - 2)).as_bytes())?; + let context_value = json!({ + "Execution":{"Input":input.value()}, + "Map":{"Item":{"Value":item.value(),"Index":0,"Source":"STATE_DATA"}} + }); + assert!(serde_json::to_vec(&context_value)?.len() > 1024 * 1024); + let context = EvaluationContext::from_value(&context_value)?; + assert_eq!( + "$$.Map.Item.Value" + .parse::()? + .select(&input, &context)? + .to_payload()? + .value(), + item.value() + ); + EvaluationContext::from_value(&json!({ + "Execution":{"Input":input.value()}, + "Map":{"Item":{"Value":input.value(),"Index":0}} + }))?; + assert!(matches!( + "$$.Execution.Input" + .parse::()? + .select(&item, &context)? + .to_payload(), + Err(AslError::DataLimit) + )); + Ok(()) +} + +#[test] +fn test_should_bound_exact_internal_context_encoding_bytes() -> TestResult { + let mut parts = vec![Value::String("x".repeat(262_141)); 16]; + parts.pop(); + parts.push(Value::String("x".repeat(262_140))); + let exact = Value::Array(parts); + assert_eq!(serde_json::to_vec(&exact)?.len(), 4 * 1024 * 1024); + EvaluationContext::from_value(&exact)?; + let mut excess = exact; + if let Some(Value::String(last)) = excess.as_array_mut().and_then(|array| array.last_mut()) { + last.push('x'); + } + assert!(matches!( + EvaluationContext::from_value(&excess), + Err(AslError::DataLimit) + )); + Ok(()) +} diff --git a/specs/index.md b/specs/index.md index e0f0b6ea..6d75b7ba 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Main-service registration, registry, ASL engine and the remaining P2–P5 requirements remain pending. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. The bounded compiled path/payload foundation is implemented, with separate context headroom and independent review. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-30-asl-design.md b/specs/rustack-workflows-30-asl-design.md index 916bfba0..5ed67a92 100644 --- a/specs/rustack-workflows-30-asl-design.md +++ b/specs/rustack-workflows-30-asl-design.md @@ -33,6 +33,36 @@ Use winnow for an explicit ASL subset rather than treating a general JSONPath cr Return Missing, Single or Selection, distinguishing JSON null from no match. Wildcard/slice selections remain arrays even with one match. Array ordering is stable; object wildcard uses deterministic key order as a local behavior, not an AWS object-order guarantee. Reference lookup failures do not panic or become null. +#### 2.1.1 Executable path and payload foundation + +The first `rustack-stepfunctions-core` slice exposes only implemented pure semantics: immutable Payload, compiled Path/ReferencePath and bounded result merging. It does not supply a registry, handler or advertised service capability. Core depends on shared JSON validation, not HTTP, Lambda or an SDK. Reuse the workspace winnow1.0 parser (current1.0.4 [API](https://docs.rs/winnow/1.0.4/winnow/)); no new external dependency is required. + +Payload constructors validate at most262,144 original UTF-8 JSON bytes, depth64 and65,536 aggregate members through the shared duplicate-key preflight. Preserve original bytes for API idempotency and expose only an immutable parsed value. Constructors accepting an existing Value borrow it; validate structure and finish bounded serialization before cloning, so rejection never recursively drops a caller-owned deeply nested tree. Serialization must stop at262,144 bytes rather than build an oversized buffer first. All diagnostic Debug/Display implementations redact values, paths and source text. Typed source errors remain available without public input excerpts. + +EvaluationContext has an independent internal envelope:4MiB,68 container levels and131,584 aggregate members, with each decoded string/key at most262,144 bytes. This accommodates full-size/depth/member-count Execution.Input and Map.Item.Value plus fixed context metadata. Compact exponent lexemes can expand during numeric serialization:65,535 copies of1e9 fit262,141 source bytes but encode to about852KiB. The pinned serializer uses at most24 bytes per numeric value; a conservative allowance of both original256KiB sources plus24 extra bytes per131,584 members and256KiB metadata remains below4MiB. Test both exponent-heavy inputs together and mixed exponent/string payloads, rather than deriving context capacity from string fixtures alone. It is immutable, constructed from a borrowed validated tree before cloning, and redacts Debug. Engine-owned typed builders must later constrain the actual context schema and per-field metadata. Selecting from context still materializes through the ordinary payload's256KiB/depth64/member65,536 boundary; the context root itself need not fit that output boundary. Converting an existing Payload into EvaluationContext shares its immutable Arc and requires no serialization. + +Paths are at most1,024 UTF-8 bytes and128 segments. Dot member shorthand starts with an ASCII letter/underscore or non-ASCII non-control scalar; subsequent characters may also be ASCII digits. Bracket members use single quotes and support only escaped single quote/backslash; reject control characters and unterminated/unknown escapes. Bracket indices are decimal usize values with checked conversion; slices allow omitted lower/upper bounds, defaulting to zero/array length, and never negative indices, steps or whitespace. The whole string must parse. A general path may use either root; ReferencePath rejects wildcard/slice segments, and a separate result-target conversion rejects the context root. This is the local subset of the [AWS path contract](https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-paths.html), not full JSONPath support. + +Evaluation borrows immutable payload/context values and bounds work to10,000 visited candidates/selected members per call. A wildcard/slice makes the result a Selection even with zero or one match; failed singular traversal is Missing, distinct from Single(null). Preserve array traversal order; explicitly sort object wildcard keys independently of serde_json feature unification. No missing/type mismatch is converted to null. Materialization returns a bounded Payload and raises a typed missing/data-limit/work-limit error. Selection materialization streams references through the bounded writer before building its owned value. + +Result merging accepts only the validated data-root ReferencePath. The root replaces the original payload. Nested object members may be created only as objects; writing through scalar/null or a missing member immediately followed by an array index fails. Existing arrays require an in-range index; never grow an array. Apply edits to an owned copy, preserving the original payload on every failure, and enforce the full output depth/member/byte limits after the edit. Discarding a result is a separate data-pipeline decision, not a special path spelling. + +```text +┌─ bounded source text ─────────────┐ ┌─ bounded path text ────────────┐ +│ JSON preflight → Payload │ │ winnow → Path / ReferencePath │ +│ original bytes + immutable tree │ │ root + checked typed segments │ +└────────────────┬────────────────┘ └─────────────────┬──────────────┘ + └──────────────────┬──────────────────┘ + ▼ +┌─ pure evaluator / result merge ────────────────────────────────────┐ +│ borrowed candidates;10,000 work units; deterministic selections │ +│ Missing / Single / Selection → bounded writer → validated Payload │ +│ typed redacted failures; original input never mutated │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Tests must exercise the actual evaluator against independent examples in the semantic fixture catalog, exact source/compiled/expansion limits, hostile escapes/overflow, context access, consecutive wildcards, null/missing/empty selection, scalar merges and non-growing arrays. Property tests cover arbitrary UTF-8 parsing and quoted-key round trips; this foundation does not satisfy complete compiler/engine acceptance. + ### 2.2 Data processing ```text diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 3fc972d2..90af50f0 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -174,3 +174,9 @@ Run `make audit-codegen deny-codegen` for the excluded developer-tool graph as w `make test-stepfunctions-http` exercises all18 typed action mappings, operation-correlated DTO encoding, numeric dates, source-level2MiB body limits, total deadlines, redaction and strict production-library Clippy. Reject duplicate/unknown JSON fields, header ambiguity, signed service/region/action mismatches, nondecimal Content-Length and every trailer frame. Cover exact byte bounds, Unicode business JSON, empty/space-containing ASCII trace headers, oversized/invalid handler errors and8MiB response overflow without mutation replay. Cancellation cases verify accepted commands survive HTTP waiter loss; these controlled-handler cases do not establish execution-actor ownership. The actual Hyper network fixture rejects a chunked request carrying an oversized trailer without dispatch. A paused regression expires its cleanup waiter and then resumes the same root task's join, proving connection-task ownership is retained. `make test-stepfunctions-http-sdk` runs pinned ordinary boto3 with strict signed local requests against the controlled typed handler: StartExecution preserves raw input and a numeric negative fractional date, ListStateMachines decodes normally, DescribeExecution returns a modeled exception and CreateActivity reports unsupported. The fixture bounds connection admission and joins its server/connection tasks. These are transport-only proofs; the main program does not advertise/register Step Functions until a real core adapter exists. + +## Step Functions compiled path/payload qualification + +`make test-stepfunctions-core` exercises actual compiled winnow paths and transactional result merging. All seven independent path catalog cases run against the evaluator. Cover singular missing/null, context roots, deterministic object wildcards, consecutive wildcards/slices, unsupported syntax/escapes/index overflow,1,024-byte/128-segment and10,000-work boundaries, exact256KiB data,64-depth/65,536-member preflight and escaped output expansion. Three512-case properties cover arbitrary UTF-8 parsing, quoted-key round trips and merge preservation. Twelve executable public doctests and strict production Clippy qualify the API. + +Reject a borrowed50,000-depth Value without owning its recursive teardown; the test caller safely dismantles that tree iteratively. Context tests embed maximum-size/depth/member source payloads plus metadata, including compact exponent input whose normalized number representation expands, mixed/full numeric Map items, exact4MiB internal bytes and+1 rejection. Selected context output retains the normal payload limits. This pure semantic foundation does not satisfy compiler, registry or execution acceptance. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index d8ede8ef..316bcb27 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -159,6 +159,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 1.3/1.4 historical container effects | Implemented/reviewed; other mutation classes remain | §4.8/D38 add bounded durable per-container intents before creation and known IDs before deletion. The restart pass preserves unknown absence, blocks unresolved relays, validates exact identities and resumes after cancellation. W95–W96 are fixed and independently rechecked.24 new scripted/unit cases pass, including create-before-effect and durable-erase reservation barriers. Workspace2,394/412 gated; boundary978 units+218 docs (1,196 total)/18 gated; standard/strict/audit/deny gates pass. Each arm64 VZ and amd64/QEMU daemon passes11 distinct actual recovery/pool/SDK/wire cases; scoped container/private-alias queries are empty afterward. Actual directory reacquisition and historical pair retirement pass on both. This does not cover historical image aliases or enable production startup. | | 2.1 generated model/timestamps | Implemented/reviewed; HTTP/registry still pending | API§1.1/D40 generates37 official operations from312 pinned shapes with exact numeric required/optional epoch dates and strict DTO policy.5 generator cases,34 model cases,1 doctest, regeneration consistency and strict model/tool checks pass. W97–W100 are fixed/rechecked. Final current-worktree workspace2,447/414 gated, including the retained deferred image checkpoint; build/test/fmt/Clippy/rustdoc, root475/tool29-crate audit and both deny graphs pass. This is protocol/model evidence, not API/engine or real-image workflow acceptance. | | 2.1 typed HTTP transport | Implemented/reviewed; real core/registration pending | API§1.2/D41 implements18 typed ports, bounded envelopes/body/response, strict SigV4 service/region/target binding, exact DTO dates and accepted-command waiter isolation.56 unit tests, two nongated network tests, eight doctests, strict HTTP Clippy and four real signed boto3 calls pass. W101–W104 are fixed and independently rechecked. Current-worktree workspace2,505/415 gated, standard build/test/fmt/Clippy/rustdoc, audit and deny pass. No production fallback handler, main registration or engine acceptance is claimed. | +| 2.2 compiled paths/payloads | Implemented; complete compiler remains | ASL§2.1.1/D42 supplies bounded immutable payloads/context, compiled paths, borrowed missing/single/selection and transactional result merge.75 unit tests (three512-case properties),12 doctests and strict production checks pass; W105–W107 corrections cover borrowed deep-tree rejection and numeric/depth/member context headroom. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. New existing-dependency advisory findings are tracked as B06; audit/deny currently fail that baseline, with no ignores added. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 0fbf7ddd..5db81323 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -490,15 +490,28 @@ The production transport implements API§1.2/D41 without a default handler or pr Focused evidence:56 unit tests, two nongated network tests and eight doctests pass; strict production HTTP Clippy passes. The separately gated real signed boto3 fixture passes four calls against a controlled typed handler. This qualifies transport only; registry, compiler, execution engine and real Lambda/image workflow acceptance remain required. Final current-worktree standard build/test/fmt/Clippy/rustdoc pass with2,505 tests/415 gated (including the deferred Docker checkpoint). Workspace476/tool29 dependency audits and both deny graphs pass. +## P2 compiled path/payload independent review + +ASL§2.1.1/D42 implements bounded immutable business payloads, a separate internal context, typed compiled paths, borrowed selection and transactional result writes. Review corrections: + +| ID | Priority / source | Correction | Verification | +|---|---|---|---| +| W105 | P2; `crates/rustack-stepfunctions-core/src/payload.rs` | Borrow constructed Values; validate and encode before cloning so rejection owns no recursive tree Drop |50,000-depth payload/context rejection with caller-owned iterative teardown | +| W106 | P2; `crates/rustack-stepfunctions-core/src/path.rs` | Use a separate EvaluationContext with byte/depth/member headroom | Full256KiB,64-depth and65,536-member source inputs plus metadata/Map items | +| W107 | P2; `crates/rustack-stepfunctions-core/src/payload.rs` | Derive4MiB internal context cap from numeric-normalization expansion, rather than original source byte lengths alone | Compact exponent-heavy inputs, max string/full numeric Map items, exact cap/+1; selected output still bounded | + +Focused evidence:75 unit tests (including three512-case properties),12 doctests and ordinary/strict core Clippy pass. All seven independent path catalog cases execute against the evaluator. The independent read-only reviewer rechecked all three corrections and reported no remaining in-slice findings. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. Full compiler/registry/engine and M1 acceptance remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | |---|---|---|---| -| B01 | P1; `Cargo.toml:171`, `Cargo.lock`, runtime §7.2/D27 | **Closed:** all16 original advisory findings resolved; current cargo audit has no vulnerabilities and deny passes all four checks, without new ignores. Patched compatible dependencies and ZIP9 replace affected versions; unused local SDK SigV4a crypto graph is removed | Retain audit/deny, hostile ZIP and regional SDK regressions through P1 acceptance; this is not full executor/platform acceptance | +| B01 | P1; `Cargo.toml:171`, `Cargo.lock`, runtime §7.2/D27 | **Closed for the original baseline:** all16 original advisory findings were resolved; audit and all four deny checks passed at that closure without new ignores. Later database additions are tracked separately in B06. Patched compatible dependencies and ZIP9 replace affected versions; unused local SDK SigV4a crypto graph is removed | Retain audit/deny, hostile ZIP and regional SDK regressions through P1 acceptance; this is not full executor/platform acceptance | | B02 | P2; `crates/rustack-kms-model/src/epoch_seconds.rs:39` | Existing optional epoch helper truncates negative fractional seconds and casts a negative fraction to u32; it is unsuitable as the generator's new codec | Do not reuse it in P2. Existing KMS correction is outside this workflow feature; retain as a repository follow-up without claiming it fixed | | B03 | P2; Lambda model/HTTP/core and shared auth/core/runtime production libraries | **Closed:** the six-library strict target passes without new ignores after correcting123 fresh baseline findings, including typed MAC initialization errors, guarded indexing and method-specific Errors contracts | Retain `make test-lambda-boundaries` through P1 acceptance; this is not global workspace strict-lint or Docker executor acceptance | | B04 | P2; `crates/rustack-lambda-model/src/input.rs`, `provider.rs` code-update dispatch | **Closed:** explicit DryRun, strict deployment DTOs and bounded decoder implemented; ZIP/S3/image nonmutation and live SDK rejection/absence/publication checks pass | Retain regressions through P1 acceptance; this fix does not establish Docker Image Invoke | | B05 | P2; `crates/rustack-lambda-core/src/docker/environment/worker.rs`, call/idle event mapping | **Closed:** BootstrapReady does not prove event consumption; shared event classification now preserves <=32KiB InitError as Initialization in initialization/Idle/Busy, with oversized/unexpected events remaining Protocol | Four composed session/HTTP/relay/client/worker regressions cover abandoned polling/offers, Idle/Busy32KiB/32KiB+1, a blocked container-absence barrier before reply, redacted diagnostics and joined control/log/API retirement. Independent read-only review reports no findings; no actual Docker execution is claimed by these scripted-container cases | +| B06 | P1; existing `Cargo.toml` zstd0.13.3 and transitive squib-loader0.2.0 dependency chain | **Open, newly observed database update:** the audit source grew from1,355 to1,378 advisories during this slice; current audit fails4 vulnerabilities (RUSTSEC-2026-0391/0403/0398/0410) and reports10 unsound warnings. Required patched versions are zstd≥0.14.3, zstd-safe≥8.2.0 and zstd-sys≥2.1.2. Current cargo deny fails advisories while bans/licenses/sources pass. No vulnerable dependency was introduced by the pure core crate | Remediate the direct snapshot and transitive Squib graph, retain archive/kernel-loader regressions and rerun audit/deny; no new advisory ignores or blanket green gate claim | The P0 record distinguishes unchanged workspace advisory failures from the clean probe dependency graph; it never reports a fully green security baseline. diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 74a476dd..607d3db6 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -366,3 +366,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: protocol qualification is independently testable without claiming workflow execution; core keeps domain/compiler semantics and cleanup ownership. Encoded page budgets prevent successful responses from relying on unbounded allocation. - **Pinned by**: [API §1.2](./rustack-workflows-10-api-design.md#12-typed-http-boundary-before-registry-wiring), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. Reuse existing HTTP/auth/JSON/Tokio/async-trait dependencies. Unknown-action404 follows the current official common-error reference; no AWS error-precedence execution is claimed. + +## D42 — Compile bounded paths against immutable validated payloads + +- **Context**: the registry/compiler and every state evaluator need shared data semantics before the execution owner exists. +- **Alternatives considered**: a general JSONPath engine admits unsupported syntax; interpreting path strings on every transition repeats validation; arbitrary Value cloning/serialization can allocate far beyond the output quota. +- **Decision**: compile a bounded winnow subset into private typed segments; distinguish missing/single/selection results, borrow immutable payloads and materialize through a bounded writer. Validate result-target roots and prohibit array growth. Construct from borrowed trees only after structural/encoded checks; give the internal context separate byte/depth/member headroom while retaining normal limits on selected outputs. +- **Why**: one pure implementation pins data behavior for future compiler/engine consumers without coupling core to HTTP, an SDK or Docker. Original business input bytes survive for name idempotency. +- **Pinned by**: [ASL§2.1.1](./rustack-workflows-30-asl-design.md#211-executable-path-and-payload-foundation), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Reuse the already resolved winnow1.0.4 and shared JSON/bytes/serde dependencies. This is a semantic foundation, not engine acceptance. From ff6f36533b56ad75cd372faa9c3975c7318303a2 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 07:06:03 -0700 Subject: [PATCH 035/106] phase 2: compile bounded Choice predicates with exact timestamp ordering Implement ASL section 2.4.1 and D43: literal and Path comparisons, short-circuit logic, exact fractional timestamps, conservative float boundaries and Unicode glob matching under shared budgets. Independent review closes W108-W109. Enable existing JSON float_roundtrip without a new package/version. 125 core unit tests, 15 doctests and strict checks pass. Standard workspace gates pass with 2630 tests/415 gated in the current worktree, plus model regeneration and actual signed SDK transport regressions. B06 existing zstd audit/deny failures remain open; no ignores added. Compiler/registry/engine and complete phase acceptance remain required. --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/rustack-stepfunctions-core/Cargo.toml | 1 + .../rustack-stepfunctions-core/src/choice.rs | 502 ++++++++++++++++++ .../src/choice/glob.rs | 85 +++ .../src/choice/tests.rs | 456 ++++++++++++++++ .../rustack-stepfunctions-core/src/error.rs | 6 + crates/rustack-stepfunctions-core/src/lib.rs | 2 + .../rustack-stepfunctions-core/src/payload.rs | 13 + specs/index.md | 2 +- specs/rustack-workflows-30-asl-design.md | 10 + .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 1 + specs/rustack-workflows-93-review.md | 11 + specs/rustack-workflows-99-decisions.md | 9 + 15 files changed, 1105 insertions(+), 2 deletions(-) create mode 100644 crates/rustack-stepfunctions-core/src/choice.rs create mode 100644 crates/rustack-stepfunctions-core/src/choice/glob.rs create mode 100644 crates/rustack-stepfunctions-core/src/choice/tests.rs diff --git a/Cargo.lock b/Cargo.lock index ca410fd6..b533c6dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4195,6 +4195,7 @@ name = "rustack-stepfunctions-core" version = "0.9.0" dependencies = [ "bytes", + "chrono", "proptest", "rstest", "rustack-core", diff --git a/Cargo.toml b/Cargo.toml index 1f948cf5..91cb5e27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,7 +125,7 @@ url = "2.5.8" # Serialization bollard-stubs = { version = "=1.53.1-rc.29.3.1", default-features = false } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["float_roundtrip"] } ciborium = "0.2" quick-xml = "0.41" zstd = { version = "0.13.3", default-features = false } diff --git a/crates/rustack-stepfunctions-core/Cargo.toml b/crates/rustack-stepfunctions-core/Cargo.toml index b3618a76..75adfc15 100644 --- a/crates/rustack-stepfunctions-core/Cargo.toml +++ b/crates/rustack-stepfunctions-core/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] rustack-core = { workspace = true } bytes = { workspace = true } +chrono = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/rustack-stepfunctions-core/src/choice.rs b/crates/rustack-stepfunctions-core/src/choice.rs new file mode 100644 index 00000000..dd5d3857 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice.rs @@ -0,0 +1,502 @@ +//! Compiled short-circuit Choice predicates with shared evaluation budgets. + +use std::{borrow::Cow, cmp::Ordering, fmt}; + +use chrono::{DateTime, Timelike}; +use rustack_core::json::{JsonLimits, from_slice}; +use serde_json::{Number, Value}; + +use crate::{AslError, EvaluationContext, Payload, ReferencePath, payload::validate_predicate}; + +mod glob; +use glob::Glob; + +/// Immutable compiled Choice predicate; rule transition targets are compiler-owned. +#[derive(Clone)] +#[non_exhaustive] +pub struct Predicate(Node); + +#[derive(Clone)] +enum Node { + And(Vec), + Or(Vec), + Not(Box), + Compare { + variable: ReferencePath, + comparison: Comparison, + }, +} + +#[derive(Clone)] +enum Comparison { + Test(Test, bool), + String(Order, Operand), + Numeric(Order, Operand), + Timestamp(Order, Operand), + Boolean(Operand), + Matches(Glob), +} + +#[derive(Clone)] +enum Operand { + Literal(T), + Path(ReferencePath), +} + +enum Resolved<'a, T> { + Literal(&'a T), + Value(&'a Value), +} + +enum Family { + String, + Numeric, + Timestamp, +} + +#[derive(Clone, Copy)] +enum Test { + Present, + Null, + String, + Numeric, + Boolean, + Timestamp, +} + +#[derive(Clone, Copy)] +enum Order { + Equal, + Less, + Greater, + LessEqual, + GreaterEqual, +} + +#[derive(Clone)] +struct Timestamp { + seconds: i64, + fraction: String, + source_bytes: usize, +} + +impl Timestamp { + fn compare(&self, other: &Self) -> Ordering { + self.seconds + .cmp(&other.seconds) + .then_with(|| self.fraction.cmp(&other.fraction)) + } +} + +impl Order { + fn accepts(self, ordering: Ordering) -> bool { + match self { + Self::Equal => ordering.is_eq(), + Self::Less => ordering.is_lt(), + Self::Greater => ordering.is_gt(), + Self::LessEqual => !ordering.is_gt(), + Self::GreaterEqual => !ordering.is_lt(), + } + } +} + +impl fmt::Debug for Predicate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Predicate()") + } +} + +impl Predicate { + /// Preflight source JSON before compiling a predicate without its Next field. + /// + /// # Errors + /// Rejects duplicate/unknown fields, invalid operators and structural limits. + /// + /// ``` + /// use rustack_stepfunctions_core::Predicate; + /// let predicate = Predicate::parse(br#"{"Variable":"$.n","NumericEquals":3}"#)?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn parse(input: &[u8]) -> Result { + let limits = JsonLimits::new(1024 * 1024, 64, 4096, 262_144, 262_144)?; + let value: Value = from_slice(input, limits)?; + Self::compile(&value) + } + + /// Compile an existing predicate tree, borrowing before validation/cloning. + /// + /// # Errors + /// Rejects invalid predicates, more than1,024 nodes or32 predicate levels. + /// + /// ``` + /// use rustack_stepfunctions_core::Predicate; + /// let predicate = Predicate::compile(&serde_json::json!({"Variable":"$.x","IsPresent":true}))?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn compile(value: &Value) -> Result { + validate_predicate(value)?; + Self::node(value, 0, &mut 1024) + } + + /// Evaluate left-to-right against validated data and internal context. + /// + /// # Errors + /// Missing references (except `IsPresent`), unsupported numeric ranges or + /// shared work/string-byte exhaustion fail explicitly, without coercion. + /// + /// ``` + /// use rustack_stepfunctions_core::{Predicate, Payload, EvaluationContext}; + /// let data = Payload::parse(br#"{"n":3}"#)?; + /// let predicate = Predicate::compile(&serde_json::json!({"Variable":"$.n","NumericEquals":3}))?; + /// assert!(predicate.evaluate(&data,&EvaluationContext::from(&data))?); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn evaluate(&self, data: &Payload, context: &EvaluationContext) -> Result { + self.eval( + data, + context, + &mut Budget { + units: 10_000, + bytes: 1024 * 1024, + }, + ) + } + + fn node(value: &Value, depth: usize, remaining: &mut usize) -> Result { + if depth >= 32 { + return Err(AslError::InvalidPredicate); + } + *remaining = remaining.checked_sub(1).ok_or(AslError::InvalidPredicate)?; + let object = value.as_object().ok_or(AslError::InvalidPredicate)?; + for (key, is_and) in [("And", true), ("Or", false)] { + if let Some(children) = object.get(key) { + if object.len() != 1 { + return Err(AslError::InvalidPredicate); + } + let children = children + .as_array() + .filter(|v| !v.is_empty() && v.len() <= 128) + .ok_or(AslError::InvalidPredicate)?; + let compiled = children + .iter() + .map(|v| Self::node(v, depth.saturating_add(1), remaining)) + .collect::, _>>()?; + return Ok(Self(if is_and { + Node::And(compiled) + } else { + Node::Or(compiled) + })); + } + } + if let Some(child) = object.get("Not") { + if object.len() != 1 { + return Err(AslError::InvalidPredicate); + } + return Ok(Self(Node::Not(Box::new(Self::node( + child, + depth.saturating_add(1), + remaining, + )?)))); + } + if object.len() != 2 { + return Err(AslError::InvalidPredicate); + } + let variable = object + .get("Variable") + .and_then(Value::as_str) + .ok_or(AslError::InvalidPredicate)? + .parse()?; + let (operator, value) = object + .iter() + .find(|(k, _)| k.as_str() != "Variable") + .ok_or(AslError::InvalidPredicate)?; + Ok(Self(Node::Compare { + variable, + comparison: comparison(operator, value)?, + })) + } + + fn eval( + &self, + data: &Payload, + context: &EvaluationContext, + budget: &mut Budget, + ) -> Result { + budget.step()?; + match &self.0 { + Node::And(children) => { + for child in children { + if !child.eval(data, context, budget)? { + return Ok(false); + } + } + Ok(true) + } + Node::Or(children) => { + for child in children { + if child.eval(data, context, budget)? { + return Ok(true); + } + } + Ok(false) + } + Node::Not(child) => Ok(!child.eval(data, context, budget)?), + Node::Compare { + variable, + comparison, + } => { + let value = variable.lookup(data, context); + if let Comparison::Test(Test::Present, expected) = comparison { + return Ok(value.is_some() == *expected); + } + evaluate( + comparison, + value.ok_or(AslError::Missing)?, + data, + context, + budget, + ) + } + } + } +} + +fn comparison(operator: &str, value: &Value) -> Result { + let test = match operator { + "IsPresent" => Some(Test::Present), + "IsNull" => Some(Test::Null), + "IsString" => Some(Test::String), + "IsNumeric" => Some(Test::Numeric), + "IsBoolean" => Some(Test::Boolean), + "IsTimestamp" => Some(Test::Timestamp), + _ => None, + }; + if let Some(test) = test { + return Ok(Comparison::Test( + test, + value.as_bool().ok_or(AslError::InvalidPredicate)?, + )); + } + if operator == "StringMatches" { + return Ok(Comparison::Matches(Glob::compile( + value.as_str().ok_or(AslError::InvalidPredicate)?, + )?)); + } + let (name, path) = operator + .strip_suffix("Path") + .map_or((operator, false), |s| (s, true)); + if name == "BooleanEquals" { + return Ok(Comparison::Boolean(operand(value, path, Value::as_bool)?)); + } + let (family, suffix) = if let Some(s) = name.strip_prefix("String") { + (Family::String, s) + } else if let Some(s) = name.strip_prefix("Numeric") { + (Family::Numeric, s) + } else if let Some(s) = name.strip_prefix("Timestamp") { + (Family::Timestamp, s) + } else { + return Err(AslError::InvalidPredicate); + }; + let order = match suffix { + "Equals" => Order::Equal, + "LessThan" => Order::Less, + "GreaterThan" => Order::Greater, + "LessThanEquals" => Order::LessEqual, + "GreaterThanEquals" => Order::GreaterEqual, + _ => return Err(AslError::InvalidPredicate), + }; + match family { + Family::String => Ok(Comparison::String( + order, + operand(value, path, |v| v.as_str().map(str::to_owned))?, + )), + Family::Numeric => Ok(Comparison::Numeric( + order, + operand(value, path, |v| number(v).ok().flatten())?, + )), + Family::Timestamp => Ok(Comparison::Timestamp( + order, + operand(value, path, |v| v.as_str().and_then(timestamp))?, + )), + } +} + +fn operand( + value: &Value, + path: bool, + literal: impl FnOnce(&Value) -> Option, +) -> Result, AslError> { + if path { + Ok(Operand::Path( + value.as_str().ok_or(AslError::InvalidPredicate)?.parse()?, + )) + } else { + Ok(Operand::Literal( + literal(value).ok_or(AslError::InvalidPredicate)?, + )) + } +} + +fn resolve<'a, T>( + operand: &'a Operand, + data: &'a Payload, + context: &'a EvaluationContext, +) -> Result, AslError> { + match operand { + Operand::Literal(value) => Ok(Resolved::Literal(value)), + Operand::Path(path) => Ok(Resolved::Value( + path.lookup(data, context).ok_or(AslError::Missing)?, + )), + } +} + +fn number(value: &Value) -> Result, AslError> { + const LIMIT: f64 = 9_007_199_254_740_992.0; + if let Some(n) = value.as_u64() { + if n > 9_007_199_254_740_992 { + return Err(AslError::NumericRange); + } + } + if let Some(n) = value.as_i64() { + if !(-9_007_199_254_740_992..=9_007_199_254_740_992).contains(&n) { + return Err(AslError::NumericRange); + } + } + let result = value.as_f64(); + let floating = value.as_number().is_some_and(Number::is_f64); + if result.is_some_and(|n| !n.is_finite() || n.abs() > LIMIT || (floating && n.abs() == LIMIT)) { + return Err(AslError::NumericRange); + } + Ok(result) +} + +fn timestamp(value: &str) -> Option { + let date = DateTime::parse_from_rfc3339(value).ok()?; + if date.nanosecond() >= 1_000_000_000 { + None + } else { + let fraction = if let Some(dot) = value.find('.') { + let tail = value.get(dot.checked_add(1)?..)?; + let end = tail + .find(|ch: char| !ch.is_ascii_digit()) + .unwrap_or(tail.len()); + tail.get(..end)?.trim_end_matches('0') + } else { + "" + }; + Some(Timestamp { + seconds: date.timestamp(), + fraction: fraction.to_owned(), + source_bytes: value.len(), + }) + } +} + +fn evaluate( + comparison: &Comparison, + value: &Value, + data: &Payload, + context: &EvaluationContext, + budget: &mut Budget, +) -> Result { + match comparison { + Comparison::Test(test, expected) => { + let actual = match test { + Test::Present => true, + Test::Null => value.is_null(), + Test::String => value.is_string(), + Test::Numeric => value.is_number(), + Test::Boolean => value.is_boolean(), + Test::Timestamp => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text).is_some() + } else { + false + } + } + }; + Ok(actual == *expected) + } + Comparison::Boolean(operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(*value), + Resolved::Value(value) => value.as_bool(), + }; + Ok(value.as_bool().zip(expected).is_some_and(|(a, b)| a == b)) + } + Comparison::Numeric(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(*value), + Resolved::Value(value) => number(value)?, + }; + Ok(number(value)? + .zip(expected) + .and_then(|(a, b)| a.partial_cmp(&b)) + .is_some_and(|ordering| order.accepts(ordering))) + } + Comparison::String(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => Some(value.as_str()), + Resolved::Value(value) => value.as_str(), + }; + if let Some((a, b)) = value.as_str().zip(expected) { + budget.text(a.len().saturating_add(b.len()))?; + Ok(order.accepts(a.cmp(b))) + } else { + Ok(false) + } + } + Comparison::Timestamp(order, operand) => { + let expected = match resolve(operand, data, context)? { + Resolved::Literal(value) => { + budget.text(value.source_bytes)?; + Some(Cow::Borrowed(value)) + } + Resolved::Value(value) => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text).map(Cow::Owned) + } else { + None + } + } + }; + let actual = if let Some(text) = value.as_str() { + budget.text(text.len())?; + timestamp(text) + } else { + None + }; + Ok(actual + .zip(expected) + .is_some_and(|(a, b)| order.accepts(a.compare(&b)))) + } + Comparison::Matches(glob) => { + if let Some(text) = value.as_str() { + budget.text(text.len())?; + glob.matches(text, budget) + } else { + Ok(false) + } + } + } +} + +struct Budget { + units: usize, + bytes: usize, +} +impl Budget { + fn step(&mut self) -> Result<(), AslError> { + self.units = self.units.checked_sub(1).ok_or(AslError::WorkLimit)?; + Ok(()) + } + fn text(&mut self, bytes: usize) -> Result<(), AslError> { + self.bytes = self.bytes.checked_sub(bytes).ok_or(AslError::WorkLimit)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-core/src/choice/glob.rs b/crates/rustack-stepfunctions-core/src/choice/glob.rs new file mode 100644 index 00000000..59711678 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice/glob.rs @@ -0,0 +1,85 @@ +//! Bounded Unicode glob, independent of user regular expressions. + +use super::Budget; +use crate::AslError; + +#[derive(Clone)] +pub(super) struct Glob(Vec); +#[derive(Clone, Copy)] +enum Token { + Literal(char), + Star, +} + +impl Glob { + pub(super) fn compile(pattern: &str) -> Result { + if pattern.len() > 4096 { + return Err(AslError::InvalidPredicate); + } + let mut tokens = Vec::new(); + let mut chars = pattern.chars(); + while let Some(ch) = chars.next() { + let token = match ch { + '*' => { + if matches!(tokens.last(), Some(Token::Star)) { + continue; + } + Token::Star + } + '\\' => match chars.next() { + Some(ch @ ('*' | '\\')) => Token::Literal(ch), + _ => return Err(AslError::InvalidPredicate), + }, + ch => Token::Literal(ch), + }; + if tokens.len() >= 1024 { + return Err(AslError::InvalidPredicate); + } + tokens.push(token); + } + Ok(Self(tokens)) + } + + pub(super) fn matches(&self, text: &str, budget: &mut Budget) -> Result { + let mut position = 0_usize; + let mut token = 0_usize; + let mut star = None; + while let Some(ch) = text.get(position..).and_then(|s| s.chars().next()) { + budget.step()?; + match self.0.get(token) { + Some(Token::Literal(expected)) if *expected == ch => { + position = position + .checked_add(ch.len_utf8()) + .ok_or(AslError::WorkLimit)?; + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + } + Some(Token::Star) => { + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + if token == self.0.len() { + return Ok(true); + } + star = Some((token, position)); + } + _ => { + let Some((after, backup)) = star else { + return Ok(false); + }; + let Some(next) = text.get(backup..).and_then(|s| s.chars().next()) else { + return Ok(false); + }; + let advanced = backup + .checked_add(next.len_utf8()) + .ok_or(AslError::WorkLimit)?; + position = advanced; + token = after; + star = Some((after, advanced)); + } + } + } + while matches!(self.0.get(token), Some(Token::Star)) { + budget.step()?; + token = token.checked_add(1).ok_or(AslError::WorkLimit)?; + } + Ok(token == self.0.len()) + } +} diff --git a/crates/rustack-stepfunctions-core/src/choice/tests.rs b/crates/rustack-stepfunctions-core/src/choice/tests.rs new file mode 100644 index 00000000..a8364c6f --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/choice/tests.rs @@ -0,0 +1,456 @@ +use std::error::Error; + +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::json::JsonError; +use serde_json::{Value, json}; + +use crate::{AslError, EvaluationContext, Payload, Predicate}; + +type TestResult = Result<(), Box>; + +fn evaluate(rule: &Value, data: &Value) -> Result { + let predicate = Predicate::compile(rule)?; + let data = Payload::from_value(data)?; + predicate.evaluate(&data, &EvaluationContext::from(&data)) +} + +#[test] +fn test_should_execute_all_independent_choice_catalog_examples() -> TestResult { + let catalog: Value = serde_json::from_str(include_str!( + "../../../../tests/workflows/semantic-cases.json" + ))?; + let mut count = 0; + for case in catalog["cases"].as_array().ok_or("missing cases")? { + if case["kind"] != "choice" { + continue; + } + assert_eq!( + evaluate(&case["rule"], &case["input"].clone())?, + case["expected"].as_bool().ok_or("missing expected")? + ); + count += 1; + } + assert_eq!(count, 4); + Ok(()) +} + +#[test] +fn test_should_cover_all_ordered_literal_and_path_operator_variants() -> TestResult { + let families = [ + ("String", vec![json!("a"), json!("b"), json!("c")]), + ("Numeric", vec![json!(-1), json!(0), json!(1)]), + ( + "Timestamp", + vec![ + json!("2020-01-01T00:00:00Z"), + json!("2020-01-02T00:00:00Z"), + json!("2020-01-03T00:00:00Z"), + ], + ), + ]; + let operators = [ + ("Equals", [false, true, false]), + ("LessThan", [true, false, false]), + ("GreaterThan", [false, false, true]), + ("LessThanEquals", [true, true, false]), + ("GreaterThanEquals", [false, true, true]), + ]; + for (family, values) in families { + let middle = values.get(1).ok_or("missing middle")?; + for (suffix, expected) in operators { + for path in [false, true] { + for (index, left) in values.iter().enumerate() { + let op = format!("{family}{suffix}{}", if path { "Path" } else { "" }); + let operand = if path { + json!("$.right") + } else { + middle.clone() + }; + let rule = json!({"Variable":"$.left",op:operand}); + assert_eq!( + evaluate(&rule, &json!({"left":left,"right":middle}))?, + *expected.get(index).ok_or("missing expected")?, + "{family} {suffix} {path} {index}" + ); + } + } + } + } + for path in [false, true] { + for value in [false, true] { + let op = if path { + "BooleanEqualsPath" + } else { + "BooleanEquals" + }; + let operand = if path { json!("$.right") } else { json!(true) }; + assert_eq!( + evaluate( + &json!({"Variable":"$.left",op:operand}), + &json!({"left":value,"right":true}) + )?, + value + ); + } + } + Ok(()) +} + +#[rstest] +#[case("IsPresent",json!(null),true)] +#[case("IsNull",json!(null),true)] +#[case("IsString",json!("7"),true)] +#[case("IsNumeric",json!(7),true)] +#[case("IsBoolean",json!(true),true)] +#[case("IsTimestamp",json!("2020-01-01T00:00:00Z"),true)] +#[case("IsNull",json!(0),false)] +#[case("IsString",json!(7),false)] +#[case("IsNumeric",json!("7"),false)] +#[case("IsBoolean",json!("true"),false)] +#[case("IsTimestamp",json!("invalid"),false)] +#[case("IsTimestamp",json!("2016-12-31T23:59:60Z"),false)] +fn test_should_apply_type_tests_without_coercion( + #[case] op: &str, + #[case] value: Value, + #[case] expected: bool, +) -> TestResult { + for requested in [true, false] { + assert_eq!( + evaluate(&json!({"Variable":"$.x",op:requested}), &json!({"x":value}))?, + expected == requested + ); + } + Ok(()) +} + +#[test] +fn test_should_short_circuit_in_definition_order_and_preserve_missing_errors() -> TestResult { + let present = json!({"Variable":"$.absent","IsPresent":true}); + let missing = json!({"Variable":"$.absent","IsString":true}); + assert!(!evaluate(&json!({"And":[present,missing]}), &json!({}))?); + assert!(evaluate( + &json!({"Or":[{"Not":present},missing]}), + &json!({}) + )?); + assert!(matches!( + evaluate(&json!({"And":[missing,present]}), &json!({})), + Err(AslError::Missing) + )); + for op in [ + "IsNull", + "IsString", + "IsNumeric", + "IsBoolean", + "IsTimestamp", + ] { + assert!(matches!( + evaluate(&json!({"Variable":"$.absent",op:false}), &json!({})), + Err(AslError::Missing) + )); + } + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","StringEqualsPath":"$.absent"}), + &json!({"x":1}) + ), + Err(AslError::Missing) + )); + Ok(()) +} + +#[test] +fn test_should_compare_instants_and_avoid_numeric_precision_aliasing() -> TestResult { + assert!(!evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.1234567891Z"}), + &json!({"x":"2020-01-01T00:00:00.1234567892Z"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.1234567891000Z"}), + &json!({"x":"2020-01-01T01:00:00.1234567891+01:00"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00.123Z"}), + &json!({"x":"2020-01-01T01:00:00.123+01:00"}) + )?); + assert!(!evaluate( + &json!({"Variable":"$.x","TimestampEquals":"2020-01-01T00:00:00Z"}), + &json!({"x":"2020-01-01"}) + )?); + assert!(evaluate( + &json!({"Variable":"$.x","NumericEquals":9_007_199_254_740_992_u64}), + &json!({"x":9_007_199_254_740_992_u64}) + )?); + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","NumericEquals":9_007_199_254_740_992_u64}), + &json!({"x":9_007_199_254_740_993_u64}) + ), + Err(AslError::NumericRange) + )); + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","NumericEqualsPath":"$.right"}), + &json!({"x":0,"right":-9_007_199_254_740_993_i64}) + ), + Err(AslError::NumericRange) + )); + assert!(evaluate( + &json!({"Variable":"$.x","IsNumeric":true}), + &json!({"x":u64::MAX}) + )?); + for (op, operand, left) in [ + ("StringEquals", json!("1"), json!(1)), + ("NumericEquals", json!(1), json!("1")), + ("BooleanEquals", json!(true), json!(1)), + ] { + assert!(!evaluate( + &json!({"Variable":"$.x",op:operand}), + &json!({"x":left}) + )?); + } + let predicate = + Predicate::compile(&json!({"Variable":"$$.State.Name","StringEquals":"current"}))?; + assert!(predicate.evaluate( + &Payload::parse(b"null")?, + &EvaluationContext::from_value(&json!({"State":{"Name":"current"}}))? + )?); + Ok(()) +} + +#[test] +fn test_should_reject_raw_float_boundary_lexemes_instead_of_aliasing_large_numbers() -> TestResult { + let integer = Predicate::parse(br#"{"Variable":"$.x","NumericEquals":9007199254740992}"#)?; + for text in [ + "9007199254740993.0", + "9007199254740993e0", + "9007199254740992.0", + "-9007199254740993.0", + "-9007199254740993e0", + ] { + let data = Payload::parse(format!("{{\"x\":{text}}}").as_bytes())?; + assert!( + matches!( + integer.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::NumericRange) + ), + "{text}" + ); + assert!( + matches!( + Predicate::parse( + format!("{{\"Variable\":\"$.x\",\"NumericEquals\":{text}}}").as_bytes() + ), + Err(AslError::InvalidPredicate) + ), + "{text}" + ); + } + let inside = Predicate::parse(br#"{"Variable":"$.x","NumericEquals":9007199254740991.0}"#)?; + let data = Payload::parse(br#"{"x":9007199254740991e0}"#)?; + assert!(inside.evaluate(&data, &EvaluationContext::from(&data))?); + Ok(()) +} + +#[test] +fn test_should_charge_precompiled_timestamp_literal_bytes_to_shared_budget() -> TestResult { + let actual = format!("2020-01-01T00:00:00.{}Z", "0".repeat(180_000)); + let expected = format!("2020-01-01T00:00:00.{}1Z", "0".repeat(180_000)); + let comparison = json!({"Variable":"$.x","TimestampLessThan":expected}); + let data = Payload::from_value(&json!({"x":actual}))?; + let two = Predicate::compile(&json!({"And":[comparison,comparison]}))?; + assert!(two.evaluate(&data, &EvaluationContext::from(&data))?); + let three = Predicate::compile(&json!({"And":[comparison,comparison,comparison]}))?; + assert!(matches!( + three.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::WorkLimit) + )); + let timestamp = format!("2020-01-01T00:00:00.{}1Z", "0".repeat(220_000)); + let comparison = json!({"Variable":"$.x","TimestampLessThan":timestamp}); + let rule = json!({"And":[comparison,comparison,comparison,comparison,comparison]}); + assert!(Predicate::compile(&rule).is_err()); + // A shared Path operand supplies the same long fraction repeatedly without + // exceeding the separately bounded source definition. + let comparison = json!({"Variable":"$.x","TimestampLessThanPath":"$.right"}); + let data = Payload::from_value(&json!({"x":"2020-01-01T00:00:00Z","right":timestamp}))?; + let rule = Predicate::compile( + &json!({"And":[comparison,comparison,comparison,comparison,comparison]}), + )?; + assert!(matches!( + rule.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +#[rstest] +#[case("", "", true)] +#[case("*", "", true)] +#[case("**", "中🙂", true)] +#[case("中*文", "中🙂文", true)] +#[case("a\\*b", "a*b", true)] +#[case("a\\\\b", "a\\b", true)] +#[case("a?b", "a?b", true)] +#[case("a?b", "axb", false)] +#[case("*aba*b", "ababab", true)] +#[case("a*b*c", "abcc", true)] +#[case("a*b*c", "abdcx", false)] +#[case("literal", "Literal", false)] +fn test_should_match_compiled_unicode_globs( + #[case] pattern: &str, + #[case] input: &str, + #[case] expected: bool, +) -> TestResult { + assert_eq!( + evaluate( + &json!({"Variable":"$.x","StringMatches":pattern}), + &json!({"x":input}) + )?, + expected + ); + Ok(()) +} + +#[rstest] +#[case(json!({}))] +#[case(json!([]))] +#[case(json!({"And":[]}))] +#[case(json!({"Or":[]}))] +#[case(json!({"And":[],"Not":{}}))] +#[case(json!({"Variable":"$.x","IsNull":"true"}))] +#[case(json!({"Variable":"$.x","NumericEquals":true}))] +#[case(json!({"Variable":"$.x","NumericEquals":u64::MAX}))] +#[case(json!({"Variable":"$.x","TimestampEquals":"invalid"}))] +#[case(json!({"Variable":"$.x","StringEquals":7}))] +#[case(json!({"Variable":"$.x","StringMatches":"\\q"}))] +#[case(json!({"Variable":"$.x","StringMatches":"\\"}))] +#[case(json!({"Variable":"$.x","IsNull":true,"IsString":false}))] +#[case(json!({"Variable":"$.x","Unknown":true}))] +#[case(json!({"Variable":"$.x","StringMatchesPath":"$.y"}))] +fn test_should_reject_ambiguous_invalid_or_unknown_predicates(#[case] value: Value) { + assert!(matches!( + Predicate::compile(&value), + Err(AslError::InvalidPredicate) + )); +} + +#[test] +fn test_should_bound_source_structure_compiled_nodes_depth_and_glob_tokens() -> TestResult { + assert!(matches!( + Predicate::parse(br#"{"Variable":"$.a","IsNull":true,"IsNull":false}"#), + Err(AslError::Json(JsonError::Duplicate)) + )); + assert!(Predicate::compile(&json!({"Variable":"$[*]","IsNull":true})).is_err()); + assert!(Predicate::compile(&json!({"Variable":"$.x","StringEqualsPath":"$[*]"})).is_err()); + let leaf = json!({"Variable":"$.x","IsPresent":true}); + let mut nested = leaf.clone(); + for _ in 0..31 { + nested = json!({"Not":nested}); + } + Predicate::compile(&nested)?; + assert!(matches!( + Predicate::compile(&json!({"Not":nested})), + Err(AslError::InvalidPredicate) + )); + Predicate::compile(&json!({"And":vec![leaf.clone();128]}))?; + assert!(matches!( + Predicate::compile(&json!({"And":vec![leaf.clone();129]})), + Err(AslError::InvalidPredicate) + )); + let group = json!({"And":vec![leaf;8]}); + assert!(matches!( + Predicate::compile(&json!({"And":vec![group;128]})), + Err(AslError::InvalidPredicate) + )); + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"🙂".repeat(1024)}))?; + assert!(matches!( + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"a".repeat(1025)})), + Err(AslError::InvalidPredicate) + )); + assert!(matches!( + Predicate::compile(&json!({"Variable":"$.x","StringMatches":"🙂".repeat(1025)})), + Err(AslError::InvalidPredicate) + )); + Ok(()) +} + +#[test] +fn test_should_share_step_and_string_byte_limits_across_the_complete_predicate() -> TestResult { + assert!(matches!( + evaluate( + &json!({"Variable":"$.x","StringMatches":"*b"}), + &json!({"x":"a".repeat(20_000)}) + ), + Err(AslError::WorkLimit) + )); + assert!(evaluate( + &json!({"Variable":"$.x","StringMatches":"a*"}), + &json!({"x":"a".repeat(20_000)}) + )?); + let comparison = json!({"Variable":"$.x","StringEqualsPath":"$.x"}); + let input = json!({"x":"a".repeat(260_000)}); + assert!(evaluate( + &json!({"And":[comparison,comparison]}), + &input.clone() + )?); + assert!(matches!( + evaluate(&json!({"And":[comparison,comparison,comparison]}), &input), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +#[test] +fn test_should_reject_borrowed_extreme_predicate_without_recursive_drop_and_redact() -> TestResult { + let mut tree = Value::Null; + for _ in 0..50_000 { + tree = Value::Array(vec![tree]); + } + let rejected = Predicate::compile(&tree); + let mut owned = vec![tree]; + while let Some(value) = owned.pop() { + if let Value::Array(mut children) = value { + owned.append(&mut children); + } + } + assert!(matches!(rejected, Err(AslError::Json(JsonError::Depth)))); + let predicate = Predicate::compile(&json!({"Variable":"$.secret","StringEquals":"private"}))?; + assert_eq!(format!("{predicate:?}"), "Predicate()"); + Ok(()) +} + +fn glob_oracle(pattern: &str, text: &str) -> bool { + let chars: Vec<_> = text.chars().collect(); + let mut row = vec![false; chars.len() + 1]; + row[0] = true; + for ch in pattern.chars() { + let mut next = vec![false; row.len()]; + if ch == '*' { + next[0] = row[0]; + } + for (i, input) in chars.iter().enumerate() { + next[i + 1] = if ch == '*' { + next[i] || row[i + 1] + } else { + row[i] && ch == *input + }; + } + row = next; + } + row.last().copied().unwrap_or(false) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + #[test] + fn test_should_match_an_independent_glob_dynamic_program(pattern in "[ab*]{0,12}", text in "[ab]{0,18}") { + let actual=evaluate(&json!({"Variable":"$.x","StringMatches":pattern}),&json!({"x":text})).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(actual,glob_oracle(&pattern,&text)); + } + + #[test] + fn test_should_preserve_exact_integer_order_inside_the_supported_range(a in -1_000_000_i64..1_000_000, b in -1_000_000_i64..1_000_000) { + let actual=evaluate(&json!({"Variable":"$.a","NumericLessThanPath":"$.b"}),&json!({"a":a,"b":b})).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(actual,a Result<(), AslError> { Ok(()) } +pub(crate) fn validate_predicate(value: &Value) -> Result<(), AslError> { + validate_value( + value, + TreeLimits { + bytes: 1024 * 1024, + depth: 64, + members: 4096, + }, + )?; + encode_with_limit(value, 1024 * 1024)?; + Ok(()) +} + fn check_container( depth: usize, length: usize, diff --git a/specs/index.md b/specs/index.md index 6d75b7ba..1483a06e 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. The bounded compiled path/payload foundation is implemented, with separate context headroom and independent review. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Bounded compiled paths/payloads and Choice predicates are implemented and independently reviewed, including context headroom, exact timestamp fractions and conservative numeric comparisons. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-30-asl-design.md b/specs/rustack-workflows-30-asl-design.md index 5ed67a92..61fd6409 100644 --- a/specs/rustack-workflows-30-asl-design.md +++ b/specs/rustack-workflows-30-asl-design.md @@ -99,6 +99,16 @@ Choice supports short-circuit And/Or/Not, IsPresent/IsNull/IsString/IsNumeric/Is Wait Seconds is nonnegative; timestamps must be valid RFC3339. Dynamic type errors fail. Past timestamps finish immediately. Apply AWS second-level timestamp truncation, compute remaining duration using wall time once, then use a monotonic timer to avoid NTP jumps. Overall deadlines still bound long waits. No dedicated sleeping thread per state. +#### 2.4.1 Executable Choice predicate contract + +Compile a Predicate independently of its containing rule's Next target; the machine compiler owns the target. A predicate is either a sole And/Or array (1–128 children), a sole Not object, or exactly Variable plus one supported comparison/test. Reject unknown/duplicate operators and invalid literal types, paths and patterns before execution. Variable and comparison Path operands are ReferencePaths. The public constructor borrows a Value and validates bounded structure before any recursive compilation:1MiB encoded JSON, depth64,4,096 members,262,144-byte strings/keys. Limit the resulting predicate to1,024 nodes and32 nested predicate levels; these local limits must become compiler diagnostics. + +And/Or evaluate left-to-right and short-circuit, including errors in unvisited operands. IsPresent alone safely observes a missing reference; every other test/comparison raises a typed missing-reference error. Resolve both comparison references before type comparison; present values of the wrong type yield false. IsNumeric tests the JSON number type. Numeric comparisons accept integers in[-2^53,2^53] and floating-point values strictly inside(-2^53,2^53). Reject boundary floats conservatively because an original out-of-range decimal/exponent lexeme can round to exactly2^53; retaining only its parsed f64 cannot distinguish it from an exact boundary float. Reject other out-of-range values rather than silently aliasing large integers. Enable the existing serde_json float_roundtrip feature centrally so equivalent interior decimal/exponent forms convert correctly to binary64; arbitrary_precision remains disabled. Compile-time numeric literals outside the range reject. Boolean/string comparison never coerces another type. Timestamp literals parse at compilation; dynamic invalid timestamps compare false. Validate RFC3339 offsets with chrono, then represent the UTC whole seconds and the complete decimal fraction separately; remove trailing fractional zeros and compare exact fractions beyond nanosecond precision. Reject leap-second representations and do not equate complete timestamp strings lexically. Debit original literal and dynamic timestamp byte lengths before parsing/comparison. + +StringMatches compiles a bounded glob: at most4,096 UTF-8 pattern bytes and1,024 tokens, literal Unicode scalars, `*` for zero or more scalars and only `\\*`/`\\\\` escapes. Braces/question marks are ordinary characters; reject dangling/unknown escapes. Collapse consecutive stars. Evaluate with checked UTF-8 offsets, no user regex and no unbounded recursion. A trailing star can accept the remainder directly. + +Each evaluation has one shared10,000-unit budget for predicate/glob steps and1MiB aggregate compared string/timestamp bytes. Debit before work; exhaustion is a typed WorkLimit failure, never false. This prevents compound comparisons and glob backtracking from monopolizing the execution actor. These are local safety limits, not an AWS parity claim. Keep compiled predicates and diagnostics redacted. Tests cover the supported literal/Path operator matrix, logical order, missing/null/wrong type, timestamp offsets, binary64 boundaries, Unicode glob escapes/work exhaustion and borrowed extreme-depth rejection. Do not claim complete Choice state/compiler/engine acceptance from this evaluator alone. Primary operation names/short-circuit examples: [AWS Choice JSONPath rules](https://docs.aws.amazon.com/step-functions/latest/dg/state-choice.html#choice-state-rules-jsonpath). + ## 3. Execution ownership and scheduling One ExecutionActor owns execution data, all branch frames, attempt generations, retry counters, timers and history. Branches are frames, not an unbounded tree of Tokio tasks. A ready queue drives pure transitions, yielding after at most 64 transitions per tick. Bounded supervised JoinSet futures perform external calls and return typed results; pure evaluation never performs network IO. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 90af50f0..462d90c2 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -180,3 +180,9 @@ The actual Hyper network fixture rejects a chunked request carrying an oversized `make test-stepfunctions-core` exercises actual compiled winnow paths and transactional result merging. All seven independent path catalog cases run against the evaluator. Cover singular missing/null, context roots, deterministic object wildcards, consecutive wildcards/slices, unsupported syntax/escapes/index overflow,1,024-byte/128-segment and10,000-work boundaries, exact256KiB data,64-depth/65,536-member preflight and escaped output expansion. Three512-case properties cover arbitrary UTF-8 parsing, quoted-key round trips and merge preservation. Twelve executable public doctests and strict production Clippy qualify the API. Reject a borrowed50,000-depth Value without owning its recursive teardown; the test caller safely dismantles that tree iteratively. Context tests embed maximum-size/depth/member source payloads plus metadata, including compact exponent input whose normalized number representation expands, mixed/full numeric Map items, exact4MiB internal bytes and+1 rejection. Selected context output retains the normal payload limits. This pure semantic foundation does not satisfy compiler, registry or execution acceptance. + +## Step Functions Choice predicate qualification + +The existing `make test-stepfunctions-core` target includes the compiled predicate evaluator: all90 ordered family/operator/literal-Path combinations, BooleanEquals variants, six type tests, logical short-circuit/error order and four independent Choice catalog cases. Two additional512-case properties compare the Unicode-glob implementation with an independent dynamic-programming oracle and preserve integer order. Validate source duplicates/unknown operators,1,024-node/32-level/128-child and pattern/work/string-byte limits, borrowed50,000-depth rejection, redaction, context references and wrong-type behavior. + +Raw JSON regressions must include decimal/exponent forms at and beyond the conservative float endpoints, their negative counterparts and correctly rounded interior equivalents; constructing json! from already-rounded Rust f64 values is insufficient. Keep integer endpoints valid and IsNumeric a type test. Timestamp cases compare offsets, fractions beyond nine digits and trailing-zero equivalence; valid large literal and Path predicates must exhaust the shared byte budget explicitly. Workspace serde_json float_roundtrip is enabled, so rerun full standard workspace gates and the generated-model/HTTP signed SDK targets for feature-unification regressions. The excluded generator retains its own graph. This qualifies predicates only; state target selection and the complete compiler/execution actor remain required. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index 316bcb27..f005ca98 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -160,6 +160,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 2.1 generated model/timestamps | Implemented/reviewed; HTTP/registry still pending | API§1.1/D40 generates37 official operations from312 pinned shapes with exact numeric required/optional epoch dates and strict DTO policy.5 generator cases,34 model cases,1 doctest, regeneration consistency and strict model/tool checks pass. W97–W100 are fixed/rechecked. Final current-worktree workspace2,447/414 gated, including the retained deferred image checkpoint; build/test/fmt/Clippy/rustdoc, root475/tool29-crate audit and both deny graphs pass. This is protocol/model evidence, not API/engine or real-image workflow acceptance. | | 2.1 typed HTTP transport | Implemented/reviewed; real core/registration pending | API§1.2/D41 implements18 typed ports, bounded envelopes/body/response, strict SigV4 service/region/target binding, exact DTO dates and accepted-command waiter isolation.56 unit tests, two nongated network tests, eight doctests, strict HTTP Clippy and four real signed boto3 calls pass. W101–W104 are fixed and independently rechecked. Current-worktree workspace2,505/415 gated, standard build/test/fmt/Clippy/rustdoc, audit and deny pass. No production fallback handler, main registration or engine acceptance is claimed. | | 2.2 compiled paths/payloads | Implemented; complete compiler remains | ASL§2.1.1/D42 supplies bounded immutable payloads/context, compiled paths, borrowed missing/single/selection and transactional result merge.75 unit tests (three512-case properties),12 doctests and strict production checks pass; W105–W107 corrections cover borrowed deep-tree rejection and numeric/depth/member context headroom. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. New existing-dependency advisory findings are tracked as B06; audit/deny currently fail that baseline, with no ignores added. | +| 2.2 compiled Choice predicates | Implemented/reviewed; state/compiler integration remains | ASL§2.4.1/D43 implements typed literal/Path comparisons, logical short-circuit, exact timestamp fractions, conservative numeric interoperability and bounded glob/shared budgets. Core125 unit cases/15 doctests and strict checks pass, including90 operator combinations and four independent Choice catalog cases. W108–W109 are fixed/rechecked. Shared float_roundtrip adds no external package/version. Final current-worktree standard workspace gates pass with2,630/415 gated; regenerated model/strict HTTP and signed SDK regressions pass. Complete compiler, transition selection and engine remain required; B06 stays open. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index 5db81323..bad3c072 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -502,6 +502,17 @@ ASL§2.1.1/D42 implements bounded immutable business payloads, a separate intern Focused evidence:75 unit tests (including three512-case properties),12 doctests and ordinary/strict core Clippy pass. All seven independent path catalog cases execute against the evaluator. The independent read-only reviewer rechecked all three corrections and reported no remaining in-slice findings. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. Full compiler/registry/engine and M1 acceptance remain required. +## P2 compiled Choice independent review + +ASL§2.4.1/D43 implements compiled predicate semantics, shared evaluation budgets and a bounded Unicode glob. The independent read-only reviewer rechecked the following corrections and reported no remaining in-slice findings. + +| ID | Priority / source | Closed correction | Verification | +|---|---|---|---| +| W108 | P2; `crates/rustack-stepfunctions-core/src/choice.rs` timestamp representation/evaluation | Preserve full normalized decimal fractions beyond chrono nanoseconds; debit cached literal and dynamic source lengths | Distinct10-digit fractions, equivalent offsets/trailing zeros and valid long-literal/Path rules exhausting shared budgets | +| W109 | P2; `crates/rustack-stepfunctions-core/src/choice.rs` numeric comparison | Reject float endpoints conservatively to prevent out-of-range source lexemes aliasing integer2^53; keep integer endpoints valid; enable existing float_roundtrip for correctly rounded interior forms | Raw decimal/exponent positive/negative boundaries and equivalent interior lexemes; type-only IsNumeric remains independent | + +Focused evidence:125 total core unit cases and15 doctests pass, including the90-combination matrix, four independent Choice catalog cases and two new512-case properties. Ordinary all-targets and strict production core Clippy pass. This is predicate evidence, not a complete Choice state or compiler/engine acceptance. Final standard workspace build/test/fmt/Clippy/rustdoc pass with2,630 tests/415 gated, including retained deferred Docker changes. Model regeneration and strict model/HTTP targets plus the real signed SDK fixture pass after shared feature unification. B06 audit/deny failures remain explicitly open. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 607d3db6..287d454d 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -375,3 +375,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: one pure implementation pins data behavior for future compiler/engine consumers without coupling core to HTTP, an SDK or Docker. Original business input bytes survive for name idempotency. - **Pinned by**: [ASL§2.1.1](./rustack-workflows-30-asl-design.md#211-executable-path-and-payload-foundation), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. Reuse the already resolved winnow1.0.4 and shared JSON/bytes/serde dependencies. This is a semantic foundation, not engine acceptance. + +## D43 — Compile Choice predicates and bound evaluation across the whole rule + +- **Context**: branching semantics need validated literal/Path operands and short-circuit behavior before the execution actor interprets Choice states. +- **Alternatives considered**: arbitrary JSON inspection on each transition repeats validation; user regexes complicate resource bounds; binary64 coercion silently aliases large integers; per-child budgets let compounds multiply work without bound. +- **Decision**: compile a private typed predicate tree, preserve left-to-right short-circuit and missing-reference errors, normalize timestamps to instants, use a bounded Unicode glob and share one step/string-byte budget across the complete predicate. +- **Why**: a pure reusable evaluator pins semantics and makes local limits explicit before machine compilation and scheduling. It remains separate from transition-target ownership. +- **Pinned by**: [ASL§2.4.1](./rustack-workflows-30-asl-design.md#241-executable-choice-predicate-contract), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Reuse existing chrono, JSON and reference paths. Numeric integers use the closed±2^53 range; floats exclude its endpoints to avoid source-rounding aliases. Enable shared serde_json float_roundtrip for correctly rounded interior numeric lexemes; no arbitrary_precision or new external version is added. Timestamp fractions retain precision beyond nanoseconds. No complete Choice state/engine acceptance is claimed. From 45ee5eb6828743af26dab94c87cf0379d31d2e49 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 07:10:08 -0700 Subject: [PATCH 036/106] phase 2: compile and resolve bounded Wait sources Implement ASL section 2.4.2 and D44 with four typed sources, strict static/dynamic validation, target-only timestamp truncation and supplied-clock duration calculation. Reuse bounded component parsing with Choice. Independent review reports no findings. Core166 unit tests, 18 doctests and strict checks pass. Standard workspace gates pass with2671 tests/415 gated in the current worktree. B06 existing audit/deny failures remain open. Monotonic scheduling, generations, registry/compiler and complete M1 acceptance remain required; deferred Docker image changes are untouched. --- .../rustack-stepfunctions-core/src/choice.rs | 11 +- .../rustack-stepfunctions-core/src/error.rs | 6 + crates/rustack-stepfunctions-core/src/lib.rs | 2 + .../rustack-stepfunctions-core/src/payload.rs | 7 +- crates/rustack-stepfunctions-core/src/wait.rs | 161 +++++++++++++++++ .../src/wait/tests.rs | 164 ++++++++++++++++++ specs/index.md | 2 +- specs/rustack-workflows-30-asl-design.md | 8 + .../rustack-workflows-70-verification-plan.md | 4 + specs/rustack-workflows-90-impl-plan.md | 1 + specs/rustack-workflows-93-review.md | 6 + specs/rustack-workflows-99-decisions.md | 9 + 12 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 crates/rustack-stepfunctions-core/src/wait.rs create mode 100644 crates/rustack-stepfunctions-core/src/wait/tests.rs diff --git a/crates/rustack-stepfunctions-core/src/choice.rs b/crates/rustack-stepfunctions-core/src/choice.rs index dd5d3857..7065ca71 100644 --- a/crates/rustack-stepfunctions-core/src/choice.rs +++ b/crates/rustack-stepfunctions-core/src/choice.rs @@ -3,10 +3,12 @@ use std::{borrow::Cow, cmp::Ordering, fmt}; use chrono::{DateTime, Timelike}; -use rustack_core::json::{JsonLimits, from_slice}; use serde_json::{Number, Value}; -use crate::{AslError, EvaluationContext, Payload, ReferencePath, payload::validate_predicate}; +use crate::{ + AslError, EvaluationContext, Payload, ReferencePath, + payload::{parse_component, validate_component}, +}; mod glob; use glob::Glob; @@ -118,8 +120,7 @@ impl Predicate { /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) /// ``` pub fn parse(input: &[u8]) -> Result { - let limits = JsonLimits::new(1024 * 1024, 64, 4096, 262_144, 262_144)?; - let value: Value = from_slice(input, limits)?; + let value = parse_component(input)?; Self::compile(&value) } @@ -134,7 +135,7 @@ impl Predicate { /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) /// ``` pub fn compile(value: &Value) -> Result { - validate_predicate(value)?; + validate_component(value)?; Self::node(value, 0, &mut 1024) } diff --git a/crates/rustack-stepfunctions-core/src/error.rs b/crates/rustack-stepfunctions-core/src/error.rs index c1e353e6..7df690b4 100644 --- a/crates/rustack-stepfunctions-core/src/error.rs +++ b/crates/rustack-stepfunctions-core/src/error.rs @@ -9,6 +9,12 @@ use thiserror::Error; #[derive(Error)] #[non_exhaustive] pub enum AslError { + /// Unsupported or ambiguous static Wait source definition. + #[error("invalid ASL Wait source")] + InvalidWait, + /// A dynamic Wait value has an invalid type, date or supported range. + #[error("invalid ASL Wait value")] + WaitValue, /// Unsupported, ambiguous or invalid Choice predicate definition. #[error("invalid ASL Choice predicate")] InvalidPredicate, diff --git a/crates/rustack-stepfunctions-core/src/lib.rs b/crates/rustack-stepfunctions-core/src/lib.rs index 4730eb36..14c537e4 100644 --- a/crates/rustack-stepfunctions-core/src/lib.rs +++ b/crates/rustack-stepfunctions-core/src/lib.rs @@ -10,8 +10,10 @@ mod choice; mod error; mod path; mod payload; +mod wait; pub use choice::Predicate; pub use error::AslError; pub use path::{MatchKind, Path, PathMatches, ReferencePath, ResultPath}; pub use payload::{EvaluationContext, MAX_PAYLOAD_BYTES, Payload}; +pub use wait::WaitSource; diff --git a/crates/rustack-stepfunctions-core/src/payload.rs b/crates/rustack-stepfunctions-core/src/payload.rs index 6be2b103..04777fe7 100644 --- a/crates/rustack-stepfunctions-core/src/payload.rs +++ b/crates/rustack-stepfunctions-core/src/payload.rs @@ -217,7 +217,7 @@ fn validate_value(value: &Value, limits: TreeLimits) -> Result<(), AslError> { Ok(()) } -pub(crate) fn validate_predicate(value: &Value) -> Result<(), AslError> { +pub(crate) fn validate_component(value: &Value) -> Result<(), AslError> { validate_value( value, TreeLimits { @@ -303,3 +303,8 @@ impl io::Write for LimitedWriter { Ok(()) } } + +pub(crate) fn parse_component(input: &[u8]) -> Result { + let limits = JsonLimits::new(1024 * 1024, 64, 4096, 262_144, 262_144)?; + from_slice(input, limits).map_err(AslError::Json) +} diff --git a/crates/rustack-stepfunctions-core/src/wait.rs b/crates/rustack-stepfunctions-core/src/wait.rs new file mode 100644 index 00000000..cfe746ad --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/wait.rs @@ -0,0 +1,161 @@ +//! Pure Wait source resolution; monotonic timers remain execution-actor-owned. + +use std::{fmt, time::Duration}; + +use chrono::{DateTime, Timelike, Utc}; +use serde_json::Value; + +use crate::{ + AslError, EvaluationContext, Payload, ReferencePath, + payload::{parse_component, validate_component}, +}; + +const MAX_WAIT_SECONDS: u64 = 365 * 24 * 60 * 60; + +/// Compiled source for a single STANDARD Wait state, without transition fields. +#[derive(Clone)] +#[non_exhaustive] +pub struct WaitSource(Source); + +#[derive(Clone)] +enum Source { + Seconds(Duration), + SecondsPath(ReferencePath), + Timestamp(DateTime), + TimestampPath(ReferencePath), +} + +impl fmt::Debug for WaitSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("WaitSource()") + } +} + +impl WaitSource { + /// Preflight source JSON and compile exactly one Wait source member. + /// + /// # Errors + /// Rejects duplicates/unknown fields, invalid static values and source limits. + /// + /// ``` + /// let source = rustack_stepfunctions_core::WaitSource::parse(br#"{"Seconds":5}"#)?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn parse(input: &[u8]) -> Result { + let value = parse_component(input)?; + Self::compile(&value) + } + + /// Compile a borrowed tree before owning any path or timestamp. + /// + /// # Errors + /// Rejects ambiguous/missing sources and unsupported static seconds/dates. + /// + /// ``` + /// let source = rustack_stepfunctions_core::WaitSource::compile( + /// &serde_json::json!({"SecondsPath":"$.delay"}) + /// )?; + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn compile(value: &Value) -> Result { + validate_component(value)?; + let object = value + .as_object() + .filter(|v| v.len() == 1) + .ok_or(AslError::InvalidWait)?; + let (field, value) = object.iter().next().ok_or(AslError::InvalidWait)?; + let source = match field.as_str() { + "Seconds" => Source::Seconds(seconds(value).ok_or(AslError::InvalidWait)?), + "Timestamp" => Source::Timestamp( + value + .as_str() + .and_then(timestamp) + .ok_or(AslError::InvalidWait)?, + ), + "SecondsPath" => { + Source::SecondsPath(value.as_str().ok_or(AslError::InvalidWait)?.parse()?) + } + "TimestampPath" => { + Source::TimestampPath(value.as_str().ok_or(AslError::InvalidWait)?.parse()?) + } + _ => return Err(AslError::InvalidWait), + }; + Ok(Self(source)) + } + + /// Resolve once using the actor's supplied wall-time sample; never sleeps. + /// + /// The actor adds the returned relative duration to its paired monotonic + /// sample with checked arithmetic and retains independent execution expiry. + /// + /// # Errors + /// Missing dynamic paths, invalid dynamic values and waits over365 days fail. + /// + /// ``` + /// use rustack_stepfunctions_core::{WaitSource,Payload,EvaluationContext}; + /// let data=Payload::parse(b"{}")?; + /// let source=WaitSource::parse(br#"{"Seconds":5}"#)?; + /// assert_eq!(source.resolve(&data,&EvaluationContext::from(&data),chrono::Utc::now())?, + /// std::time::Duration::from_secs(5)); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn resolve( + &self, + data: &Payload, + context: &EvaluationContext, + now: DateTime, + ) -> Result { + let target = match &self.0 { + Source::Seconds(duration) => return Ok(*duration), + Source::SecondsPath(path) => { + return seconds(path.lookup(data, context).ok_or(AslError::Missing)?) + .ok_or(AslError::WaitValue); + } + Source::Timestamp(target) => *target, + Source::TimestampPath(path) => path + .lookup(data, context) + .ok_or(AslError::Missing)? + .as_str() + .and_then(timestamp) + .ok_or(AslError::WaitValue)?, + }; + if now.nanosecond() >= 1_000_000_000 { + return Err(AslError::WaitValue); + } + if target <= now { + return Ok(Duration::ZERO); + } + let duration = target + .signed_duration_since(now) + .to_std() + .map_err(|_| AslError::WaitValue)?; + if duration > Duration::from_secs(MAX_WAIT_SECONDS) { + Err(AslError::WaitValue) + } else { + Ok(duration) + } + } +} + +fn seconds(value: &Value) -> Option { + value + .as_u64() + .filter(|n| *n <= MAX_WAIT_SECONDS) + .map(Duration::from_secs) +} + +fn timestamp(value: &str) -> Option> { + if value.as_bytes().get(10) != Some(&b'T') || value.ends_with('z') { + return None; + } + let parsed = DateTime::parse_from_rfc3339(value).ok()?; + if parsed.nanosecond() >= 1_000_000_000 { + return None; + } + parsed + .with_nanosecond(0) + .map(|date| date.with_timezone(&Utc)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-core/src/wait/tests.rs b/crates/rustack-stepfunctions-core/src/wait/tests.rs new file mode 100644 index 00000000..2f83b262 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/wait/tests.rs @@ -0,0 +1,164 @@ +use std::{error::Error, time::Duration}; + +use chrono::{DateTime, Utc}; +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::json::JsonError; +use serde_json::{Value, json}; + +use crate::{AslError, EvaluationContext, Payload, WaitSource}; + +type TestResult = Result<(), Box>; + +fn now() -> Result, AslError> { + DateTime::from_timestamp(0, 0).ok_or(AslError::WaitValue) +} + +#[rstest] +#[case(json!({"Seconds":0}),json!({}),0)] +#[case(json!({"Seconds":5}),json!({}),5)] +#[case(json!({"Seconds":31_536_000}),json!({}),31_536_000)] +#[case(json!({"SecondsPath":"$.delay"}),json!({"delay":7}),7)] +#[case(json!({"Timestamp":"1970-01-01T00:00:05Z"}),json!({}),5)] +#[case(json!({"Timestamp":"1970-01-01T00:00:05.999999999999Z"}),json!({}),5)] +#[case(json!({"Timestamp":"1970-01-01T01:00:05+01:00"}),json!({}),5)] +#[case(json!({"Timestamp":"1970-01-01T00:00:00Z"}),json!({}),0)] +#[case(json!({"Timestamp":"1969-12-31T23:59:59Z"}),json!({}),0)] +#[case(json!({"Timestamp":"1971-01-01T00:00:00Z"}),json!({}),31_536_000)] +#[case(json!({"TimestampPath":"$.date"}),json!({"date":"1970-01-01T00:00:05Z"}),5)] +fn test_should_resolve_all_wait_source_forms( + #[case] source: Value, + #[case] input: Value, + #[case] seconds: u64, +) -> TestResult { + let source = WaitSource::compile(&source)?; + let input = Payload::from_value(&input)?; + assert_eq!( + source.resolve(&input, &EvaluationContext::from(&input), now()?)?, + Duration::from_secs(seconds) + ); + assert_eq!(format!("{source:?}"), "WaitSource()"); + Ok(()) +} + +#[rstest] +#[case(json!({}))] +#[case(json!(null))] +#[case(json!({"Unknown":1}))] +#[case(json!({"Seconds":1,"Timestamp":"1970-01-01T00:00:00Z"}))] +#[case(json!({"Seconds":1,"Comment":"unknown component field"}))] +#[case(json!({"Seconds":1.0}))] +#[case(json!({"Seconds":-1}))] +#[case(json!({"Seconds":true}))] +#[case(json!({"Seconds":"1"}))] +#[case(json!({"Seconds":31_536_001}))] +#[case(json!({"Seconds":99_999_999}))] +#[case(json!({"Timestamp":"1970-01-01"}))] +#[case(json!({"Timestamp":"1970-01-01t00:00:00Z"}))] +#[case(json!({"Timestamp":"1970-01-01T00:00:00z"}))] +#[case(json!({"Timestamp":"2016-12-31T23:59:60Z"}))] +#[case(json!({"Timestamp":"2020-02-30T00:00:00Z"}))] +#[case(json!({"Timestamp":0}))] +#[case(json!({"SecondsPath":0}))] +fn test_should_reject_ambiguous_or_invalid_static_wait_sources(#[case] value: Value) { + assert!(matches!( + WaitSource::compile(&value), + Err(AslError::InvalidWait) + )); +} + +#[rstest] +#[case("SecondsPath",json!({"value":null}))] +#[case("SecondsPath",json!({"value":"5"}))] +#[case("SecondsPath",json!({"value":1.0}))] +#[case("SecondsPath",json!({"value":-1}))] +#[case("SecondsPath",json!({"value":31_536_001}))] +#[case("TimestampPath",json!({"value":5}))] +#[case("TimestampPath",json!({"value":"invalid"}))] +#[case("TimestampPath",json!({"value":"1970-01-01t00:00:00Z"}))] +#[case("TimestampPath",json!({"value":"1971-01-01T00:00:01Z"}))] +fn test_should_fail_invalid_dynamic_values_without_defaulting( + #[case] field: &str, + #[case] input: Value, +) -> TestResult { + let source = WaitSource::compile(&json!({field:"$.value"}))?; + let input = Payload::from_value(&input)?; + assert!(matches!( + source.resolve(&input, &EvaluationContext::from(&input), now()?), + Err(AslError::WaitValue) + )); + Ok(()) +} + +#[test] +fn test_should_preserve_missing_reference_class_and_validate_paths() -> TestResult { + let input = Payload::parse(b"{}")?; + for field in ["SecondsPath", "TimestampPath"] { + let source = WaitSource::compile(&json!({field:"$.missing"}))?; + assert!(matches!( + source.resolve(&input, &EvaluationContext::from(&input), now()?), + Err(AslError::Missing) + )); + assert!(matches!( + WaitSource::compile(&json!({field:"$[*]"})), + Err(AslError::InvalidReference) + )); + } + let context = EvaluationContext::from_value(&json!({"State":{"delay":9}}))?; + let source = WaitSource::compile(&json!({"SecondsPath":"$$.State.delay"}))?; + assert_eq!( + source.resolve(&input, &context, now()?)?, + Duration::from_secs(9) + ); + assert!(matches!( + WaitSource::parse(br#"{"Seconds":1,"Seconds":2}"#), + Err(AslError::Json(JsonError::Duplicate)) + )); + assert!(matches!( + WaitSource::parse(br#"{"Seconds":1e0}"#), + Err(AslError::InvalidWait) + )); + Ok(()) +} + +#[test] +fn test_should_truncate_only_the_target_preserving_current_fraction_and_supplied_clock() +-> TestResult { + let source = WaitSource::parse(br#"{"Timestamp":"1970-01-01T00:00:10.999999999Z"}"#)?; + let input = Payload::parse(b"{}")?; + let context = EvaluationContext::from(&input); + let sample = DateTime::from_timestamp(9, 250_000_000).ok_or("invalid test clock")?; + assert_eq!( + source.resolve(&input, &context, sample)?, + Duration::from_millis(750) + ); + let target_past = DateTime::from_timestamp(10, 1).ok_or("invalid test clock")?; + assert_eq!( + source.resolve(&input, &context, target_past)?, + Duration::ZERO + ); + let overlong = WaitSource::parse(br#"{"Timestamp":"1971-01-01T00:00:01Z"}"#)?; + assert!(matches!( + overlong.resolve(&input, &context, now()?), + Err(AslError::WaitValue) + )); + // A new entry uses its new sample, without the component reading a clock. + assert_eq!( + source.resolve(&input, &context, now()?)?, + Duration::from_secs(10) + ); + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + #[test] + fn test_should_resolve_static_and_dynamic_seconds_equivalently(seconds in 0_u64..=31_536_000) { + let input=Payload::from_value(&json!({"seconds":seconds})).map_err(|e| TestCaseError::fail(e.to_string()))?; + let context=EvaluationContext::from(&input); + let clock=now().map_err(|e| TestCaseError::fail(e.to_string()))?; + let static_source=WaitSource::compile(&json!({"Seconds":seconds})).map_err(|e| TestCaseError::fail(e.to_string()))?; + let dynamic_source=WaitSource::compile(&json!({"SecondsPath":"$.seconds"})).map_err(|e| TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(static_source.resolve(&input,&context,clock).map_err(|e| TestCaseError::fail(e.to_string()))?,dynamic_source.resolve(&input,&context,clock).map_err(|e| TestCaseError::fail(e.to_string()))?); + } +} diff --git a/specs/index.md b/specs/index.md index 1483a06e..4a833661 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Bounded compiled paths/payloads and Choice predicates are implemented and independently reviewed, including context headroom, exact timestamp fractions and conservative numeric comparisons. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Bounded compiled paths/payloads, Choice predicates and pure Wait source resolution are implemented and independently reviewed, including context headroom, exact timestamp fractions and conservative numeric comparisons. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-30-asl-design.md b/specs/rustack-workflows-30-asl-design.md index 61fd6409..784d5cc8 100644 --- a/specs/rustack-workflows-30-asl-design.md +++ b/specs/rustack-workflows-30-asl-design.md @@ -109,6 +109,14 @@ StringMatches compiles a bounded glob: at most4,096 UTF-8 pattern bytes and1,024 Each evaluation has one shared10,000-unit budget for predicate/glob steps and1MiB aggregate compared string/timestamp bytes. Debit before work; exhaustion is a typed WorkLimit failure, never false. This prevents compound comparisons and glob backtracking from monopolizing the execution actor. These are local safety limits, not an AWS parity claim. Keep compiled predicates and diagnostics redacted. Tests cover the supported literal/Path operator matrix, logical order, missing/null/wrong type, timestamp offsets, binary64 boundaries, Unicode glob escapes/work exhaustion and borrowed extreme-depth rejection. Do not claim complete Choice state/compiler/engine acceptance from this evaluator alone. Primary operation names/short-circuit examples: [AWS Choice JSONPath rules](https://docs.aws.amazon.com/step-functions/latest/dg/state-choice.html#choice-state-rules-jsonpath). +#### 2.4.2 Executable Wait source contract + +Compile WaitSource from an object containing exactly one of Seconds, SecondsPath, Timestamp or TimestampPath; the containing machine compiler owns Type, transforms and transitions. Reuse bounded source preflight/borrowed-tree validation before compilation. Static and dynamic seconds must be nonnegative JSON integers; reject floating-point spellings (including1.0), strings and booleans. The general ASL integer field ceiling is99,999,999; this STANDARD-only profile additionally caps a single resolved wait to31,536,000 seconds (365 days). Reject an overlong static Seconds at compilation and an overlong dynamic/datetime wait at entry. Other overall execution/operator deadlines can end the execution sooner. + +Reference sources compile to singular ReferencePath values. Missing lookup raises Missing; present invalid type/range/date raises a distinct WaitValue failure and never silently defaults. Absolute dates use RFC3339 with uppercase T and uppercase Z when no numeric offset is present; permit explicit numeric offsets, reject leap seconds, lowercase separators, date-only text and invalid calendar values. Validate dates once for static timestamps and on entry for path timestamps. Truncate the target to its whole second before subtraction, retaining the current wall clock's fractional precision. Past/equal targets resolve to zero; future targets resolve to a checked std::time::Duration within the STANDARD cap. + +Resolution is pure and receives the caller's sampled UTC wall time; it performs no IO, sleep or clock reads. The execution actor must sample its wall/monotonic pair once, add the resolved relative duration to that monotonic instant with checked arithmetic and register a generation-tagged timer bounded by the independent execution deadline. Do not recompute an absolute wait from wall time while the same state entry is suspended, or respond to subsequent NTP jumps. A new state entry evaluates its source afresh. WaitSource owns no task or timer handle and does not qualify scheduler/cancellation acceptance by itself. Tests cover all four source forms, exact caps, invalid/double fields, duplicates, path missing/null/wrong type, UTC offsets, fractional target truncation, fractional current time and past targets. Primary contract: [AWS Wait state](https://docs.aws.amazon.com/step-functions/latest/dg/state-wait.html). + ## 3. Execution ownership and scheduling One ExecutionActor owns execution data, all branch frames, attempt generations, retry counters, timers and history. Branches are frames, not an unbounded tree of Tokio tasks. A ready queue drives pure transitions, yielding after at most 64 transitions per tick. Bounded supervised JoinSet futures perform external calls and return typed results; pure evaluation never performs network IO. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 462d90c2..7ce0163c 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -186,3 +186,7 @@ Reject a borrowed50,000-depth Value without owning its recursive teardown; the t The existing `make test-stepfunctions-core` target includes the compiled predicate evaluator: all90 ordered family/operator/literal-Path combinations, BooleanEquals variants, six type tests, logical short-circuit/error order and four independent Choice catalog cases. Two additional512-case properties compare the Unicode-glob implementation with an independent dynamic-programming oracle and preserve integer order. Validate source duplicates/unknown operators,1,024-node/32-level/128-child and pattern/work/string-byte limits, borrowed50,000-depth rejection, redaction, context references and wrong-type behavior. Raw JSON regressions must include decimal/exponent forms at and beyond the conservative float endpoints, their negative counterparts and correctly rounded interior equivalents; constructing json! from already-rounded Rust f64 values is insufficient. Keep integer endpoints valid and IsNumeric a type test. Timestamp cases compare offsets, fractions beyond nine digits and trailing-zero equivalence; valid large literal and Path predicates must exhaust the shared byte budget explicitly. Workspace serde_json float_roundtrip is enabled, so rerun full standard workspace gates and the generated-model/HTTP signed SDK targets for feature-unification regressions. The excluded generator retains its own graph. This qualifies predicates only; state target selection and the complete compiler/execution actor remain required. + +## Step Functions Wait source qualification + +The core target includes all four Wait source forms and a512-case static/dynamic seconds equivalence property. Test exact365-day STANDARD caps, zero/past waits, uppercase timestamp separators, numeric offsets, target-only truncation, long target fractions, fractional current samples, duplicate/unknown fields and dynamic missing/null/wrong-type/range errors. Confirm the component returns only a relative Duration and never reads a clock or starts a timer. Core166 unit cases/18 doctests and strict checks pass. These tests do not qualify generation-tagged timer ownership, monotonic scheduling, execution deadline races or cancellation; the execution actor remains responsible for those gates. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index f005ca98..a881c735 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -161,6 +161,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 2.1 typed HTTP transport | Implemented/reviewed; real core/registration pending | API§1.2/D41 implements18 typed ports, bounded envelopes/body/response, strict SigV4 service/region/target binding, exact DTO dates and accepted-command waiter isolation.56 unit tests, two nongated network tests, eight doctests, strict HTTP Clippy and four real signed boto3 calls pass. W101–W104 are fixed and independently rechecked. Current-worktree workspace2,505/415 gated, standard build/test/fmt/Clippy/rustdoc, audit and deny pass. No production fallback handler, main registration or engine acceptance is claimed. | | 2.2 compiled paths/payloads | Implemented; complete compiler remains | ASL§2.1.1/D42 supplies bounded immutable payloads/context, compiled paths, borrowed missing/single/selection and transactional result merge.75 unit tests (three512-case properties),12 doctests and strict production checks pass; W105–W107 corrections cover borrowed deep-tree rejection and numeric/depth/member context headroom. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. New existing-dependency advisory findings are tracked as B06; audit/deny currently fail that baseline, with no ignores added. | | 2.2 compiled Choice predicates | Implemented/reviewed; state/compiler integration remains | ASL§2.4.1/D43 implements typed literal/Path comparisons, logical short-circuit, exact timestamp fractions, conservative numeric interoperability and bounded glob/shared budgets. Core125 unit cases/15 doctests and strict checks pass, including90 operator combinations and four independent Choice catalog cases. W108–W109 are fixed/rechecked. Shared float_roundtrip adds no external package/version. Final current-worktree standard workspace gates pass with2,630/415 gated; regenerated model/strict HTTP and signed SDK regressions pass. Complete compiler, transition selection and engine remain required; B06 stays open. | +| 2.2/2.3 compiled Wait source | Implemented/reviewed; actor scheduling remains | ASL§2.4.2/D44 supplies four typed sources and pure supplied-clock duration resolution; no sleeping task or timer is created. Independent review reports no findings. Core166 unit cases/18 doctests and strict checks pass; standard current-worktree workspace gates pass with2,671/415 gated, including deferred Docker changes. Monotonic timer/generation/deadline/cancellation acceptance remains required; B06 stays open. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index bad3c072..bdf08d15 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -513,6 +513,12 @@ ASL§2.4.1/D43 implements compiled predicate semantics, shared evaluation budget Focused evidence:125 total core unit cases and15 doctests pass, including the90-combination matrix, four independent Choice catalog cases and two new512-case properties. Ordinary all-targets and strict production core Clippy pass. This is predicate evidence, not a complete Choice state or compiler/engine acceptance. Final standard workspace build/test/fmt/Clippy/rustdoc pass with2,630 tests/415 gated, including retained deferred Docker changes. Model regeneration and strict model/HTTP targets plus the real signed SDK fixture pass after shared feature unification. B06 audit/deny failures remain explicitly open. +## P2 Wait source independent review + +ASL§2.4.2/D44 implements the four typed Wait sources, source/dynamic validation, supplied-clock relative-duration resolution, target-only timestamp truncation and a365-day STANDARD cap. Shared source preflight/borrow validation is reused by Choice and Wait. The independent read-only reviewer reports no findings; scheduling/timer ownership is explicitly outside this pure component. + +Core166 unit cases/18 doctests and strict production/all-targets Clippy pass, including41 Wait additions and one512-case property. Final current-worktree workspace build/test/fmt/Clippy/rustdoc pass with2,671 tests/415 gated, including the deferred Docker checkpoint. B06 audit/deny failures remain open. This does not close the compiler, registry, execution actor or M1 acceptance. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 287d454d..73652cd4 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -384,3 +384,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: a pure reusable evaluator pins semantics and makes local limits explicit before machine compilation and scheduling. It remains separate from transition-target ownership. - **Pinned by**: [ASL§2.4.1](./rustack-workflows-30-asl-design.md#241-executable-choice-predicate-contract), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. Reuse existing chrono, JSON and reference paths. Numeric integers use the closed±2^53 range; floats exclude its endpoints to avoid source-rounding aliases. Enable shared serde_json float_roundtrip for correctly rounded interior numeric lexemes; no arbitrary_precision or new external version is added. Timestamp fractions retain precision beyond nanoseconds. No complete Choice state/engine acceptance is claimed. + +## D44 — Resolve Wait sources once before monotonic scheduling + +- **Context**: absolute Wait timestamps truncate target fractions, while execution deadlines and timer ownership remain monotonic. +- **Alternatives considered**: sleeping inside a pure evaluator obscures ownership; repeatedly subtracting wall clocks follows NTP jumps; defaulting invalid dynamic values hides workflow errors. +- **Decision**: compile exactly one typed source and resolve it against validated data/context plus one supplied wall-time sample, returning a bounded relative duration. The future actor owns checked monotonic scheduling and generations. +- **Why**: static validation, dynamic failures and AWS whole-second target behavior can be pinned without a sleeping task or dependency on the HTTP caller lifetime. +- **Pinned by**: [ASL§2.4.2](./rustack-workflows-30-asl-design.md#242-executable-wait-source-contract), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Use existing chrono/std/reference paths. STANDARD local wait cap is365 days; overall execution deadlines are independent. This is source evaluation, not scheduler acceptance. From b6e5b515ac9e1df2a70a37ff0ad24bec10089b99 Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 07:32:17 -0700 Subject: [PATCH 037/106] phase 2: implement bounded ASL intrinsic expressions Implement ASL section2.3.1/D45: complete12-call allowlist, winnow AST, format escape provenance, shared work/text/materialization budgets and fallible UUID sourcing. Record historical AWS rounding/split evidence. Independent review closes W110-W111 with repeated-delimiter and late-object-mismatch regressions. Core242 unit tests/21 doctests and strict checks pass; standard workspace gates pass with2747 tests/415 gated in the current worktree. Existing getrandom/uuid dependency edges add no external package/version. B06 zstd/Squib audit/deny failures remain open without ignores. Template/pipeline/compiler and full phase acceptance remain required; deferred Docker changes remain outside this commit. --- Cargo.lock | 2 + Cargo.toml | 1 + crates/rustack-stepfunctions-core/Cargo.toml | 2 + .../rustack-stepfunctions-core/src/choice.rs | 2 +- .../rustack-stepfunctions-core/src/error.rs | 17 +- .../src/expression.rs | 203 ++++++++ .../src/expression/eval.rs | 249 ++++++++++ .../src/expression/format.rs | 103 ++++ .../src/expression/parser.rs | 191 ++++++++ .../src/expression/tests.rs | 441 ++++++++++++++++++ crates/rustack-stepfunctions-core/src/lib.rs | 2 + crates/rustack-stepfunctions-core/src/path.rs | 15 +- docs/index.md | 2 +- .../spike-stepfunctions-semantic-oracle.md | 8 + specs/index.md | 2 +- specs/rustack-workflows-30-asl-design.md | 41 ++ .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 1 + specs/rustack-workflows-93-review.md | 11 + specs/rustack-workflows-99-decisions.md | 9 + 20 files changed, 1300 insertions(+), 8 deletions(-) create mode 100644 crates/rustack-stepfunctions-core/src/expression.rs create mode 100644 crates/rustack-stepfunctions-core/src/expression/eval.rs create mode 100644 crates/rustack-stepfunctions-core/src/expression/format.rs create mode 100644 crates/rustack-stepfunctions-core/src/expression/parser.rs create mode 100644 crates/rustack-stepfunctions-core/src/expression/tests.rs diff --git a/Cargo.lock b/Cargo.lock index b533c6dd..6ea103b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4196,12 +4196,14 @@ version = "0.9.0" dependencies = [ "bytes", "chrono", + "getrandom 0.4.3", "proptest", "rstest", "rustack-core", "serde", "serde_json", "thiserror", + "uuid", "winnow", ] diff --git a/Cargo.toml b/Cargo.toml index 91cb5e27..5f7003f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ url = "2.5.8" bollard-stubs = { version = "=1.53.1-rc.29.3.1", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["float_roundtrip"] } +getrandom = { version = "0.4.3", features = ["std"] } ciborium = "0.2" quick-xml = "0.41" zstd = { version = "0.13.3", default-features = false } diff --git a/crates/rustack-stepfunctions-core/Cargo.toml b/crates/rustack-stepfunctions-core/Cargo.toml index 75adfc15..b633b0b5 100644 --- a/crates/rustack-stepfunctions-core/Cargo.toml +++ b/crates/rustack-stepfunctions-core/Cargo.toml @@ -14,10 +14,12 @@ workspace = true rustack-core = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +getrandom = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } winnow = { workspace = true } +uuid = { workspace = true } [dev-dependencies] rstest = { workspace = true } diff --git a/crates/rustack-stepfunctions-core/src/choice.rs b/crates/rustack-stepfunctions-core/src/choice.rs index 7065ca71..78a04c14 100644 --- a/crates/rustack-stepfunctions-core/src/choice.rs +++ b/crates/rustack-stepfunctions-core/src/choice.rs @@ -351,7 +351,7 @@ fn resolve<'a, T>( } } -fn number(value: &Value) -> Result, AslError> { +pub(crate) fn number(value: &Value) -> Result, AslError> { const LIMIT: f64 = 9_007_199_254_740_992.0; if let Some(n) = value.as_u64() { if n > 9_007_199_254_740_992 { diff --git a/crates/rustack-stepfunctions-core/src/error.rs b/crates/rustack-stepfunctions-core/src/error.rs index 7df690b4..3861856e 100644 --- a/crates/rustack-stepfunctions-core/src/error.rs +++ b/crates/rustack-stepfunctions-core/src/error.rs @@ -1,7 +1,8 @@ //! Redacted semantic failures with typed source ownership. -use std::fmt; +use std::{collections::TryReserveError, fmt}; +use getrandom::Error as EntropyError; use rustack_core::json::JsonError; use thiserror::Error; @@ -9,6 +10,18 @@ use thiserror::Error; #[derive(Error)] #[non_exhaustive] pub enum AslError { + /// Unsupported or invalid intrinsic syntax/arity. + #[error("invalid ASL expression")] + InvalidExpression, + /// An intrinsic encountered an invalid value, type or format. + #[error("invalid ASL intrinsic value")] + IntrinsicValue, + /// The operating system could not supply secure UUID entropy. + #[error("ASL UUID entropy unavailable")] + Entropy(#[source] EntropyError), + /// Bounded storage could not be reserved. + #[error("ASL allocation failed")] + Allocation(#[source] TryReserveError), /// Unsupported or ambiguous static Wait source definition. #[error("invalid ASL Wait source")] InvalidWait, @@ -37,7 +50,7 @@ pub enum AslError { #[error("ASL result path cannot be applied")] Merge, /// Expansion exceeds the allowed bounded evaluation work. - #[error("ASL path evaluation work limit exceeded")] + #[error("ASL evaluation work limit exceeded")] WorkLimit, /// Materialized data exceeds the payload byte allowance. #[error("ASL payload byte limit exceeded")] diff --git a/crates/rustack-stepfunctions-core/src/expression.rs b/crates/rustack-stepfunctions-core/src/expression.rs new file mode 100644 index 00000000..1d1bca73 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/expression.rs @@ -0,0 +1,203 @@ +//! Compiled expressions with explicit UUID sourcing and shared evaluation budgets. + +use std::{fmt, str::FromStr}; + +use getrandom::fill; +use uuid::{Builder, Uuid}; + +use crate::{AslError, EvaluationContext, Path, Payload}; + +mod eval; +mod format; +mod parser; + +use format::Format; + +/// Typed UUID source for deterministic/failure tests and scheduler-safe injection. +pub trait UuidSource { + /// Produce one securely sourced version4 UUID in production. + /// + /// # Errors + /// Returns entropy/admission failure; never silently substitutes a fixed ID. + /// + /// ``` + /// use rustack_stepfunctions_core::{SystemUuidSource, UuidSource}; + /// let id=SystemUuidSource::default().next_uuid()?; + /// assert_eq!(id.get_version_num(),4); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + fn next_uuid(&mut self) -> Result; +} + +/// Fallible system entropy adapter; callers own any required blocking isolation. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct SystemUuidSource; + +impl UuidSource for SystemUuidSource { + fn next_uuid(&mut self) -> Result { + let mut random = [0_u8; 16]; + fill(&mut random).map_err(AslError::Entropy)?; + Ok(Builder::from_random_bytes(random).into_uuid()) + } +} + +/// Immutable compiled path or allowlisted intrinsic, with redacted diagnostics. +#[derive(Clone)] +#[non_exhaustive] +pub struct Expression(Node); + +#[derive(Clone)] +enum Node { + Path(Path), + Literal(Payload), + Call { + function: Function, + args: Vec, + format: Option, + }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Function { + Format, + StringToJson, + JsonToString, + Array, + ArrayLength, + ArrayGetItem, + ArrayContains, + ArrayPartition, + StringSplit, + MathAdd, + Uuid, + JsonMerge, +} + +impl Function { + fn parse(name: &str) -> Result { + match name { + "States.Format" => Ok(Self::Format), + "States.StringToJson" => Ok(Self::StringToJson), + "States.JsonToString" => Ok(Self::JsonToString), + "States.Array" => Ok(Self::Array), + "States.ArrayLength" => Ok(Self::ArrayLength), + "States.ArrayGetItem" => Ok(Self::ArrayGetItem), + "States.ArrayContains" => Ok(Self::ArrayContains), + "States.ArrayPartition" => Ok(Self::ArrayPartition), + "States.StringSplit" => Ok(Self::StringSplit), + "States.MathAdd" => Ok(Self::MathAdd), + "States.UUID" => Ok(Self::Uuid), + "States.JsonMerge" => Ok(Self::JsonMerge), + _ => Err(AslError::InvalidExpression), + } + } + + fn arity(self, count: usize) -> bool { + match self { + Self::Format => (1..=128).contains(&count), + Self::Array => count <= 128, + Self::StringToJson | Self::JsonToString | Self::ArrayLength => count == 1, + Self::ArrayGetItem + | Self::ArrayContains + | Self::ArrayPartition + | Self::StringSplit + | Self::MathAdd => count == 2, + Self::Uuid => count == 0, + Self::JsonMerge => count == 3, + } + } +} + +impl fmt::Debug for Expression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Expression()") + } +} + +impl FromStr for Expression { + type Err = AslError; + fn from_str(text: &str) -> Result { + parser::compile(text).map(Self) + } +} + +impl Expression { + /// Evaluate with fallible OS entropy if a UUID call is reached. + /// + /// The OS may wait for entropy during early boot. Async execution owners use + /// `evaluate_with_source` with pre-acquired IDs or a bounded blocking worker. + /// + /// # Errors + /// Returns missing/type/format, resource-limit or entropy failure. + /// + /// ``` + /// use rustack_stepfunctions_core::{Expression,Payload,EvaluationContext}; + /// let data=Payload::parse(br#"{"n":2}"#)?; + /// let expression:Expression="States.MathAdd($.n, 3)".parse()?; + /// assert_eq!(expression.evaluate(&data,&EvaluationContext::from(&data))?.value(),&serde_json::json!(5)); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn evaluate( + &self, + data: &Payload, + context: &EvaluationContext, + ) -> Result { + self.evaluate_with_source(data, context, &mut SystemUuidSource) + } + + /// Evaluate using an explicit UUID port under one shared set of budgets. + /// + /// # Errors + /// Returns evaluation failures and rejects injected non-version4 UUIDs. + /// + /// ``` + /// use rustack_stepfunctions_core::{Expression,Payload,EvaluationContext,SystemUuidSource}; + /// let data=Payload::parse(b"null")?; + /// let expression:Expression="States.Array()".parse()?; + /// let result=expression.evaluate_with_source(&data,&EvaluationContext::from(&data),&mut SystemUuidSource::default())?; + /// assert_eq!(result.value(),&serde_json::json!([])); + /// # Ok::<(), rustack_stepfunctions_core::AslError>(()) + /// ``` + pub fn evaluate_with_source( + &self, + data: &Payload, + context: &EvaluationContext, + source: &mut S, + ) -> Result { + eval::node(&self.0, data, context, source, &mut Budget::new()) + } +} + +struct Budget { + units: usize, + text: usize, + materialized: usize, +} +impl Budget { + fn new() -> Self { + Self { + units: 10_000, + text: 4 * 1024 * 1024, + materialized: 4 * 1024 * 1024, + } + } + fn work(&mut self, count: usize) -> Result<(), AslError> { + self.units = self.units.checked_sub(count).ok_or(AslError::WorkLimit)?; + Ok(()) + } + fn text(&mut self, count: usize) -> Result<(), AslError> { + self.text = self.text.checked_sub(count).ok_or(AslError::WorkLimit)?; + Ok(()) + } + fn result(&mut self, result: &Payload) -> Result<(), AslError> { + self.materialized = self + .materialized + .checked_sub(result.original_bytes().len()) + .ok_or(AslError::WorkLimit)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustack-stepfunctions-core/src/expression/eval.rs b/crates/rustack-stepfunctions-core/src/expression/eval.rs new file mode 100644 index 00000000..39dcb968 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/expression/eval.rs @@ -0,0 +1,249 @@ +//! Bounded intrinsic evaluation with borrowed collection output views. + +use std::{ + cmp::Ordering, + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, +}; + +use serde::Serialize; +use serde_json::Value; +use uuid::{Variant, Version}; + +use super::{Budget, Function, Node, UuidSource, format::Format}; +use crate::{AslError, EvaluationContext, Payload, choice::number, payload::encode}; + +pub(super) fn node( + expression: &Node, + data: &Payload, + context: &EvaluationContext, + source: &mut S, + budget: &mut Budget, +) -> Result { + budget.work(1)?; + let result = match expression { + Node::Literal(value) => value.clone(), + Node::Path(path) => path + .select_with_work(data, context, &mut budget.units)? + .to_payload()?, + Node::Call { + function, + args, + format, + } => { + let args = args + .iter() + .map(|arg| node(arg, data, context, source, budget)) + .collect::, _>>()?; + call(*function, &args, format.as_ref(), source, budget)? + } + }; + budget.result(&result)?; + Ok(result) +} + +fn arg(args: &[Payload], index: usize) -> Result<&Value, AslError> { + args.get(index) + .map(Payload::value) + .ok_or(AslError::IntrinsicValue) +} + +fn output(value: &T) -> Result { + Payload::from_bytes(encode(value)?) +} + +fn call( + function: Function, + args: &[Payload], + format: Option<&Format>, + source: &mut S, + budget: &mut Budget, +) -> Result { + match function { + Function::Format => { + let (first, rest) = args.split_first().ok_or(AslError::IntrinsicValue)?; + if let Some(format) = format { + format.render(rest, budget) + } else { + let text = first.value().as_str().ok_or(AslError::IntrinsicValue)?; + budget.text(text.len())?; + Format::compile(text, rest.len()) + .map_err(|_| AslError::IntrinsicValue)? + .render(rest, budget) + } + } + Function::StringToJson => { + let text = arg(args, 0)?.as_str().ok_or(AslError::IntrinsicValue)?; + budget.text(text.len())?; + Payload::parse(text.as_bytes()) + } + Function::JsonToString => { + let encoded = encode(arg(args, 0)?)?; + budget.text(encoded.len())?; + let text = String::from_utf8(encoded.to_vec()).map_err(|_| AslError::IntrinsicValue)?; + Payload::from_value(&Value::String(text)) + } + Function::Array => output(&args.iter().map(Payload::value).collect::>()), + Function::ArrayLength => output( + &arg(args, 0)? + .as_array() + .ok_or(AslError::IntrinsicValue)? + .len(), + ), + Function::ArrayGetItem => { + let values = arg(args, 0)?.as_array().ok_or(AslError::IntrinsicValue)?; + let index = arg(args, 1)? + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or(AslError::IntrinsicValue)?; + output(values.get(index).ok_or(AslError::IntrinsicValue)?) + } + Function::ArrayContains => { + let values = arg(args, 0)?.as_array().ok_or(AslError::IntrinsicValue)?; + let needle = arg(args, 1)?; + let mut found = false; + for value in values { + if equal(value, needle, budget)? { + found = true; + break; + } + } + output(&found) + } + Function::ArrayPartition => { + let values = arg(args, 0)?.as_array().ok_or(AslError::IntrinsicValue)?; + let size = usize::try_from(rounded(arg(args, 1)?)?) + .ok() + .and_then(NonZeroUsize::new) + .ok_or(AslError::IntrinsicValue)?; + budget.work(values.len())?; + output(&values.chunks(size.get()).collect::>()) + } + Function::StringSplit => split(args, budget), + Function::MathAdd => { + let left = rounded(arg(args, 0)?)?; + let right = rounded(arg(args, 1)?)?; + output(&left.checked_add(right).ok_or(AslError::IntrinsicValue)?) + } + Function::Uuid => { + let id = source.next_uuid()?; + if id.get_version() != Some(Version::Random) || id.get_variant() != Variant::RFC4122 { + return Err(AslError::IntrinsicValue); + } + output(&id.to_string()) + } + Function::JsonMerge => { + let left = arg(args, 0)?.as_object().ok_or(AslError::IntrinsicValue)?; + let right = arg(args, 1)?.as_object().ok_or(AslError::IntrinsicValue)?; + if arg(args, 2)? != &Value::Bool(false) { + return Err(AslError::IntrinsicValue); + } + budget.work(left.len().saturating_add(right.len()))?; + let mut merged = BTreeMap::new(); + for (key, value) in left.iter().chain(right) { + budget.text(key.len())?; + merged.insert(key, value); + } + output(&merged) + } + } +} + +fn split(args: &[Payload], budget: &mut Budget) -> Result { + let text = arg(args, 0)?.as_str().ok_or(AslError::IntrinsicValue)?; + let separators = arg(args, 1)?.as_str().ok_or(AslError::IntrinsicValue)?; + budget.text(text.len().saturating_add(separators.len()))?; + let mut delimiters = BTreeSet::new(); + for ch in separators.chars() { + budget.work(1)?; + delimiters.insert(ch); + if delimiters.len() > 1024 { + return Err(AslError::WorkLimit); + } + } + let mut parts = Vec::new(); + for part in text + .split(|ch| delimiters.contains(&ch)) + .filter(|s| !s.is_empty()) + { + budget.work(1)?; + parts.push(part); + } + output(&parts) +} + +fn rounded(value: &Value) -> Result { + let value = value + .as_f64() + .filter(|n| n.is_finite()) + .ok_or(AslError::IntrinsicValue)?; + let floor = value.floor(); + let rounded = if value - floor >= 0.5 { + floor + 1.0 + } else { + floor + }; + if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) { + return Err(AslError::IntrinsicValue); + } + rounded + .to_string() + .parse() + .map_err(|_| AslError::IntrinsicValue) +} + +fn equal<'a>(left: &'a Value, right: &'a Value, budget: &mut Budget) -> Result { + let mut stack = vec![(left, right)]; + while let Some((left, right)) = stack.pop() { + budget.work(1)?; + match (left, right) { + (Value::Null, Value::Null) => {} + (Value::Bool(a), Value::Bool(b)) if a == b => {} + (Value::String(a), Value::String(b)) => { + budget.text(a.len().saturating_add(b.len()))?; + if a != b { + return Ok(false); + } + } + (Value::Number(a), Value::Number(b)) => { + if a.is_f64() || b.is_f64() { + if !number(left)? + .zip(number(right)?) + .is_some_and(|(a, b)| a.partial_cmp(&b).is_some_and(Ordering::is_eq)) + { + return Ok(false); + } + } else if a != b { + return Ok(false); + } + } + (Value::Array(a), Value::Array(b)) => { + if a.len() != b.len() { + return Ok(false); + } + if stack.len().saturating_add(a.len()) > budget.units { + return Err(AslError::WorkLimit); + } + stack.extend(a.iter().zip(b)); + } + (Value::Object(a), Value::Object(b)) => { + if a.len() != b.len() { + return Ok(false); + } + for (key, value) in a { + budget.work(1)?; + budget.text(key.len())?; + let Some(other) = b.get(key) else { + return Ok(false); + }; + if stack.len() >= budget.units { + return Err(AslError::WorkLimit); + } + stack.push((value, other)); + } + } + _ => return Ok(false), + } + } + Ok(true) +} diff --git a/crates/rustack-stepfunctions-core/src/expression/format.rs b/crates/rustack-stepfunctions-core/src/expression/format.rs new file mode 100644 index 00000000..cdc43f65 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/expression/format.rs @@ -0,0 +1,103 @@ +//! Escape-aware scalar formatting, independent of host formatting expressions. + +use std::{mem::take, str::from_utf8}; + +use serde_json::Value; + +use super::Budget; +use crate::{AslError, MAX_PAYLOAD_BYTES, Payload, payload::encode}; + +#[derive(Clone)] +pub(super) struct Format(Vec); +#[derive(Clone)] +enum Part { + Text(String), + Slot, +} + +impl Format { + pub(super) fn compile(text: &str, slots: usize) -> Result { + let mut parts = Vec::new(); + let mut literal = String::new(); + let mut chars = text.chars(); + let mut count = 0_usize; + while let Some(ch) = chars.next() { + match ch { + '\\' => match chars.next() { + Some(ch @ ('\'' | '{' | '}' | '\\')) => literal.push(ch), + _ => return Err(AslError::InvalidExpression), + }, + '{' => { + if chars.next() != Some('}') { + return Err(AslError::InvalidExpression); + } + count = count.checked_add(1).ok_or(AslError::InvalidExpression)?; + if count > slots { + return Err(AslError::InvalidExpression); + } + if !literal.is_empty() { + parts.push(Part::Text(take(&mut literal))); + } + parts.push(Part::Slot); + } + '}' => return Err(AslError::InvalidExpression), + ch => literal.push(ch), + } + } + if count != slots { + return Err(AslError::InvalidExpression); + } + if !literal.is_empty() { + parts.push(Part::Text(literal)); + } + Ok(Self(parts)) + } + + pub(super) fn render( + &self, + args: &[Payload], + budget: &mut Budget, + ) -> Result { + let mut args = args.iter(); + let mut output = String::new(); + for part in &self.0 { + budget.work(1)?; + match part { + Part::Text(text) => { + budget.text(text.len())?; + append(&mut output, text)?; + } + Part::Slot => { + let value = args.next().ok_or(AslError::IntrinsicValue)?.value(); + match value { + Value::String(text) => { + budget.text(text.len())?; + append(&mut output, text)?; + } + Value::Null | Value::Bool(_) | Value::Number(_) => { + let encoded = encode(value)?; + let text = from_utf8(&encoded).map_err(|_| AslError::IntrinsicValue)?; + append(&mut output, text)?; + } + _ => return Err(AslError::IntrinsicValue), + } + } + } + } + if args.next().is_some() { + return Err(AslError::IntrinsicValue); + } + Payload::from_value(&Value::String(output)) + } +} + +fn append(output: &mut String, text: &str) -> Result<(), AslError> { + if output.len().saturating_add(text.len()) > MAX_PAYLOAD_BYTES { + return Err(AslError::DataLimit); + } + output + .try_reserve(text.len()) + .map_err(AslError::Allocation)?; + output.push_str(text); + Ok(()) +} diff --git a/crates/rustack-stepfunctions-core/src/expression/parser.rs b/crates/rustack-stepfunctions-core/src/expression/parser.rs new file mode 100644 index 00000000..927c5270 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/expression/parser.rs @@ -0,0 +1,191 @@ +//! Fully consumed bounded expression grammar; no input-bearing diagnostics. + +use serde_json::{Value, from_str}; +use winnow::{ + ModalResult, Parser, + token::{any, take_while}, +}; + +use super::{Function, Node, format::Format}; +use crate::{AslError, Path, Payload}; + +pub(super) fn compile(text: &str) -> Result { + if text.is_empty() || text.len() > 4096 || text.trim() != text { + return Err(AslError::InvalidExpression); + } + if text.starts_with('$') { + return text.parse::().map(Node::Path); + } + let mut input = text; + let node = call(&mut input, 0, &mut 1024)?; + if !input.is_empty() { + return Err(AslError::InvalidExpression); + } + Ok(node) +} + +fn consume(input: &mut &str, mut token: &'static str) -> Result<(), AslError> { + let parsed: ModalResult<&str> = token.parse_next(input); + parsed.map(|_| ()).map_err(|_| AslError::InvalidExpression) +} + +fn space(input: &mut &str) -> Result<(), AslError> { + let parsed: ModalResult<&str> = + take_while(0.., |ch: char| matches!(ch, ' ' | '\t' | '\r' | '\n')).parse_next(input); + parsed.map(|_| ()).map_err(|_| AslError::InvalidExpression) +} + +fn unit(remaining: &mut usize) -> Result<(), AslError> { + *remaining = remaining + .checked_sub(1) + .ok_or(AslError::InvalidExpression)?; + Ok(()) +} + +fn call(input: &mut &str, depth: usize, remaining: &mut usize) -> Result { + if depth >= 10 { + return Err(AslError::InvalidExpression); + } + unit(remaining)?; + let name: ModalResult<&str> = take_while(1.., |ch: char| { + ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_') + }) + .parse_next(input); + let function = Function::parse(name.map_err(|_| AslError::InvalidExpression)?)?; + consume(input, "(")?; + space(input)?; + let mut args = Vec::new(); + let mut format_text = None; + if !input.starts_with(')') { + loop { + if args.len() >= 128 { + return Err(AslError::InvalidExpression); + } + let is_format = function == Function::Format && args.is_empty(); + let (arg, raw) = argument(input, depth.saturating_add(1), remaining, is_format)?; + if is_format { + format_text = raw; + } + args.push(arg); + space(input)?; + if input.starts_with(')') { + break; + } + consume(input, ",")?; + space(input)?; + } + } + consume(input, ")")?; + if !function.arity(args.len()) { + return Err(AslError::InvalidExpression); + } + if function == Function::JsonMerge { + if let Some(Node::Literal(value)) = args.get(2) { + if value.value() != &Value::Bool(false) { + return Err(AslError::InvalidExpression); + } + } + } + let format = format_text + .map(|text| Format::compile(&text, args.len().saturating_sub(1))) + .transpose()?; + Ok(Node::Call { + function, + args, + format, + }) +} + +fn argument( + input: &mut &str, + depth: usize, + remaining: &mut usize, + is_format: bool, +) -> Result<(Node, Option), AslError> { + if input.starts_with("States.") { + return call(input, depth, remaining).map(|node| (node, None)); + } + unit(remaining)?; + if input.starts_with('\'') { + let (value, raw) = string(input, is_format)?; + return Ok(( + Node::Literal(Payload::from_value(&Value::String(value))?), + Some(raw), + )); + } + let token = argument_token(input)?; + if token.starts_with('$') { + return token.parse().map(|path| (Node::Path(path), None)); + } + if token.is_empty() || token.len() > 128 { + return Err(AslError::InvalidExpression); + } + let value: Value = from_str(token).map_err(|_| AslError::InvalidExpression)?; + if !matches!(value, Value::Number(_) | Value::Bool(_) | Value::Null) { + return Err(AslError::InvalidExpression); + } + Ok((Node::Literal(Payload::from_value(&value)?), None)) +} + +fn argument_token<'a>(input: &mut &'a str) -> Result<&'a str, AslError> { + let source = *input; + let mut brackets = 0_usize; + let mut quoted = false; + let mut escaped = false; + let mut end = source.len(); + for (index, ch) in source.char_indices() { + if quoted { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '\'' { + quoted = false; + } + } else { + match ch { + '\'' => quoted = true, + '[' => brackets = brackets.checked_add(1).ok_or(AslError::InvalidExpression)?, + ']' => brackets = brackets.checked_sub(1).ok_or(AslError::InvalidExpression)?, + ',' | ')' if brackets == 0 => { + end = index; + break; + } + _ => {} + } + } + } + if quoted || brackets != 0 { + return Err(AslError::InvalidExpression); + } + let token = source.get(..end).ok_or(AslError::InvalidExpression)?; + *input = source.get(end..).ok_or(AslError::InvalidExpression)?; + Ok(token.trim_end_matches([' ', '\t', '\r', '\n'])) +} + +fn string(input: &mut &str, is_format: bool) -> Result<(String, String), AslError> { + consume(input, "'")?; + let mut value = String::new(); + let mut raw = String::new(); + loop { + let parsed: ModalResult = any.parse_next(input); + match parsed.map_err(|_| AslError::InvalidExpression)? { + '\'' => return Ok((value, raw)), + '\\' => { + let parsed: ModalResult = any.parse_next(input); + let ch = parsed.map_err(|_| AslError::InvalidExpression)?; + if !matches!(ch, '\'' | '\\' | '{' | '}') { + return Err(AslError::InvalidExpression); + } + raw.push('\\'); + raw.push(ch); + value.push(ch); + } + '{' | '}' if !is_format => return Err(AslError::InvalidExpression), + ch => { + raw.push(ch); + value.push(ch); + } + } + } +} diff --git a/crates/rustack-stepfunctions-core/src/expression/tests.rs b/crates/rustack-stepfunctions-core/src/expression/tests.rs new file mode 100644 index 00000000..8d148983 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/expression/tests.rs @@ -0,0 +1,441 @@ +use std::{collections::BTreeSet, error::Error}; + +use proptest::prelude::*; +use rstest::rstest; +use rustack_core::json::JsonError; +use serde_json::{Map, Value, json}; +use uuid::{Builder, Uuid, Variant, Version}; + +use crate::{AslError, EvaluationContext, Expression, Payload, SystemUuidSource, UuidSource}; + +type TestResult = Result<(), Box>; + +#[derive(Default)] +struct Fixed(u8); +impl UuidSource for Fixed { + fn next_uuid(&mut self) -> Result { + self.0 = self.0.checked_add(1).ok_or(AslError::WorkLimit)?; + Ok(Builder::from_random_bytes([self.0; 16]).into_uuid()) + } +} + +fn evaluate(expression: &str, input: &Value) -> Result { + let expression: Expression = expression.parse()?; + let data = Payload::from_value(input)?; + expression.evaluate_with_source( + &data, + &EvaluationContext::from(&data), + &mut Fixed::default(), + ) +} + +#[test] +fn test_should_execute_all_twelve_independent_intrinsic_catalog_cases() -> TestResult { + let catalog: Value = serde_json::from_str(include_str!( + "../../../../tests/workflows/semantic-cases.json" + ))?; + let mut count = 0; + for case in catalog["cases"].as_array().ok_or("missing cases")? { + if case["kind"] != "intrinsic" { + continue; + } + let result = evaluate( + case["expression"].as_str().ok_or("missing expression")?, + &case["input"], + )?; + if case.get("expectedPattern").is_some() { + let id = Uuid::parse_str(result.value().as_str().ok_or("not UUID")?)?; + assert_eq!(id.get_version(), Some(Version::Random)); + assert_eq!(id.get_variant(), Variant::RFC4122); + } else { + assert_eq!(result.value(), &case["expected"], "{}", case["id"]); + } + count += 1; + } + assert_eq!(count, 12); + Ok(()) +} + +#[rstest] +#[case("States.Array()",json!(null),json!([]))] +#[case("States.Array(true, false, null, -1, 1.25e1)",json!({}),json!([true,false,null,-1,12.5]))] +#[case(r"States.Array('\'', '\\', '\{', '\}')",json!({}),json!(["'","\\","{","}"]))] +#[case(r"States.Format('it\'s {}', 'fine')",json!({}),json!("it's fine"))] +#[case(r"States.Format('\{\} {}', 'x')",json!({}),json!("{} x"))] +#[case(r"States.Format('\\{}', 'x')",json!({}),json!("\\x"))] +#[case(r"States.Format(States.Format('\{\}'), 'x')",json!({}),json!("x"))] +#[case("States.Format('{}:{}:{}:{}', true, null, 3, 'text')",json!({}),json!("true:null:3:text"))] +#[case("States.Format($.format, $.x)",json!({"format":"\\{ {} \\}","x":"yes"}),json!("{ yes }"))] +#[case("States.Array(States.ArrayLength($.x), States.ArrayGetItem($.x, 1))",json!({"x":[1,2]}),json!([2,2]))] +#[case("States.ArrayContains($.x, 1.0)",json!({"x":[1]}),json!(true))] +#[case("States.ArrayContains($.x, $.y)",json!({"x":[{"a":[null,2.0]}],"y":{"a":[null,2]}}),json!(true))] +#[case("States.ArrayContains($.x, $.y)",json!({"x":[{"a":2}],"y":{"b":2}}),json!(false))] +#[case("States.ArrayPartition($.x, 1.5)",json!({"x":[1,2,3]}),json!([[1,2],[3]]))] +#[case("States.ArrayPartition($.x, 1)",json!({"x":[]}),json!([]))] +#[case("States.ArrayPartition($.x, 2147483647)",json!({"x":[1]}),json!([[1]]))] +#[case("States.StringSplit($.x, ',')",json!({"x":",a,,b,"}),json!(["a","b"]))] +#[case("States.StringSplit($.x, '')",json!({"x":"ab"}),json!(["ab"]))] +#[case("States.StringSplit($.x, '')",json!({"x":""}),json!([]))] +#[case("States.StringSplit($.x, '中🙂')",json!({"x":"中a🙂b中"}),json!(["a","b"]))] +#[case(r#"States.StringToJson('\{"n":1\}')"#,json!({}),json!({"n":1}))] +#[case("States.StringSplit($.x, '\n')",json!({"x":"one\ntwo"}),json!(["one","two"]))] +#[case("States.Array('\0')",json!({}),json!(["\0"]))] +#[case("States.JsonMerge($.a, $.b, $.deep)",json!({"a":{"x":{"left":1},"keep":2},"b":{"x":{"right":3}},"deep":false}),json!({"x":{"right":3},"keep":2}))] +fn test_should_preserve_selected_intrinsic_value_and_escape_semantics( + #[case] expression: &str, + #[case] data: Value, + #[case] expected: Value, +) -> TestResult { + assert_eq!(evaluate(expression, &data)?.value(), &expected); + Ok(()) +} + +#[test] +fn test_should_pin_historical_aws_half_rounding_and_split_edges() -> TestResult { + for (left, right, result) in [ + (1.49, 1.50, 3), + (1.50, 1.51, 4), + (-1.49, -1.50, -2), + (-1.50, -1.51, -3), + (0.5, 0.0, 1), + (2.5, 0.0, 3), + (-0.5, 0.5, 1), + (-0.5, 0.0, 0), + (-1.5, 0.0, -1), + (-2.5, 0.0, -2), + ] { + assert_eq!( + evaluate("States.MathAdd($.a, $.b)", &json!({"a":left,"b":right}))?.value(), + &json!(result) + ); + } + for (input, delimiters, expected) in [ + (" ", ",", vec![" "]), + (" , ", ",", vec![" ", " "]), + (", , ,", ",", vec![" ", " "]), + (",,,,", ",", vec![]), + ( + "This.is+a,test=string", + ".+,=", + vec!["This", "is", "a", "test", "string"], + ), + ( + "split on T and \nnew line", + "T\n", + vec!["split on ", " and ", "new line"], + ), + ] { + assert_eq!( + evaluate( + "States.StringSplit($.a, $.b)", + &json!({"a":input,"b":delimiters}) + )? + .value(), + &json!(expected) + ); + } + assert_eq!( + evaluate( + "States.MathAdd($.x, 0)", + &json!({"x":0.499_999_999_999_999_94}) + )? + .value(), + &json!(0) + ); + Ok(()) +} + +#[rstest] +#[case("")] +#[case("'literal'")] +#[case("true")] +#[case(" States.Array()")] +#[case("States.Array() ")] +#[case("States.Array ()")] +#[case("States.Unknown()")] +#[case("States.MathRandom(0,1)")] +#[case("States.Array(,)")] +#[case("States.Array(1,)")] +#[case("States.Array(1 2)")] +#[case("States.Array([])")] +#[case("States.Array({})")] +#[case("States.Array(+1)")] +#[case("States.Array(1e999)")] +#[case(r"States.Array('\n')")] +#[case("States.Array('unterminated)")] +#[case(r"States.Array('unescaped{')")] +#[case("States.ArrayLength()")] +#[case("States.MathAdd(1)")] +#[case("States.UUID(1)")] +#[case("States.JsonMerge($.a,$.b,true)")] +#[case("States.Format('{}')")] +#[case("States.Format('{x}', 1)")] +#[case("States.Format('}', 1)")] +#[case("States.Format('{}', 1, 2)")] +#[case("States.Array($.x['bad)")] +fn test_should_reject_malformed_unsupported_or_ambiguous_expression_syntax(#[case] text: &str) { + assert!(text.parse::().is_err(), "{text}"); +} + +#[rstest] +#[case("States.ArrayLength(1)")] +#[case("States.ArrayGetItem(1, 0)")] +#[case("States.ArrayGetItem($.a, -1)")] +#[case("States.ArrayGetItem($.a, 1.0)")] +#[case("States.ArrayGetItem($.a, 1)")] +#[case("States.ArrayPartition($.a, 0)")] +#[case("States.ArrayPartition($.a, -1)")] +#[case("States.ArrayPartition($.a, 0.49)")] +#[case("States.MathAdd('1', 0)")] +#[case("States.MathAdd(2147483647, 1)")] +#[case("States.MathAdd(-2147483648, -1)")] +#[case("States.MathAdd(2147483648, 0)")] +#[case("States.StringSplit(1, ',')")] +#[case("States.Format('{}', $.a)")] +#[case("States.JsonMerge($.a, $.a, false)")] +fn test_should_return_typed_intrinsic_value_failures(#[case] text: &str) { + assert!( + matches!( + evaluate(text, &json!({"a":[1]})), + Err(AslError::IntrinsicValue) + ), + "{text}" + ); +} + +#[test] +fn test_should_preserve_context_and_quoted_argument_paths() -> TestResult { + let data = Payload::from_value(&json!({"a,b)":1,"quote'key":2}))?; + let context = EvaluationContext::from_value(&json!({"Execution":{"Name":"run"}}))?; + let expression: Expression = + r"States.Array($['a,b)'], $['quote\'key'], $$.Execution.Name)".parse()?; + assert_eq!( + expression + .evaluate_with_source(&data, &context, &mut Fixed::default())? + .value(), + &json!([1, 2, "run"]) + ); + assert!(matches!( + evaluate("States.Array($.missing)", &json!({})), + Err(AslError::Missing) + )); + assert!(matches!( + evaluate("States.Format($.format)", &json!({"format":"{"})), + Err(AslError::IntrinsicValue) + )); + assert!(matches!( + evaluate( + "States.StringToJson($.text)", + &json!({"text":"{\"a\":1,\"a\":2}"}) + ), + Err(AslError::Json(JsonError::Duplicate)) + )); + Ok(()) +} + +#[test] +fn test_should_enforce_expression_bytes_argument_count_and_call_nesting() -> TestResult { + let fixed = "States.Array('')".len(); + let exact = format!("States.Array('{}')", "x".repeat(4096 - fixed)); + assert_eq!(exact.len(), 4096); + exact.parse::()?; + assert!(format!("{exact}x").parse::().is_err()); + format!("States.Array({})", vec!["0"; 128].join(",")).parse::()?; + assert!( + format!("States.Array({})", vec!["0"; 129].join(",")) + .parse::() + .is_err() + ); + assert!( + format!("States.Array({})", "1".repeat(129)) + .parse::() + .is_err() + ); + let mut nested = "$.x".to_owned(); + for _ in 0..10 { + nested = format!("States.Array({nested})"); + } + evaluate(&nested, &json!({"x":1}))?; + assert!( + format!("States.Array({nested})") + .parse::() + .is_err() + ); + let full = format!("States.Array({})", vec!["0"; 127].join(",")); + let short = format!("States.Array({})", vec!["0"; 126].join(",")); + let mut groups = vec![full.clone(); 7]; + groups.push(short); + let exact_nodes = format!("States.Array({})", groups.join(",")); + assert!(exact_nodes.len() < 4096); + evaluate(&exact_nodes, &json!(null))?; + groups.pop(); + groups.push(full); + assert!(matches!( + format!("States.Array({})", groups.join(",")).parse::(), + Err(AslError::InvalidExpression) + )); + Ok(()) +} + +#[test] +fn test_should_bound_output_and_share_path_data_and_materialization_budgets() -> TestResult { + assert!(matches!( + evaluate( + "States.Format('{}{}', $.x, $.x)", + &json!({"x":"a".repeat(140_000)}) + ), + Err(AslError::DataLimit) + )); + assert!(matches!( + evaluate( + "States.JsonToString($.x)", + &json!({"x":"\"".repeat(130_000)}) + ), + Err(AslError::DataLimit) + )); + let many = format!("States.Array({})", vec!["$.x"; 128].join(",")); + assert!(matches!( + evaluate(&many, &json!({"x":"a".repeat(100_000)})), + Err(AslError::WorkLimit) + )); + assert_eq!( + evaluate("States.ArrayLength($[*])", &json!(vec![0; 9997]))?.value(), + &json!(9997) + ); + assert!(matches!( + evaluate("States.ArrayLength($[*])", &json!(vec![0; 9998])), + Err(AslError::WorkLimit) + )); + assert!(matches!( + evaluate( + "States.ArrayContains($.a, $.x)", + &json!({"a":vec!["a";100],"x":"a".repeat(100_000)}) + ), + Err(AslError::WorkLimit) + )); + let too_many: String = (0x1000..0x1401).filter_map(char::from_u32).collect(); + assert!(matches!( + evaluate( + "States.StringSplit('x', $.delim)", + &json!({"delim":too_many}) + ), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +#[test] +fn test_should_keep_large_integer_equality_exact_and_reject_float_aliases() -> TestResult { + assert_eq!( + evaluate( + "States.ArrayContains($.a, $.x)", + &json!({"a":[u64::MAX],"x":u64::MAX}) + )? + .value(), + &json!(true) + ); + assert_eq!( + evaluate( + "States.ArrayContains($.a, $.x)", + &json!({"a":[u64::MAX],"x":u64::MAX-1}) + )? + .value(), + &json!(false) + ); + let data = Payload::parse(br#"{"a":[9007199254740992],"x":9007199254740993.0}"#)?; + let expression: Expression = "States.ArrayContains($.a, $.x)".parse()?; + assert!(matches!( + expression.evaluate(&data, &EvaluationContext::from(&data)), + Err(AslError::NumericRange) + )); + Ok(()) +} + +#[test] +fn test_should_charge_duplicate_delimiters_and_failed_object_key_scans() -> TestResult { + assert!(matches!( + evaluate( + "States.StringSplit('x', $.separators)", + &json!({"separators":"a".repeat(20_000)}) + ), + Err(AslError::WorkLimit) + )); + let mut candidate = Map::new(); + for index in 0..199 { + candidate.insert(format!("k{index:03}"), json!(0)); + } + let mut needle = candidate.clone(); + candidate.insert("z".into(), json!(0)); + needle.insert("zz".into(), json!(0)); + let small = json!({"items":[candidate],"needle":needle}); + assert_eq!( + evaluate("States.ArrayContains($.items, $.needle)", &small)?.value(), + &json!(false) + ); + let repeated = json!({"items":vec![candidate;100],"needle":needle}); + assert!(matches!( + evaluate("States.ArrayContains($.items, $.needle)", &repeated), + Err(AslError::WorkLimit) + )); + Ok(()) +} + +struct Fails; +impl UuidSource for Fails { + fn next_uuid(&mut self) -> Result { + Err(AslError::Entropy(getrandom::Error::UNSUPPORTED)) + } +} +struct InvalidId; +impl UuidSource for InvalidId { + fn next_uuid(&mut self) -> Result { + Ok(Uuid::nil()) + } +} + +#[test] +fn test_should_use_typed_uuid_sources_and_retain_redacted_entropy_failure() -> TestResult { + let expression: Expression = "States.UUID()".parse()?; + let data = Payload::parse(b"null")?; + let context = EvaluationContext::from(&data); + let mut system = SystemUuidSource; + let mut ids = BTreeSet::new(); + for _ in 0..8 { + let id = system.next_uuid()?; + assert_eq!(id.get_version(), Some(Version::Random)); + ids.insert(id); + } + assert_eq!(ids.len(), 8); + let fixed = expression.evaluate_with_source(&data, &context, &mut Fixed::default())?; + assert_eq!( + fixed.value(), + &json!(Builder::from_random_bytes([1; 16]).into_uuid().to_string()) + ); + let error = expression + .evaluate_with_source(&data, &context, &mut Fails) + .err() + .ok_or("missing failure")?; + assert!(error.source().is_some()); + assert_eq!(format!("{error:?}"), "ASL UUID entropy unavailable"); + assert!(matches!( + expression.evaluate_with_source(&data, &context, &mut InvalidId), + Err(AslError::IntrinsicValue) + )); + assert_eq!( + format!("{:?}", "States.Array('private')".parse::()?), + "Expression()" + ); + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + #[test] + fn test_should_parse_arbitrary_utf8_expressions_without_panicking(text in ".{0,4200}") { let _=text.parse::(); } + + #[test] + fn test_should_preserve_checked_integer_addition(a in -1_000_000_i32..1_000_000,b in -1_000_000_i32..1_000_000) { + let result=evaluate("States.MathAdd($.a,$.b)",&json!({"a":a,"b":b})).map_err(|e|TestCaseError::fail(e.to_string()))?; + prop_assert_eq!(result.value(),&json!(a+b)); + } +} diff --git a/crates/rustack-stepfunctions-core/src/lib.rs b/crates/rustack-stepfunctions-core/src/lib.rs index 14c537e4..86ec5397 100644 --- a/crates/rustack-stepfunctions-core/src/lib.rs +++ b/crates/rustack-stepfunctions-core/src/lib.rs @@ -8,12 +8,14 @@ mod choice; mod error; +mod expression; mod path; mod payload; mod wait; pub use choice::Predicate; pub use error::AslError; +pub use expression::{Expression, SystemUuidSource, UuidSource}; pub use path::{MatchKind, Path, PathMatches, ReferencePath, ResultPath}; pub use payload::{EvaluationContext, MAX_PAYLOAD_BYTES, Payload}; pub use wait::WaitSource; diff --git a/crates/rustack-stepfunctions-core/src/path.rs b/crates/rustack-stepfunctions-core/src/path.rs index e65411dd..46c6aaf0 100644 --- a/crates/rustack-stepfunctions-core/src/path.rs +++ b/crates/rustack-stepfunctions-core/src/path.rs @@ -252,18 +252,27 @@ impl Path { &self, data: &'a Payload, context: &'a EvaluationContext, + ) -> Result, AslError> { + let mut remaining = MAX_WORK; + self.select_with_work(data, context, &mut remaining) + } + + pub(crate) fn select_with_work<'a>( + &self, + data: &'a Payload, + context: &'a EvaluationContext, + remaining: &mut usize, ) -> Result, AslError> { let root = match self.root { Root::Data => data.value(), Root::Context => context.value(), }; let mut values = vec![root]; - let mut remaining = MAX_WORK; for segment in &self.segments { let mut next = Vec::new(); for value in values { - spend(&mut remaining, 1)?; - select_segment(segment, value, &mut next, &mut remaining)?; + spend(remaining, 1)?; + select_segment(segment, value, &mut next, remaining)?; } values = next; } diff --git a/docs/index.md b/docs/index.md index 8767ee7a..12bb200c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ - [Bollard Docker Transport Boundaries](./research/study-bollard-docker-transport-boundaries.md) - Pinned response collection, attach framing/EOF and diagnostic behavior; generated-type reuse, bounded production wire implementation, platform identity/default merge validation, owned image pin/quota/cleanup preparation and actual RIC/diagnostic qualification; Bollard and historical Moby sources are pinned. - [Container Lambda Relay Spike](./research/spike-lambda-container-relay.md) - Actual binary attach/shared-netns/official-RIC execution on arm64 and emulated amd64, with reproducible probes and explicit qualification limits. -- [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors and AWS-validated nested result shapes. +- [Step Functions Semantic Oracle Spike](./research/spike-stepfunctions-semantic-oracle.md) - Pinned Smithy/SDK/fixture evidence, independently reproduced numeric timestamp gap, callback errors, AWS-validated nested result shapes and historical intrinsic rounding/split boundaries. - [Step Functions and Container Lambda Survey](./research/survey-stepfunctions-lambda-container-runtime.md) - Official ASL/integration/image/runtime contracts, inspected Rustack gaps, current dependency APIs and the experiments required before implementation. - [LocalStack S3 Implementation Research](./research/localstack-s3-research.md) - Comprehensive analysis of what a Rust-based LocalStack S3 service needs to support for feature parity. - [s3s Crate Ecosystem Research](./research/s3s-crate-research.md) - Comprehensive analysis of the s3s Rust crate for building S3-compatible services. diff --git a/docs/research/spike-stepfunctions-semantic-oracle.md b/docs/research/spike-stepfunctions-semantic-oracle.md index 0d377c64..8d5a266a 100644 --- a/docs/research/spike-stepfunctions-semantic-oracle.md +++ b/docs/research/spike-stepfunctions-semantic-oracle.md @@ -34,3 +34,11 @@ Primary references: [Smithy JSON timestamp serialization](https://smithy.io/2.0/ ## Evidence limits No live AWS differential suite was run, and no complete emulator was used as a semantic oracle. Historical AWS-validated snapshots supplement current official documentation only where they do not conflict. Current guide-compatible States.Runtime behavior is selected for unevaluable paths; internal Path/Intrinsic/ResultPath diagnostics remain separate. Native x86 hardware, Docker Desktop/rootless qualification, full execution-history parity and the final performance gates remain unverified. + +## Intrinsic boundary evidence inspected during P2 + +The same pinned LocalStack submodule contains AWS-validated historical intrinsic snapshots; these are observed outputs, not emulator implementation copied into Rustack. The [math test](../../vendors/localstack/tests/aws/services/stepfunctions/v2/intrinsic_functions/test_math_operations.py) and [snapshot](../../vendors/localstack/tests/aws/services/stepfunctions/v2/intrinsic_functions/test_math_operations.snapshot.json), validated2023-09-13, show -1.5→-1, -0.5→0, 0.5→1 and2.5→3 for addition with zero. The source comments about negative-half Java behavior conflict with those recorded values; use the recorded values. This pins nearest rounding with ties toward positive infinity. A floor/fraction comparison avoids introducing a separate addition-rounding error immediately below0.5. + +The [split snapshot](../../vendors/localstack/tests/aws/services/stepfunctions/v2/intrinsic_functions/test_string_operations.snapshot.json), recorded2024-12-05, drops empty fragments at leading, repeated and trailing separators, preserves spaces and treats all delimiter scalars as separators. `,,,,` split on comma yields an empty array. New production tests use these independent edge inputs plus the existing12-case intrinsic catalog. Empty-delimiter behavior and int32 result overflow remain explicit local policies, not new cloud observations. + +The [generic snapshot](../../vendors/localstack/tests/aws/services/stepfunctions/v2/intrinsic_functions/test_generic.snapshot.json), recorded2023-02-09, also shows a cloud object-formatting extension. The current formal ASL scalar-only Format contract takes precedence in Rustack's selected subset; array/object substitutions reject. Scalar/null/numeric/string formatting and escaped brace provenance are covered separately. No new AWS execution or complete workflow-engine acceptance is claimed. diff --git a/specs/index.md b/specs/index.md index 4a833661..c8642a2e 100644 --- a/specs/index.md +++ b/specs/index.md @@ -6,7 +6,7 @@ Status: P0 evidence complete; P1 remains in progress. Implemented and independently reviewed foundations include the shared runtime/HTTP/relay/client, bounded Docker wire, immutable image resolution and leased deployment/provider settlement, persistent directory ownership, allocation policy, retained environment lifecycle and FIFO/warm/LRU dispatch. Delivery-based handler clocks and invocation-scoped observed log windows are qualified through actual image execution. Per-call pool cleanup settlement is retained above dispatch workers and resumable after cancellation or waiter expiry. The Docker Executor/provider/HTTP composition is independently reviewed and exercised through ordinary boto3 against actual images, using controlled fixture readiness. Durable container intents now survive directory reacquisition; historical recovery retains unknown absence and retires only exactly verified identities, with independently reviewed, actual pair-retirement evidence on both retained daemons. Startup network qualification pins full bridge IDs; actual pinned Rust probes validate kernel resource/isolation settings and joined retirement without granting readiness. B01 dependency security, B03 strict boundaries, B04 deployment validation and B05 post-bootstrap initialization classification are closed. See the [implementation ledger](./rustack-workflows-90-impl-plan.md#11-implementation-execution-record) and [review](./rustack-workflows-93-review.md) for exact evidence. -Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Bounded compiled paths/payloads, Choice predicates and pure Wait source resolution are implemented and independently reviewed, including context headroom, exact timestamp fractions and conservative numeric comparisons. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. +Production Docker selection remains unavailable until the production startup coordinator, remaining historical image/other effects, allowed pulls, endpoint reachability and final startup/configuration acceptance pass M0. The P2 generated Step Functions model/numeric timestamp foundation is implemented and independently reviewed. The typed HTTP transport is implemented and independently reviewed, including signed ordinary SDK qualification against a controlled handler. Bounded compiled paths/payloads, Choice predicates, pure Wait sources and the12-call intrinsic expression profile are implemented and independently reviewed, including context headroom, exact timestamp fractions and conservative numeric comparisons. Main-service registration, registry, complete ASL compiler/engine and the remaining P2–P5 requirements remain pending. Newly refreshed zstd advisories are tracked as open B06 in the review; the earlier B01 closure is historical evidence, not a current clean audit claim. Per explicit user direction, these workflow components now take priority over the retained unfinished Docker-image checkpoint; final image/platform acceptance is still required. The complete requested scope remains M0–M4. All feature documents below are in English; start here for the owned ASL engine and actual Docker image Lambda execution. | Order | Specification | Type / purpose | |---|---|---| diff --git a/specs/rustack-workflows-30-asl-design.md b/specs/rustack-workflows-30-asl-design.md index 784d5cc8..64183ec2 100644 --- a/specs/rustack-workflows-30-asl-design.md +++ b/specs/rustack-workflows-30-asl-design.md @@ -93,6 +93,47 @@ Support `Execution.{Id,Input,Name,RoleArn,StartTime}`, `StateMachine.{Id,Name}`, Intrinsic allowlist: States.Format, StringToJson, JsonToString, Array, ArrayLength, ArrayGetItem, ArrayContains, ArrayPartition, StringSplit, MathAdd, UUID and JsonMerge. JsonMerge supports only `States.JsonMerge(a,b,false)` shallow merging. Reject all others at creation. Nesting is at most ten; each evaluation permits at most 10,000 AST operations and remains within the output budget. P0 fixtures pin escaping, Format, StringSplit delimiter behavior, MathAdd rounding/ranges, shallow merge and UUID format. Do not substitute Rust formatter/split semantics. Production UUIDs use secure randomness; deterministic tests inject a typed ID source rather than seed production randomness. +#### 2.3.1 Executable expression contract + +Expression compiles a complete Path or an allowlisted States call, without evaluation during creation. Use the existing winnow lexer with a private typed AST. Bound each expression to4,096 UTF-8 bytes,1,024 nodes, ten nested calls and128 arguments per call. Full consumption is mandatory; reject surrounding whitespace, unknown functions, wrong arity, trailing commas, malformed numbers and unsupported syntax. Internal ASCII whitespace may surround arguments/commas. Arguments may be single-quoted strings, JSON numbers/booleans/null, compiled Paths or nested calls; object/array literals are constructed through Array/StringToJson. Numeric lexemes are at most128 bytes and use existing correctly rounded JSON conversion. These are local bounds, not full ASL parser support. + +Decode apostrophe/backslash/brace escapes exactly once. String literals support only `\'`, `\\`, `\{` and `\}` escapes; reject dangling/unknown escapes. Business control characters already decoded from the outer JSON string remain data and are safely JSON-encoded in results. Preserve a format literal's escape provenance until Format processes it, so an escaped brace is not accidentally reinterpreted as a placeholder. Literal non-format braces must be escaped. Format accepts only `{}` slots, checks slot/argument equality and accepts scalar substitutions (string without JSON quotes; number/bool/null compact JSON spelling); array/object substitutions fail under the formal ASL scalar profile. A runtime first argument may itself be a path or nested expression yielding a format string. No host formatter, regex or string interpolation evaluates user syntax. + +| Call | Arity | Selected value behavior | +|---|---:|---| +| Format |1–128| Scalar replacement; brace/backslash escapes; dynamic format checked at evaluation | +| StringToJson |1| Bounded duplicate-key-safe JSON decoding of a string | +| JsonToString |1| Compact bounded JSON text, then ordinary JSON string encoding | +| Array |0–128| Preserve argument order | +| ArrayLength |1| Array count | +| ArrayGetItem |2| Nonnegative JSON integer index; no growth, coercion or missing→null | +| ArrayContains |2| Bounded structural JSON equality; numeric representation differences use supported finite comparison | +| ArrayPartition |2| Positive chunk size rounded as below; stable final short chunk | +| StringSplit |2| Each delimiter scalar separates; discard empty fragments; empty delimiter preserves nonempty input as one fragment | +| MathAdd |2| Round each finite argument to nearest integer, ties toward positive infinity; require int32 operands and checked int32 sum | +| UUID |0| Fallible OS entropy, RFC4122 variant/version4; no hidden deterministic production seed | +| JsonMerge |3| Objects only, false third value; right-side keys replace left, no recursive merge | + +MathAdd/partition rounding is pinned to the historical AWS-validated math fixtures, including negative halves; implement floor plus a fractional comparison, avoiding `floor(x+0.5)` rounding at adjacent representable values. ArrayGetItem deliberately requires an integer JSON number rather than rounding floats. For mixed numeric structural equality, apply the Choice interoperability range so large integer/float values cannot silently alias; equal integer representations compare exactly. Empty-delimiter split and checked result overflow are explicit local choices. Primary language/function contracts: [ASL](https://states-language.net/spec.html), [AWS intrinsics](https://docs.aws.amazon.com/step-functions/latest/dg/intrinsic-functions.html); historical boundaries are recorded in the [semantic memo](../docs/research/spike-stepfunctions-semantic-oracle.md). + +Every evaluation shares10,000 AST/data work units,4MiB of scanned text and4MiB of cumulatively materialized encoded payloads. These local budgets include nested calls, array equality/partition members and string delimiter construction (at most1,024 unique delimiter scalars). Reject exhaustion with WorkLimit; never return a partial result. Each intermediate result independently satisfies the256KiB/depth64/member65,536 payload contract, including connector/context selections before another function can discard them. Encode arrays/partitions/merges through borrowed views and the bounded writer before building an owned result; bound variadic argument retention by the shared materialization budget. Allocation overhead is separate from encoded byte accounting. Callers cannot mutate source payload/context. + +Expression::evaluate uses a concrete OS-backed UUID source. A separate synchronous UuidSource port enables typed deterministic/failure fixtures; validate injected UUID version/variant before publishing. Use getrandom0.4.3 fill and uuid's existing random-byte builder, preserving typed entropy failures rather than using a panicking random convenience API. Both packages are already in the workspace graph; no unrelated dependency version upgrade is required. System entropy can block during early boot: the future async actor must inject pre-acquired IDs or evaluate through a bounded retained blocking worker, rather than make an entropy syscall inside its scheduling loop. Diagnostics and Debug remain redacted. Tests execute the12 independent intrinsic catalog examples and historical half/split edge inputs, nested/escaped/path/context grammar, exact limits, type/missing errors, overflow, output expansion, work exhaustion and UUID success/source failure. Expression acceptance does not qualify template/pipeline/compiler or workflow execution by itself. + +```text +┌─ bounded expression text ─────────────────────────────────────────┐ +│ winnow tokens → private call/path/scalar AST │ +│ arity + escapes + static format checks; no source in diagnostics │ +└─────────────────────────────┬────────────────────────────────────┘ + ▼ shared evaluation budgets +┌─ pure evaluator ──────────────────────────────────────────────────┐ +│ immutable data/context → bounded arguments → intrinsic operation │ +│ UUID source → fallible entropy borrowed output views │ +│ ▼ │ +│ bounded writer → validated Payload → next expression/template │ +└──────────────────────────────────────────────────────────────────┘ +``` + ### 2.4 Choice and Wait Choice supports short-circuit And/Or/Not, IsPresent/IsNull/IsString/IsNumeric/IsBoolean/IsTimestamp; String/Numeric/Timestamp Equals/LessThan/GreaterThan/LessThanEquals/GreaterThanEquals and their Path variants; BooleanEquals/Path and StringMatches. Do not coerce strings to numbers. IsPresent safely tests missing data; other missing-reference errors are runtime failures. Present values of inappropriate comparison types yield false. Distinguish missing, null and wrong type in fixtures. StringMatches implements AWS wildcard/escaping grammar, not user regex. Reject NaN/Infinity; preserve numeric comparison precision within the documented binary64 interoperability range. diff --git a/specs/rustack-workflows-70-verification-plan.md b/specs/rustack-workflows-70-verification-plan.md index 7ce0163c..44359f12 100644 --- a/specs/rustack-workflows-70-verification-plan.md +++ b/specs/rustack-workflows-70-verification-plan.md @@ -190,3 +190,9 @@ Raw JSON regressions must include decimal/exponent forms at and beyond the conse ## Step Functions Wait source qualification The core target includes all four Wait source forms and a512-case static/dynamic seconds equivalence property. Test exact365-day STANDARD caps, zero/past waits, uppercase timestamp separators, numeric offsets, target-only truncation, long target fractions, fractional current samples, duplicate/unknown fields and dynamic missing/null/wrong-type/range errors. Confirm the component returns only a relative Duration and never reads a clock or starts a timer. Core166 unit cases/18 doctests and strict checks pass. These tests do not qualify generation-tagged timer ownership, monotonic scheduling, execution deadline races or cancellation; the execution actor remains responsible for those gates. + +## Step Functions expression/intrinsic qualification + +`make test-stepfunctions-core` now executes the12 independent intrinsic catalog cases plus historical AWS rounding/split edge inputs. Cover format escape provenance (including nested/dynamic formatting), quoted/context paths, scalar substitutions, shallow merge, structural numeric equality, checked int32 arithmetic, JSON duplicate rejection, Unicode delimiter sets and source/UUID failures. Exact4,096-byte,128-argument, ten-call and1,024-node boundaries are exercised; the next byte/argument/call/node rejects. Two new512-case properties cover hostile UTF-8 compilation and checked addition. + +Expansion cases exercise bounded string/JSON output, shared path work, cumulative argument materialization, string scans, delimiter cardinality and repeated delimiter work. Repeated200-key late object mismatches must consume lookup work even when comparison returns false before queued child visits. Keep a valid single mismatch as a control. The concrete entropy adapter produces valid distinct v4 IDs; injected failures preserve typed sources with redacted diagnostics and invalid injected IDs reject. Actor-side entropy isolation and template/pipeline/compiler integration remain separate gates. Current core242 unit cases/21 doctests and strict production/all-targets Clippy pass. diff --git a/specs/rustack-workflows-90-impl-plan.md b/specs/rustack-workflows-90-impl-plan.md index a881c735..ebacfe4d 100644 --- a/specs/rustack-workflows-90-impl-plan.md +++ b/specs/rustack-workflows-90-impl-plan.md @@ -162,6 +162,7 @@ The complete implementation objective includes P0–P5. This ledger records evid | 2.2 compiled paths/payloads | Implemented; complete compiler remains | ASL§2.1.1/D42 supplies bounded immutable payloads/context, compiled paths, borrowed missing/single/selection and transactional result merge.75 unit tests (three512-case properties),12 doctests and strict production checks pass; W105–W107 corrections cover borrowed deep-tree rejection and numeric/depth/member context headroom. Standard workspace build/test/fmt/Clippy/rustdoc pass with2,580 tests/415 gated, including the retained deferred Docker checkpoint. New existing-dependency advisory findings are tracked as B06; audit/deny currently fail that baseline, with no ignores added. | | 2.2 compiled Choice predicates | Implemented/reviewed; state/compiler integration remains | ASL§2.4.1/D43 implements typed literal/Path comparisons, logical short-circuit, exact timestamp fractions, conservative numeric interoperability and bounded glob/shared budgets. Core125 unit cases/15 doctests and strict checks pass, including90 operator combinations and four independent Choice catalog cases. W108–W109 are fixed/rechecked. Shared float_roundtrip adds no external package/version. Final current-worktree standard workspace gates pass with2,630/415 gated; regenerated model/strict HTTP and signed SDK regressions pass. Complete compiler, transition selection and engine remain required; B06 stays open. | | 2.2/2.3 compiled Wait source | Implemented/reviewed; actor scheduling remains | ASL§2.4.2/D44 supplies four typed sources and pure supplied-clock duration resolution; no sleeping task or timer is created. Independent review reports no findings. Core166 unit cases/18 doctests and strict checks pass; standard current-worktree workspace gates pass with2,671/415 gated, including deferred Docker changes. Monotonic timer/generation/deadline/cancellation acceptance remains required; B06 stays open. | +| 2.2 compiled expressions/intrinsics | Implemented/reviewed; template/pipeline/compiler remain | ASL§2.3.1/D45 implements all12 selected intrinsics, bounded winnow AST, format provenance, shared work/text/materialization budgets and fallible UUID source. W110–W111 are fixed/rechecked. Core242 unit cases/21 doctests and strict checks pass; standard current-worktree workspace gates pass with2,747/415 gated. Existing getrandom/uuid dependency edges add no package/version. B06 audit/deny remains open; actual template/compiler/actor integration is still required. | | 1.4, P2–P5 | Pending | Remain required; ordinary CLI Docker startup/platform and workflow-engine acceptance remain incomplete. | P0 measurements validate architecture feasibility, not production throughput or reduced delivery estimates. Keep the existing P1–P5 estimates until an implemented executor/engine provides calibration. B04 side-effect-free code-update DryRun and bounded request validation are implemented and independently reviewed with live SDK evidence. The task1.3 persistent directory capability (§4.1/D28) is implemented/reviewed. Immutable allocation policy (§4.2/D30) and retained environment ownership (§4.3/D31) are also implemented/reviewed. Bounded warm/LRU checkout (§4.4/D32) is implemented/reviewed. Delivery-based handler timing (§3.8/D33) is implemented/reviewed. Invocation log windows (§3.9/D34) are implemented/reviewed. B05 initialization-error classification is now implemented/reviewed. The supervised pool invocation receipt (§4.5/D35) is implemented/reviewed. The §4.6/D36 Docker Executor/provider composition is implemented/reviewed with controlled-readiness actual SDK evidence. Startup network/resource qualification (§4.7/D37) is implemented/reviewed and independently exercised with actual pinned Rust images. Container-specific durable restart reconciliation (§4.8/D38) is now implemented; historical image aliases and other outstanding mutation classes remain. Next complete those effects and allowed pulls, then combine the independent proofs in the production startup coordinator with endpoint reachability, task1.4 configuration/capability publication and complete platform acceptance. B01/B03/B04/B05 are now closed with the security, six-library strict boundary and live SDK/API evidence in §§7.1–7.2/D27. These baseline closures do not enable Docker execution or close M0; runtime consumers/probes and P2–P5 remain required. Docker execution is not yet advertised as available in Rustack. diff --git a/specs/rustack-workflows-93-review.md b/specs/rustack-workflows-93-review.md index bdf08d15..fa6ecdb3 100644 --- a/specs/rustack-workflows-93-review.md +++ b/specs/rustack-workflows-93-review.md @@ -519,6 +519,17 @@ ASL§2.4.2/D44 implements the four typed Wait sources, source/dynamic validation Core166 unit cases/18 doctests and strict production/all-targets Clippy pass, including41 Wait additions and one512-case property. Final current-worktree workspace build/test/fmt/Clippy/rustdoc pass with2,671 tests/415 gated, including the deferred Docker checkpoint. B06 audit/deny failures remain open. This does not close the compiler, registry, execution actor or M1 acceptance. +## P2 expression/intrinsic independent review + +ASL§2.3.1/D45 implements the complete12-call selected intrinsic profile, winnow AST, format provenance, shared resource accounting and fallible typed UUID sourcing. The independent read-only reviewer rechecked both corrections and reported no remaining in-slice findings. + +| ID | Priority / source | Closed correction | Verification | +|---|---|---|---| +| W110 | P2; `crates/rustack-stepfunctions-core/src/expression/eval.rs` split delimiter construction | Debit work before every construction step, including duplicate scalars |20,000 repeated identical delimiters exhaust the shared budget | +| W111 | P2; `crates/rustack-stepfunctions-core/src/expression/eval.rs` structural object equality | Charge key lookup work before lookup and bound queued comparison work | One200-key late mismatch returns false;100 repeated mismatches exhaust rather than bypass aggregate work | + +Core242 unit cases/21 doctests and strict checks pass, including all12 independent catalog cases and two new512-case properties. Final current-worktree workspace build/test/fmt/Clippy/rustdoc pass with2,747 tests/415 gated, including retained deferred Docker changes. Audit still reports the same4 vulnerabilities/10 unsound warnings in the existing zstd/Squib graph (477 packages); deny passes bans/licenses/sources and fails advisories under open B06. No external package/version was added and no ignores were introduced. Template/pipeline/compiler, async entropy isolation and full workflow acceptance remain required. + ## Existing baseline issues affecting final gates | ID | Priority / source | Verified evidence | Required disposition | diff --git a/specs/rustack-workflows-99-decisions.md b/specs/rustack-workflows-99-decisions.md index 73652cd4..2a79bc47 100644 --- a/specs/rustack-workflows-99-decisions.md +++ b/specs/rustack-workflows-99-decisions.md @@ -393,3 +393,12 @@ Supersede decisions with a new ID rather than rewriting their rationale after im - **Why**: static validation, dynamic failures and AWS whole-second target behavior can be pinned without a sleeping task or dependency on the HTTP caller lifetime. - **Pinned by**: [ASL§2.4.2](./rustack-workflows-30-asl-design.md#242-executable-wait-source-contract), [verification](./rustack-workflows-70-verification-plan.md). - **Date**: 2026-10-26. Use existing chrono/std/reference paths. STANDARD local wait cap is365 days; overall execution deadlines are independent. This is source evaluation, not scheduler acceptance. + +## D45 — Compile intrinsics with shared budgets and preserve format escape provenance + +- **Context**: Parameters/ResultSelector and the machine compiler need one reusable, bounded expression implementation. +- **Alternatives considered**: host formatting/splitting gives different semantics; reparsing on every state entry repeats validation; decoding all format escapes before slot recognition reinterprets escaped braces; per-call budgets multiply work under nesting; infallible UUID helpers can panic on entropy failure. +- **Decision**: compile the12-call allowlist into a private AST, preserve literal formatting provenance, share work/text/materialization budgets and encode borrowed collection views before owning results. Inject a typed UUID source and use fallible OS entropy in the concrete convenience adapter. +- **Why**: fixed bounds and historical independent edge inputs make expression behavior reviewable before template/compiler consumers. Async execution owners keep entropy acquisition off the scheduler loop. +- **Pinned by**: [ASL§2.3.1](./rustack-workflows-30-asl-design.md#231-executable-expression-contract), [semantic memo](../docs/research/spike-stepfunctions-semantic-oracle.md), [verification](./rustack-workflows-70-verification-plan.md). +- **Date**: 2026-10-26. Add normal dependency edges to already-resolved getrandom0.4.3/uuid1.26.0 without updating external versions. Formal scalar Format is selected over the historical cloud's object formatting extension. From 6c45434ada60e69bcc46a57e3ef0dfe7d557d28b Mon Sep 17 00:00:00 2001 From: Tyr Chen Date: Mon, 26 Oct 2026 07:47:49 -0700 Subject: [PATCH 038/106] phase 2: implement compiled templates and state data-flow phases Implement ASL section2.2.1/D46 with normalized collision-free templates, aggregate expression limits, shared evaluation budgets and streaming prepared output. Pure prepare/finish phases preserve original state input across resource calls. Independent review reports no findings. Core270 unit tests/31 doctests and strict checks pass, including four independent pipeline cases, exact4096/4097 combined-node limits and two new512-case properties. Standard workspace gates pass with2775 tests/415 gated in the current worktree. B06 audit/deny failure remains open. Machine compiler, actors/registry/Lambda bridge and full phase acceptance remain required; deferred Docker changes remain outside this commit. --- .../src/dataflow.rs | 228 ++++++++++++++++++ .../src/dataflow/tests.rs | 182 ++++++++++++++ .../rustack-stepfunctions-core/src/error.rs | 6 + .../src/expression.rs | 44 +++- crates/rustack-stepfunctions-core/src/lib.rs | 4 + .../src/template.rs | 224 +++++++++++++++++ .../src/template/tests.rs | 183 ++++++++++++++ specs/index.md | 2 +- specs/rustack-workflows-30-asl-design.md | 10 + .../rustack-workflows-70-verification-plan.md | 6 + specs/rustack-workflows-90-impl-plan.md | 3 +- specs/rustack-workflows-93-review.md | 6 + specs/rustack-workflows-99-decisions.md | 9 + 13 files changed, 901 insertions(+), 6 deletions(-) create mode 100644 crates/rustack-stepfunctions-core/src/dataflow.rs create mode 100644 crates/rustack-stepfunctions-core/src/dataflow/tests.rs create mode 100644 crates/rustack-stepfunctions-core/src/template.rs create mode 100644 crates/rustack-stepfunctions-core/src/template/tests.rs diff --git a/crates/rustack-stepfunctions-core/src/dataflow.rs b/crates/rustack-stepfunctions-core/src/dataflow.rs new file mode 100644 index 00000000..a7af59b8 --- /dev/null +++ b/crates/rustack-stepfunctions-core/src/dataflow.rs @@ -0,0 +1,228 @@ +//! Pure input/result phases; execution owners retain raw input across resources. + +use std::fmt; + +use serde_json::Value; + +use crate::{ + AslError, EvaluationContext, Path, Payload, ResultPath, SystemUuidSource, Template, UuidSource, + expression::Budget, + payload::{parse_component, validate_component}, +}; + +#[derive(Clone)] +enum Filter { + Empty, + Path(Path), +} +#[derive(Clone)] +enum Merge { + Replace, + Discard, + Path(ResultPath), +} + +/// Compiled state transformations with explicit original-input/result boundaries. +#[derive(Clone)] +#[non_exhaustive] +pub struct DataFlow { + input: Filter, + parameters: Option