diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 1c6e559c1..2552f6472 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -34,6 +34,9 @@ on: - tests/Unit/ServerParityWorkerListingContractTest.mjs - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs + - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityServiceCatalogContractTest.mjs + - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -81,6 +84,9 @@ on: - tests/Unit/ServerParityWorkerListingContractTest.mjs - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs + - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityServiceCatalogContractTest.mjs + - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -305,8 +311,10 @@ jobs: mysql-execution: needs: published-sdk-adapters runs-on: ubuntu-latest - # Corrected marker/range/clock coverage expands the job; scenario budgets stay fixed. - timeout-minutes: 75 + # Corrected marker/range/clock coverage expands the job, and the fixture step + # runs every later profile before failing; scenario budgets stay fixed. + # MySQL runs take 75 to 85 minutes; PostgreSQL's jobs take about 60. + timeout-minutes: 105 strategy: fail-fast: false matrix: @@ -362,7 +370,7 @@ jobs: - name: Create TLS fixture certificates run: | set -euo pipefail - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) mkdir -p mysql-tls parity-evidence chmod 0777 parity-evidence docker run --rm --user="$(id -u):$(id -g)" --network=none \ @@ -426,12 +434,15 @@ jobs: docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --entrypoint mysql \ "$MYSQL_IMAGE" -h parity-mysql -uroot -pparity-disposable-password \ -e 'CREATE DATABASE native_ref CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) + # The physical layout stays the frozen published 2.5.14 one; behavior + # runs on the current reference after migrating that database forward. + layout=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint php \ -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=php_ref \ -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ - "$image" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt + "$layout" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ @@ -457,6 +468,11 @@ jobs: -e DB_HOST=parity-mysql -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ -v "$PWD:/source:ro" -w /source "$image" \ scripts/conformance/server-parity/mysql-schema.php compare php_ref native_ref > parity-evidence/catalog-comparison.json + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint php \ + -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=php_ref \ + -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ + "$image" artisan server:bootstrap --force > parity-evidence/published-upgrade-bootstrap.txt - name: Compare three runtimes, recover killed native nodes and refuse published PHP takeover id: prepare_runtime_comparison run: | @@ -466,7 +482,7 @@ jobs: bash scripts/conformance/server-parity/container-log-tails.sh } trap container_log_tails ERR - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --entrypoint composer \ -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress @@ -554,6 +570,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + native_record_status=0 docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR -e MYSQL_IMAGE "$image" scripts/conformance/server-parity.mjs record \ @@ -564,7 +581,8 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json || native_record_status=$? + compare_status=0 docker run --rm --user=1000:1000 --entrypoint node --network=none \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \ @@ -574,7 +592,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? bash scripts/conformance/server-parity/published-sdk-patch-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 @@ -686,6 +704,10 @@ jobs: catalog php_ref > parity-evidence/catalog-after.json cmp parity-evidence/php-before.json parity-evidence/php-after.json cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json + # A native fixture difference fails the step only after every later + # check has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 - name: Compare separate configured role-token cohort run: bash scripts/conformance/server-parity/role-profile.sh mysql - name: Back up, take over and resume a separate published PHP upgrade source @@ -819,7 +841,7 @@ jobs: run: | set -euo pipefail mkdir -p parity-evidence parity-summary - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --network=none --cpus=1 --memory=256m --memory-swap=256m \ --entrypoint node -e PARITY_JOB_OUTCOME -e GITHUB_SHA \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence:ro" -v "$PWD/parity-summary:/summary" \ @@ -835,8 +857,10 @@ jobs: postgres-storage: needs: published-sdk-adapters runs-on: ubuntu-latest - # Corrected marker/range/clock coverage expands the job; scenario budgets stay fixed. - timeout-minutes: 75 + # Corrected marker/range/clock coverage expands the job, and the fixture step + # runs every later profile before failing; scenario budgets stay fixed. + # MySQL runs take 75 to 85 minutes; PostgreSQL's jobs take about 60. + timeout-minutes: 105 strategy: fail-fast: false matrix: @@ -922,12 +946,15 @@ jobs: "$POSTGRES_IMAGE" --version > parity-evidence/postgres-version.txt docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ -e PGPASSWORD=parity-disposable-password --entrypoint createdb "$POSTGRES_IMAGE" -h parity-pg -U parity native_ref - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) + # The physical layout stays the frozen published 2.5.14 one; behavior + # runs on the current reference after migrating that database forward. + layout=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint php \ -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=php_ref \ -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ - "$image" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt + "$layout" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ @@ -953,6 +980,11 @@ jobs: -e DB_HOST=parity-pg -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ -v "$PWD:/source:ro" -w /source "$image" \ scripts/conformance/server-parity/postgres-schema.php compare php_ref native_ref > parity-evidence/catalog-comparison.json + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint php \ + -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=php_ref \ + -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ + "$image" artisan server:bootstrap --force > parity-evidence/published-upgrade-bootstrap.txt - name: Compare three runtimes, recover killed native nodes and refuse published PHP takeover id: prepare_runtime_comparison env: @@ -964,7 +996,7 @@ jobs: bash scripts/conformance/server-parity/container-log-tails.sh } trap container_log_tails ERR - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --entrypoint composer \ -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress @@ -1051,6 +1083,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + native_record_status=0 docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ @@ -1061,7 +1094,8 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json || native_record_status=$? + compare_status=0 docker run --rm --user=1000:1000 --entrypoint node --network=none \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \ @@ -1071,7 +1105,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? bash scripts/conformance/server-parity/published-sdk-patch-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 @@ -1183,6 +1217,10 @@ jobs: catalog php_ref > parity-evidence/catalog-after.json cmp parity-evidence/php-before.json parity-evidence/php-after.json cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json + # A native fixture difference fails the step only after every later + # check has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 - name: Compare separate configured role-token cohort env: POSTGRES_IMAGE: ${{ matrix.image }} @@ -1360,7 +1398,7 @@ jobs: run: | set -euo pipefail mkdir -p parity-evidence parity-summary - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --network=none --cpus=1 --memory=256m --memory-swap=256m \ --entrypoint node -e PARITY_JOB_OUTCOME -e GITHUB_SHA \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence:ro" -v "$PWD/parity-summary:/summary" \ @@ -1376,8 +1414,10 @@ jobs: execution-and-codec: needs: published-sdk-adapters runs-on: ubuntu-latest - # Corrected marker/range/clock coverage expands the job; scenario budgets stay fixed. - timeout-minutes: 75 + # Corrected marker/range/clock coverage expands the job, and the fixture step + # runs every later profile before failing; scenario budgets stay fixed. + # MySQL runs take 75 to 85 minutes; PostgreSQL's jobs take about 60. + timeout-minutes: 105 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Fetch this run's published SDK adapters @@ -1438,7 +1478,7 @@ jobs: -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \ mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -c \ 'cargo test --locked --offline --all-targets > /evidence/rust-tests.txt && cargo build --locked --offline && cargo run --locked --offline --example codec-fixtures -- ../tests/Fixtures/ServerParity/Codec/v1.json > /evidence/rust-codec.json' - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --network=none --entrypoint php \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity/codec.php tests/Fixtures/ServerParity/Codec/v1.json \ @@ -1461,16 +1501,16 @@ jobs: bash scripts/conformance/server-parity/container-log-tails.sh } trap container_log_tails ERR - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) mkdir -p parity-state parity-evidence chmod 0777 parity-state parity-evidence - touch parity-state/php.sqlite parity-state/embedded.sqlite + touch parity-state/php.sqlite parity-state/embedded.sqlite parity-state/php-layout.sqlite chmod 0666 parity-state/*.sqlite docker run --rm --user="$(id -u):$(id -g)" --entrypoint composer \ -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt @@ -1491,6 +1531,12 @@ jobs: -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$image" artisan server:bootstrap --force >> parity-evidence/bootstrap.txt done + # The physical layout native creates stays the frozen published 2.5.14 one. + layout=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + docker run --rm --user=1000:1000 --entrypoint php \ + -v "$PWD/parity-state:/state" -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/php-layout.sqlite \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ + "$layout" artisan server:bootstrap --force >> parity-evidence/bootstrap.txt # The API server and the CPU-limited maintenance loop share one published PHP # SQLite file. Let SQLite serialize their writers instead of refusing at 5 s. docker run -d --name "$RESOURCE_SCOPE-server" --network "$RESOURCE_SCOPE" --network-alias=parity-server \ @@ -1542,6 +1588,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + native_record_status=0 docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ @@ -1552,7 +1599,8 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json || native_record_status=$? + compare_status=0 docker run --rm --user=1000:1000 --entrypoint node \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \ @@ -1562,7 +1610,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ - --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json + --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? bash scripts/conformance/server-parity/published-sdk-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 @@ -1648,7 +1696,7 @@ jobs: # writable so SQLite can manage its shared-memory cache; use Rust's UID. docker run --rm --user="$(id -u):$(id -g)" --network=none --entrypoint php \ -v "$PWD:/source:ro" -v "$PWD/parity-state:/state" -w /source "$image" \ - scripts/conformance/server-parity/schema.php /state/php.sqlite /state/rust.sqlite > parity-evidence/schema-comparison.json + scripts/conformance/server-parity/schema.php /state/php-layout.sqlite /state/rust.sqlite > parity-evidence/schema-comparison.json docker logs "$RESOURCE_SCOPE-rust" > parity-evidence/rust-killed.log 2>&1 docker run -d --name "$RESOURCE_SCOPE-recovered" --network "$RESOURCE_SCOPE" --network-alias=parity-recovered \ --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ @@ -1673,6 +1721,10 @@ jobs: php scripts/conformance/server-parity/cooperative-restart.php finish http://parity-recovered:8080 /evidence/cooperative-restart-receipt.json > /evidence/cooperative-restart-finish.json php scripts/conformance/server-parity/restart.php finish http://parity-recovered:8080 /evidence/restart-receipt.json > /evidence/restart-finish.json ' + # A native fixture difference fails the step only after every later + # check has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 - name: Compare separate configured role-token cohort env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1796,7 +1848,7 @@ jobs: run: | set -euo pipefail mkdir -p parity-evidence parity-summary - image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) docker run --rm --user="$(id -u):$(id -g)" --network=none --cpus=1 --memory=256m --memory-swap=256m \ --entrypoint node -e PARITY_JOB_OUTCOME -e GITHUB_SHA \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence:ro" -v "$PWD/parity-summary:/summary" \ diff --git a/docs/rust-server-port-checkpoints/2026-10-22-definition-routing.md b/docs/rust-server-port-checkpoints/2026-10-22-definition-routing.md new file mode 100644 index 000000000..4297e7380 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-definition-routing.md @@ -0,0 +1,65 @@ +# October 22, workflow definition routing + +- **Gap.** Native records the definition fingerprint a start's worker cohort + agrees on, with `workflow_definition_fingerprint_source: worker`, as PHP + does. It then offered that run's workflow tasks to any worker, so a worker + with another definition could take a pinned run while the recorded one was + live. +- **Source.** Server 2.10.0's `WorkflowDefinitionRouting::forWorker`, + which `WorkflowTaskPoller::matchesRecordedWorkflowDefinition` applies at + claim, and `WorkflowReplayEligibility`. + - **Decision.** For a workflow task of a worker-pinned run: + - a worker advertising the recorded fingerprint for the type takes it + (`matching_definition`); + - without a recorded fingerprint (`recorded_definition_missing`), or + without one from the worker (`worker_definition_missing`), nobody + does; + - a worker with another definition waits while an eligible peer + advertises the recorded one (`matching_definition_preferred`); + - otherwise it takes the task only if it advertises + `workflow_entry_arguments_validation` (`replay_required`, else + `entry_validation_unavailable`). + + An unpinned run goes to any worker (`not_worker_pinned`). Native + refuses to start with `DW_V2_PIN_TO_RECORDED_FINGERPRINT` off, so + pinning is always on. + - **Eligible peers** (`eligibleWorkers`). Active registrations on the + run's namespace and queue that: + - match the build the task or run requires; + - support the type, with workflow capacity above zero; + - replay the run (`WorkflowReplayEligibility::allows`). That means a + heartbeat inside the stale window and cooperative-cancellation support + when the run has a request, at the protocol version the worker last + reported. It also means the memo and typed search attribute + capabilities its history needs. + - **Protocol version.** Each registration and heartbeat records its + protocol version under `_worker_protocol_version` in the worker's process + metrics, after the metrics it reported. A heartbeat without metrics + keeps the stored ones. Worker listings therefore show the key, as PHP + 2.10.0's do; 2.5.14 had no such key. +- **Reports** (`inspect` and `forQueue`). + - Describe, repair and debug report `definition_routing` for the run's + next due ready task: + - the context (`pinning_enabled`, `worker_pinned`, + `recorded_fingerprint`, type, queue and required build); + - PHP's status, the counts of eligible workers by definition, up to ten + example workers, `blocked` and `runnable_task`; + - PHP's resolution text. + - A blocked route adds debug's `workflow_definition_routing_blocked` + warning and `diagnostic_status: needs_attention`. + - Task queue describe samples up to ten runs with a due ready task. A + blocked sample sets `repair.needs_attention`. +- **Tests** (`definition_routing.rs`). + - A pinned run goes to the worker with its recorded definition. Workers + with another definition, or without one, wait even when they validate + entry arguments. + - Once the recorded definition leaves, only an entry-validating worker + takes the run. + - An unpinned run goes to any worker. + - Registrations and heartbeats record the protocol version with the + reported metrics. + - Describe, debug, task queue and repair report a blocked route, then the + route an entry-validating replacement unblocks. + - The worker listing test now expects the protocol key. + - Full SQLite suite: 398 passed. Routing, worker, poll, child, cooperative, + debug, task queue, repair and describe tests: PostgreSQL 134, MySQL 134. diff --git a/docs/rust-server-port-checkpoints/2026-10-22-poll-waits.md b/docs/rust-server-port-checkpoints/2026-10-22-poll-waits.md new file mode 100644 index 000000000..bb1c9ad37 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-poll-waits.md @@ -0,0 +1,42 @@ +# October 22, worker poll waits and operation names + +- **Gap.** A workflow or activity poll without `timeout_seconds` waited 5 + seconds; PHP waits `DW_WORKER_POLL_TIMEOUT` seconds, 30 by default. A + query-task poll without one also waited 5 seconds where PHP starts from + the same setting. Native refused an explicit `null` wait, which PHP's + `nullable` rule accepts. +- **Source.** Server 2.10.0's `LongPoller::until`, + `WorkflowQueryTaskBroker::queryPollTimeoutSeconds` and + `WorkflowController::validateOperationName`. +- **Poll waits.** + - A workflow or activity poll without a wait, or with `null`, waits + `max(0, DW_WORKER_POLL_TIMEOUT)` seconds. The setting is not capped. + - A query-task poll waits the requested time or that setting, capped by + `DW_QUERY_TASK_POLL_TIMEOUT` (default 5); zero answers at once. + - A requested wait passes Laravel's `integer` rule (so `"5"` is five + seconds) and must lie from 0 to 60. +- **Operation names.** + - Signal, update and query requests check the name first, after the + payload transport check, as PHP's controller does. Empty names, names + over 256 bytes and names with ASCII control characters get PHP's + `validation_failed` errors and messages. + - PHP's signal and update name columns hold 255 bytes. Native keeps its + storage guard and refuses a 256-byte or all-space name with + `invalid_signal_name` or `invalid_update_name`. Query names are not + stored in those columns, so a 256-byte query name is accepted. +- **Fixed along the way.** A query poll from a worker registered without + `capabilities` answered 503 `storage_unavailable`, because the claim read + the missing list as required. It now polls empty, as PHP's `stringArray` + does. +- **Tests** (`poll_waits.rs`). + - Workflow, activity and query polls without a wait, or with `null`, take + the configured second. + - A zero setting and a requested `"0"` answer at once. + - Requested waits of 61, -1, `"soon"` and 1.5 are refused. + - A task started during a default wait is claimed. + - A 255-byte signal name is accepted; 256-byte signal and update names + are refused with the storage reasons. Too-long names and names with + control characters get PHP's messages for each kind. + - Full SQLite run (all targets): 228 unit, 397 execution and 2 upgrade + tests passed. Poll, signal, update and query tests: PostgreSQL 122, + MySQL 122. diff --git a/docs/rust-server-port-checkpoints/2026-10-22-worker-admission.md b/docs/rust-server-port-checkpoints/2026-10-22-worker-admission.md new file mode 100644 index 000000000..c30d1653a --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-worker-admission.md @@ -0,0 +1,60 @@ +# October 22, worker registration admission + +- **Gap.** Native accepted a registration without the `capability_manifest` + PHP requires from protocol 1.18. It also skipped most of PHP's request + rules and capability guards, refused a missing `worker_id` or type list, + and answered metadata-capability mismatches in its own shape. +- **Source.** Server 2.10.0 `WorkerController::register`: + - its Laravel rules; + - the prepared-local-activity, Nexus and cooperative-cancellation guards; + - `guardPortableWorkerCapabilityManifest`; + - `WorkflowMetadataCapabilityPolicy::firstProtocolMismatch`; + - the message-streams floor; + - `portableWorkerCapabilityManifest`. +- **Request rules.** Laravel trims strings and turns empty ones into null + first; native now does the same for the whole body. + - Every field follows PHP's rules: `runtime` from PHP's list, sizes, + capacities, task slots, process metrics, heartbeat interval. + - `capability_manifest` is required from protocol 1.18 and must hold the + `local_activities`, `worker_sessions` and `sticky_execution` entries. + Each entry needs `supported` (boolean) and `minimum_protocol_version`. + - Failures answer Laravel's 422 `validation_failed` with its messages. The + rule engine gained `numeric` and `required_array_keys`. A `*.field` rule + now covers only array items, as Laravel's expansion does. +- **Guards.** Each answers 409 with `registered: false` and PHP's fields, in + PHP's order: + - `prepared_local_activity_cancellation_policy_capability_mismatch`, + `prepared_local_activity_group_capability_mismatch` and + `prepared_local_activity_capability_mismatch`; + - `nexus_cooperative_cancellation_capability_mismatch` and + `cooperative_cancellation_protocol_mismatch`, below protocol 1.20; + - `worker_capability_manifest_mismatch`, when a flat affinity capability + and its `supported` flag disagree; + - `worker_capability_protocol_floor_mismatch`, when an entry's floor is + below 1.18 or above the request's protocol; + - `workflow_metadata_capability_protocol_mismatch`, for `memo_upserts` + below 1.14 and `typed_search_attributes` below 1.16. This now uses PHP's + `capability`, `requested_version` and `remediation` fields, not native's + former `worker_capability` and `recorded`; + - `message_streams_unavailable`, below 1.15. +- **Accepted registrations.** + - A missing `worker_id` gets a ULID, as PHP does. + - Missing type lists are stored empty. + - Capabilities keep their non-empty strings. + - The manifest is stored normalized: the three entries, a trimmed floor, + and non-empty `implementation` and `reason`. +- **Worker protocol bodies.** Registration refusals now carry + `protocol_version` and `server_capabilities`, as `WorkerProtocol::json` + does. Other worker-plane refusals still lack them (#46's shape sweep). +- **Tests.** + - `worker_admission.rs` uses the server's own router. It covers the + manifest requirement and its errors, field rules with Laravel's + messages, the generated id, the stored manifest, and every guard. + - The other tests' router adds the manifest an SDK sends from 1.18, + matching each registration's flat capabilities. + - Two cooperative tests registered `cooperative_cancellation` at protocol + 1.19, which PHP refuses. They now register at 1.20 and still poll at the + protocol each case tests. + - Full SQLite run (all targets): 230 unit, 401 execution and 2 upgrade + tests passed. Worker, registration, admission, cooperative and + namespace tests: PostgreSQL 124, MySQL 124. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index bd7b02079..91089e775 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -14,71 +14,121 @@ qualify a later tree. ## Current state -Updated October 20, 2026. - -- **Merged on October 20.** - - [#507](https://github.com/durable-workflow/server/pull/507) gives codec work under the transition lock its own permit. - - [#508](https://github.com/durable-workflow/server/pull/508) keeps explicit worker command declarations. - - [#509](https://github.com/durable-workflow/server/pull/509) reports poll statuses as the worker protocol does. - - [#515](https://github.com/durable-workflow/server/pull/515) retries the PHP reference's lock-pressure refusals in the probes. - - [#516](https://github.com/durable-workflow/server/pull/516) lets held long polls wait without the transition lock. - - [#517](https://github.com/durable-workflow/server/pull/517) designs the backup-first takeover. - - [#499](https://github.com/durable-workflow/server/pull/499) and [#510](https://github.com/durable-workflow/server/pull/510) replay a live query lease only to its own poll request and remember query poll answers per request. - - [#542](https://github.com/durable-workflow/server/pull/542) records repeated matched PHP/native performance pairs. - - This branch: [#543](https://github.com/durable-workflow/server/pull/543) (immediate termination) with [#544](https://github.com/durable-workflow/server/pull/544) (worker listing and registration facts). +Updated October 22, 2026. + +- **Merged since October 20.** + - **Backup-first takeover** on SQLite, PostgreSQL and MySQL/MariaDB: + [#523](https://github.com/durable-workflow/server/pull/523), + [#529](https://github.com/durable-workflow/server/pull/529), + [#532](https://github.com/durable-workflow/server/pull/532), + [#533](https://github.com/durable-workflow/server/pull/533), + [#536](https://github.com/durable-workflow/server/pull/536), + [#538](https://github.com/durable-workflow/server/pull/538) and + [#539](https://github.com/durable-workflow/server/pull/539). This covers the + fingerprint, the layouts and WAL preparation. It also covers a PHP-accepted + signal and an expired PHP lease resumed under native, and the refusal of + unsupported features. + - **Multi-node and API parity.** + - [#549](https://github.com/durable-workflow/server/pull/549): the + multi-node kill and restart qualification. + - [#548](https://github.com/durable-workflow/server/pull/548): ownership + guard refusals. + - [#550](https://github.com/durable-workflow/server/pull/550) and + [#551](https://github.com/durable-workflow/server/pull/551): the search + attribute registry and namespace update. + - [#594](https://github.com/durable-workflow/server/pull/594): history + export. + - [#595](https://github.com/durable-workflow/server/pull/595) and + [#596](https://github.com/durable-workflow/server/pull/596): retention and + activity timeouts. + - **The October 21-22 trains.** + [#629](https://github.com/durable-workflow/server/pull/629), + [#638](https://github.com/durable-workflow/server/pull/638), + [#641](https://github.com/durable-workflow/server/pull/641), + [#649](https://github.com/durable-workflow/server/pull/649), + [#659](https://github.com/durable-workflow/server/pull/659) and + [#665](https://github.com/durable-workflow/server/pull/665) covered: + - input envelopes, run repair, legacy settings, activity deadlines, + runtime credentials and principals; + - query and update admission, signals rejected after a run closes, start + fingerprints and all-of barriers; + - cooperative waits, child cancellation and parent-close policies, + continue-as-new messages, history pages, declared signals and build IDs. + - **GHSA-6q48-6c7w-9gw2 trust remediation.** + [#671](https://github.com/durable-workflow/server/pull/671) moves the Rust + locks to the fixed webpki roots. + [#672](https://github.com/durable-workflow/server/pull/672) pins the image's + TLS toolchains and checks every trust store. - **Open.** - - The backup-first takeover stack: [#523](https://github.com/durable-workflow/server/pull/523) - (fingerprint), [#529](https://github.com/durable-workflow/server/pull/529) - (layouts) and [#532](https://github.com/durable-workflow/server/pull/532), - [#533](https://github.com/durable-workflow/server/pull/533) and - [#536](https://github.com/durable-workflow/server/pull/536) (SQLite, - PostgreSQL and MySQL/MariaDB takeover). Then - [#538](https://github.com/durable-workflow/server/pull/538) (SQLite WAL - preparation) and [#539](https://github.com/durable-workflow/server/pull/539) - (a PHP-accepted signal and an expired PHP lease resumed under native, plus - refusal of unsupported features). Its top passed all 13 parity jobs. - - [#548](https://github.com/durable-workflow/server/pull/548): PHP's - ownership guard refusals for a lost workflow task claim. - - [#549](https://github.com/durable-workflow/server/pull/549): the - multi-node kill and restart qualification and its results. - - [#550](https://github.com/durable-workflow/server/pull/550): the search - attribute registry. [#551](https://github.com/durable-workflow/server/pull/551): - namespace update. -- **Parity surface.** With this branch, native serves 63 of published PHP's 173 API - routes. The missing families are listed below under "API surface still - missing". -- **Next action.** Merge the open PRs as their checks pass. Then port history - export, a large bundle the CLI and both SDKs use for replay. Packaging the - native binary into the server image remains an open gate. + - [#675](https://github.com/durable-workflow/server/pull/675): train of + [#667](https://github.com/durable-workflow/server/pull/667) (definition + routing), [#669](https://github.com/durable-workflow/server/pull/669) (poll + waits), [#670](https://github.com/durable-workflow/server/pull/670) (worker + admission) and [#674](https://github.com/durable-workflow/server/pull/674) + (schedule fixture probe). + - [#676](https://github.com/durable-workflow/server/pull/676): accept + published PHP's health read during or after its own task repair. + - [#673](https://github.com/durable-workflow/server/pull/673): pip, Composer + and Cargo trust in CI and conformance, in Security review. + [#677](https://github.com/durable-workflow/server/issues/677) owns the + remaining clients, with an explicit non-completion boundary. +- **Shared Server fixtures on main.** Embedded passes. Native differs on two + fixtures: + - **`signals-before-worker`:** native records `SignalRejected` after a + terminal event. + - **`worker-listing`:** native reports `_worker_protocol_version` in + `process_metrics`. + + In both, native follows published Server 2.10.0 while the fixtures still + expect the frozen 2.5.14 reference. Separately, the published PHP reference + sometimes differs from itself on `operator-metrics` (postgres-17) because of + its own repair window, which #676 handles. Moving the reference needs a + recorded contract decision, not an edited expectation. +- **Parity surface.** Native serves 97 of published PHP's 175 API routes. That + count compares `php artisan route:list --path=api` on this tree with the + native router. The missing routes are listed below. +- **Next action.** + 1. Verify the Rust suites on #675 and #676, then merge them. + 2. Record the reference decision for Server 2.10.0 and re-record the affected + fixtures from that published release, so main's Shared Server fixtures run + green. + 3. Port the missing families below. + + Packaging the native binary into the server image remains an open gate. ## API surface still missing -These published PHP 2.9.8 routes have no native handler yet, by family. The -CLI and Waterline use the first group. +These published PHP routes on this tree have no native handler yet, by family. - **Used by the CLI or Waterline.** - - Task queue build IDs. - - Namespace update, delete and external storage. - - `system` repair pass, metrics, and the operator dashboard filtered by - workflow type. - - Activities (standalone). + - Namespace delete. + - Standalone activities: list, describe and start. + - `system` metrics (JSON and Prometheus), the repair pass, and external + payload cleanup and its pass. + - Task queue priority and fairness. - **Worker protocol features.** - Worker sessions. - Update validation tasks. - - Prepared local activities. + - Local activities (prepare, checkpoint, recover, heartbeat, control, + outcome and cancellation acknowledgement). - Cancellation scopes. - - Activity cancellation acknowledgement. + - Activity cancellation acknowledgement. (Lease renewal, from Server 2.11, + is served: `POST /api/worker/activity-tasks/{task}/renew-lease`, with the + `activity_lease_renewal` capability and the `activity-lease-renewal` fixture.) - **Run operations.** - - Archive. + - Archive (instance and run). - Redrive. - Imports from embedded v2 and Waterline v1. - **Other families.** - Message streams. - - Durable workflow stream appends and closes. - - Service catalogs and endpoints. + - Durable workflow stream items and close. + - Service endpoints. (The service catalog reads from Server 2.11 are served: + native never holds catalog rows, so every list is PHP's empty page and every + identity its not-found answer; the `service-catalog` fixture checks 37 cases.) - Nexus operations. - Deployments. - Bridge adapters. + ## Known gaps found in review These are confirmed differences from the published PHP reference and open @@ -122,8 +172,14 @@ contract fixture before its correction. non-default value of the other 67 ([checkpoint](rust-server-port-checkpoints/2026-10-22-environment-contract.md)). Several of those are cheap to honor: repair timing and the multi-node - declaration. Native's defaults still differ from PHP's for two honored - settings: the worker poll timeout and operation name length. + declaration. +- **Signal and update names PHP cannot store.** Native validates signal, + update and query names as PHP does (1 to 256 bytes, no control + characters). PHP's signal and update name columns hold 255 bytes, so PHP + stores a 256-byte name on SQLite and fails with a database error on MySQL + and PostgreSQL. Native refuses such a name, and a name of only spaces, + with `invalid_signal_name` or `invalid_update_name` on every database + ([checkpoint](rust-server-port-checkpoints/2026-10-22-poll-waits.md)). - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. @@ -171,9 +227,6 @@ contract fixture before its correction. deep. Native checks every body as PHP does before any route, but its handlers parse with serde_json's 128-level limit. A body nested 129 to 512 levels deep passes the check and is then refused with a plain-text 400. -- **Registration without a capability manifest.** At protocol 1.19, - published PHP 2.5.14 refuses a registration that has no - `capability_manifest` (422). Native accepts it. - **Python worker shutdown after a late synchronous query handler (not yet diagnosed).** In the SQLite query-authority cohort on #515's run 39431491496, the published Python SDK 2.5.15 client in `late-handler-sync` @@ -182,15 +235,6 @@ contract fixture before its correction. then did not stop within the 5-second shutdown budget. Forty local repetitions of that case alone passed. Find what holds the query thread after the fence before changing either side. -- **Workflow definition routing (Server 2.10.0).** Native records the - definition fingerprint a start's worker cohort agrees on, as PHP does. - Published PHP 2.10.0 also routes a task for such a run by that - fingerprint. It prefers a live worker advertising the recorded definition. - Otherwise it offers the task only to a worker that advertises - `workflow_entry_arguments_validation`. It reports `definition_routing` in - describe, repair, debug and task queues, and stores each worker's protocol - version under `_worker_protocol_version` in its process metrics. Native - offers such a task to any worker and reports none of this. - **Start build-id pin.** Published PHP pins a new run to its start cohort's build id: a promoted build-id rollout, else the single build id among the live workers for the type, else among all the queue's live workers. Native @@ -344,6 +388,33 @@ support is not evidence of multi-node safety. ## Decisions +- **October 22: the parity reference moves from published Server 2.5.14 to 2.11.1.** + Published PHP has moved on, and the Sample App and current consumers resolve + 2.10 or later. + - A local run measured the change. It ran the shared fixtures against the + 2.11.1 image (`sha256:bf8bed13…`) with the consumer tuple unchanged. + - Three reviewed expectations followed 2.5.14 and differ under 2.11.1: + - **`signals-before-worker`:** signals a run closed before applying are + recorded as `SignalRejected` after its terminal event. + - **`worker-listing`:** every registration's stored process metrics carry + its `_worker_protocol_version`, including one that reported no metrics. + - **`schedule-policies`:** under SQLite lock pressure PHP refuses with the + structured, retryable `503 backend_lock_pressure` rather than "database + is locked". + - Native already behaved as 2.11.1 does in the first two. The first two + expectations move to 2.11.1. The probe now retries the third refusal under + its existing rule: a write is sent again only when its schedule reads + unchanged. + - The frozen 2.5.14 tuple remains in `tests/Fixtures/ServerParityBaselines` + for the takeover cohorts that pin it. + - It also remains the physical layout native creates, following the + conversion-recipe decision below. The SQLite, MySQL and PostgreSQL catalog + comparisons bootstrap with the 2.5.14 image. The MySQL and PostgreSQL + reference databases are then migrated forward to 2.11.1 before the + behavior comparison, the same upgrade PHP users run. + - The consumer tuple (PHP SDK 2.2.6, Python 2.5.0, Rust 3.4.1, CLI 2.2.0, + Waterline 2.3.1) is unchanged. The embedded target now runs Workflow 2.10.2 + from the 2.11.1 image. - Shared fixtures are expectations reviewed against public contracts, not PHP recordings promoted to expected behavior. The recorder keeps raw observations; the comparator checks the declared fixture projection. Coverage is explicit. diff --git a/rust/src/main.rs b/rust/src/main.rs index c3b3126cf..340bff972 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -206,6 +206,10 @@ fn runtime_credentials_enabled() -> Result> { /// forwarded attribution and disabled backward compatibility. Only the token /// and none drivers are qualified. fn validate_auth_configuration() -> Result<(), Box> { + // The contract audit compares this flag lossily; refuse a value that is + // not text before it could be read as PHP's default. + auth_value("DW_AUTH_BACKWARD_COMPATIBLE")?; + auth_value("WORKFLOW_SERVER_AUTH_BACKWARD_COMPATIBLE")?; if let Some(driver) = auth_setting("DW_AUTH_DRIVER", "WORKFLOW_SERVER_AUTH_DRIVER")? { if driver != "token" && driver != "none" { return Err("unsupported development authentication configuration".into()); @@ -496,6 +500,9 @@ async fn main() -> Result<(), Box> { return Err(refusal.into()); } validate_auth_configuration()?; + // Every authentication setting is read before storage is selected, so an + // unreadable value is refused as authentication configuration. + let (anonymous, runtime_credentials) = (anonymous_driver()?, runtime_credentials_enabled()?); // Published PHP reads each token under its `DW_*` name, or else its // legacy `WORKFLOW_SERVER_*` name, so an unchanged deployment starts. let worker_token = auth_setting("DW_WORKER_TOKEN", "WORKFLOW_SERVER_WORKER_TOKEN")?; @@ -549,7 +556,7 @@ async fn main() -> Result<(), Box> { }; let runtime = runtime .with_role_tokens(worker_token, operator_token, admin_token) - .with_auth_options(anonymous_driver()?, runtime_credentials_enabled()?); + .with_auth_options(anonymous, runtime_credentials); let listener = tokio::net::TcpListener::bind(address).await?; eprintln!( "Rust development runtime listening on {}", diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 9ff800835..1dca08ace 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -464,7 +464,7 @@ where "invalid_activity_retry_policy", ) })?; - Self::query("UPDATE activity_attempts SET status='failed',closed_at=$1 WHERE id=$2") + Self::query("UPDATE activity_attempts SET status='failed',lease_expires_at=NULL,closed_at=$1 WHERE id=$2") .bind(DB::bind_time(closed_at)) .bind(attempt_id) .execute(&mut *tx) diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs new file mode 100644 index 000000000..83890c97d --- /dev/null +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -0,0 +1,283 @@ +//! Runtime activity lease renewal, as published Server 2.11 serves it +//! (`ActivityTaskController::renewLease` over Workflow 2.10's +//! `ActivityLeaseRenewal`). Renewal keeps a live attempt's lease for its +//! original owner. It never records progress, moves a business deadline or +//! adds history, and an expired lease cannot be revived. +use super::laravel_rules::{self, Rule}; +use super::transaction::Transaction; +use super::{ + Result, RuntimeError, + store::{capabilities, now}, +}; +use axum::http::StatusCode; +use chrono::{DateTime, SecondsFormat, Utc}; +use serde_json::{Value, json}; + +/// Published PHP's validation rules for the request, in their order. +const RULES: &laravel_rules::Rules = &[ + ("activity_attempt_id", &[Rule::Required, Rule::String]), + ("lease_owner", &[Rule::Required, Rule::String]), +]; + +/// A worker-protocol response with PHP's protocol fields. +fn worker(status: StatusCode, mut response: Value) -> RuntimeError { + response["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + response["server_capabilities"] = capabilities(); + RuntimeError::Protocol { status, response } +} + +/// PHP's `outcomeStatus`. +fn outcome_status(reason: Option<&str>) -> StatusCode { + match reason { + None => StatusCode::OK, + Some("attempt_not_found") => StatusCode::NOT_FOUND, + Some(_) => StatusCode::CONFLICT, + } +} + +fn micros(at: DateTime) -> String { + at.to_rfc3339_opts(SecondsFormat::Micros, true) +} + +/// The rows an attempt's authority is read from. +struct Rows { + attempt: R, + activity: Option, + run: Option, + task: Option, +} + +store_methods! { + pub(crate) async fn renew_activity_lease(&self, task_id: &str, body: Value) -> Result { + let body = laravel_rules::normalize(&body); + if let Err(RuntimeError::Protocol { status, response }) = laravel_rules::validate(&body, RULES) { + return Err(worker(status, response)); + } + let attempt_id = body["activity_attempt_id"].as_str().unwrap_or_default(); + let owner = body["lease_owner"].as_str().unwrap_or_default(); + let refusal = |status, error: &str, reason: &str| { + worker(status, json!({"task_id":task_id,"activity_attempt_id":attempt_id,"error":error,"reason":reason})) + }; + let mut tx = self.begin().await?; + // PHP's ownership guard, in its order: the task in this namespace, + // then the attempt by its identity, its task and its owner. + let task = Self::query("SELECT id FROM workflow_tasks WHERE id=$1 AND namespace=$2") + .bind(task_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| refusal(StatusCode::NOT_FOUND, "Activity task not found.", "task_not_found"))?; + let Some(rows) = Self::attempt_rows(&mut tx, attempt_id).await? else { + return Err(refusal(StatusCode::NOT_FOUND, "Activity attempt not found.", "attempt_not_found")); + }; + let observed = Self::attempt_status(attempt_id, &rows, None)?; + if observed["workflow_task_id"].as_str() != Some(DB::string(&task, "id")?.as_str()) { + return Err(refusal(StatusCode::CONFLICT, "Activity attempt is not leased for this task.", "task_mismatch")); + } + if observed["lease_owner"].as_str() != Some(owner) { + let mut error = json!({"task_id":task_id,"activity_attempt_id":attempt_id, + "error":"Activity attempt lease is owned by another worker.","reason":"lease_owner_mismatch"}); + error["lease_owner"] = observed["lease_owner"].clone(); + return Err(worker(StatusCode::CONFLICT, error)); + } + if observed["can_continue"] != true { + let mut response = observed; + response["lease_renewed"] = json!(false); + return Err(worker(outcome_status(response["reason"].as_str()), response)); + } + // The renewal itself requires the owner of both the attempt and its + // task, as PHP's observation with an owner does. + let leased_by_owner = DB::optional_string(&rows.attempt, "lease_owner")?.as_deref() == Some(owner) + && match &rows.task { + Some(task) => DB::optional_string(task, "lease_owner")?.as_deref() == Some(owner), + None => false, + }; + let at = now(); + let mut status = if leased_by_owner { + let task = rows.task.as_ref().expect("task checked above"); + let mut expires = at + chrono::Duration::seconds(self.activity_task_lease_seconds); + if let Some(current) = DB::optional_instant(task, "lease_expires_at")?.filter(|current| *current > expires) { + expires = current; + } + Self::query("UPDATE activity_attempts SET lease_expires_at=$1 WHERE id=$2") + .bind(DB::bind_time(expires)) + .bind(attempt_id) + .execute(&mut *tx) + .await?; + Self::query("UPDATE workflow_tasks SET lease_expires_at=$1 WHERE id=$2") + .bind(DB::bind_time(expires)) + .bind(DB::string(task, "id")?) + .execute(&mut *tx) + .await?; + let renewed = Self::attempt_rows(&mut tx, attempt_id).await?.expect("attempt read above"); + Self::attempt_status(attempt_id, &renewed, None)? + } else { + Self::attempt_status(attempt_id, &rows, Some("lease_owner_mismatch"))? + }; + tx.commit().await?; + status["lease_renewed"] = status["can_continue"].clone(); + status["server_time"] = json!(micros(at)); + status["task_id"] = json!(task_id); + status["worker_session"] = Value::Null; + let code = outcome_status(status["reason"].as_str()); + if code != StatusCode::OK { + return Err(worker(code, status)); + } + status["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + status["server_capabilities"] = capabilities(); + Ok(status) + } + + async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { + let Some(attempt) = Self::query("SELECT * FROM activity_attempts WHERE id=$1") + .bind(attempt_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let activity = Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(DB::string(&attempt, "activity_execution_id")?) + .fetch_optional(&mut **tx) + .await?; + let run = match &activity { + Some(activity) => Self::query("SELECT * FROM workflow_runs WHERE id=$1") + .bind(DB::string(activity, "workflow_run_id")?) + .fetch_optional(&mut **tx) + .await?, + None => None, + }; + let task = match DB::optional_string(&attempt, "workflow_task_id")?.filter(|id| !id.is_empty()) { + Some(id) => Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + .bind(id) + .fetch_optional(&mut **tx) + .await?, + None => None, + }; + Ok(Some(Rows { attempt, activity, run, task })) + } + + /// Workflow's `attemptStatus` over its continuation state. `refused` + /// replaces the state with a refusal that cannot continue. + fn attempt_status(attempt_id: &str, rows: &Rows, refused: Option<&'static str>) -> Result { + let (can_continue, cancel_requested, reason) = match refused { + Some(reason) => (false, false, Some(reason)), + None => Self::continuation(rows, now())?, + }; + let text = |row: Option<&DB::Row>, column: &str| -> Result> { + Ok(match row { + Some(row) => DB::optional_string(row, column)?.filter(|value| !value.is_empty()), + None => None, + }) + }; + let instant = |row: Option<&DB::Row>, column: &str| -> Result>> { + Ok(match row { + Some(row) => DB::optional_instant(row, column)?, + None => None, + }) + }; + let attempt = Some(&rows.attempt); + let (activity, run, task) = (rows.activity.as_ref(), rows.run.as_ref(), rows.task.as_ref()); + let first = |values: [Option; 2]| values.into_iter().flatten().next(); + let time = |values: [Option>; 2]| values.into_iter().flatten().next().map(micros); + Ok(json!({ + "can_continue": can_continue, + "cancel_requested": cancel_requested, + "reason": reason, + "heartbeat_recorded": false, + "workflow_instance_id": text(run, "workflow_instance_id")?, + "workflow_run_id": first([text(run, "id")?, first([text(activity, "workflow_run_id")?, text(attempt, "workflow_run_id")?])]), + "workflow_task_id": first([text(task, "id")?, text(attempt, "workflow_task_id")?]), + "activity_execution_id": first([text(activity, "id")?, text(attempt, "activity_execution_id")?]), + "activity_attempt_id": attempt_id, + "attempt_number": DB::number(&rows.attempt, "attempt_number").ok(), + "run_status": text(run, "status")?, + "run_closed_reason": text(run, "closed_reason")?, + "run_closed_at": time([instant(run, "closed_at")?, None]), + "activity_status": text(activity, "status")?, + "attempt_status": text(attempt, "status")?, + "task_status": text(task, "status")?, + "lease_owner": first([text(attempt, "lease_owner")?, text(task, "lease_owner")?]), + "lease_expires_at": time([instant(attempt, "lease_expires_at")?, instant(task, "lease_expires_at")?]), + "last_heartbeat_at": time([instant(attempt, "last_heartbeat_at")?, instant(activity, "last_heartbeat_at")?]), + "heartbeat_deadline_at": time([instant(activity, "heartbeat_deadline_at")?, None]), + })) + } + + /// Workflow's `attemptContinuationState`: whether the attempt may keep + /// running, whether it was asked to stop, and why not. + fn continuation(rows: &Rows, at: DateTime) -> Result<(bool, bool, Option<&'static str>)> { + let attempt_status = DB::string(&rows.attempt, "status")?; + let run_status = match &rows.run { + Some(run) => Some(DB::string(run, "status")?), + None => None, + }; + if attempt_status != "running" { + return Ok(match run_status.as_deref() { + Some("cancelled") => (false, true, Some("run_cancelled")), + Some("terminated") => (false, true, Some("run_terminated")), + _ => (false, attempt_status == "cancelled", Some("attempt_closed")), + }); + } + let Some(activity) = &rows.activity else { + return Ok((false, false, Some("activity_execution_missing"))); + }; + let (Some(run), Some(run_status)) = (&rows.run, run_status) else { + return Ok((false, false, Some("workflow_run_missing"))); + }; + let activity_status = DB::string(activity, "status")?; + let task_status = match &rows.task { + Some(task) => Some(DB::string(task, "status")?), + None => None, + }; + match run_status.as_str() { + "cancelled" => return Ok((false, true, Some("run_cancelled"))), + "terminated" => return Ok((false, true, Some("run_terminated"))), + _ => {} + } + if activity_status == "cancelled" { + return Ok((false, true, Some("activity_cancelled"))); + } + if task_status.as_deref() == Some("cancelled") { + return Ok((false, true, Some("task_cancelled"))); + } + if !matches!(run_status.as_str(), "pending" | "running" | "waiting") { + return Ok((false, false, Some("run_closed"))); + } + if activity_status != "running" { + return Ok((false, false, Some("activity_not_running"))); + } + if DB::optional_string(activity, "current_attempt_id")?.as_deref() != Some(DB::string(&rows.attempt, "id")?.as_str()) + || DB::number(activity, "attempt_count")? != DB::number(&rows.attempt, "attempt_number")? + { + return Ok((false, false, Some("stale_attempt"))); + } + let Some(task) = &rows.task else { + return Ok((false, false, Some("task_not_found"))); + }; + let payload = DB::optional_document_row(task, "payload")?.unwrap_or(Value::Null); + if DB::string(task, "workflow_run_id")? != DB::string(activity, "workflow_run_id")? + || payload["activity_execution_id"].as_str() != Some(DB::string(activity, "id")?.as_str()) + || DB::number(task, "attempt_count")? != DB::number(&rows.attempt, "attempt_number")? + { + return Ok((false, false, Some("stale_task"))); + } + if task_status.as_deref() != Some("leased") { + return Ok((false, false, Some("task_not_leased"))); + } + // `ActivityAttemptAuthority::expiredReason`: the task's lease, then + // the activity's business deadlines, then the run's. + if DB::optional_instant(task, "lease_expires_at")?.is_none_or(|expires| at >= expires) { + return Ok((false, false, Some("lease_expired"))); + } + if let Some(kind) = super::activity_deadlines::expired_kind::(activity, at)? { + return Ok((false, false, Some(super::activity_deadlines::timeout_reason(kind)))); + } + for (reason, column) in [("run_timeout", "run_deadline_at"), ("execution_timeout", "execution_deadline_at")] { + if DB::optional_instant(run, column)?.is_some_and(|deadline| at >= deadline) { + return Ok((false, false, Some(reason))); + } + } + Ok((true, false, None)) + } +} diff --git a/rust/src/runtime/definition_routing.rs b/rust/src/runtime/definition_routing.rs new file mode 100644 index 000000000..2406206e6 --- /dev/null +++ b/rust/src/runtime/definition_routing.rs @@ -0,0 +1,309 @@ +//! Server 2.10.0's `WorkflowDefinitionRouting` and +//! `WorkflowReplayEligibility`. A run whose start recorded the definition +//! fingerprint its worker cohort agreed on (`workflow_definition_fingerprint_source` +//! `worker`) is pinned to that definition: its workflow tasks go to a worker +//! advertising the recorded fingerprint. A worker with another definition gets +//! them only when no live peer advertises the recorded one, and only if it +//! validates entry arguments (`workflow_entry_arguments_validation`), so it can +//! safely refuse incompatible input on replay. +//! +//! Native refuses to start with `DW_V2_PIN_TO_RECORDED_FINGERPRINT` turned off, +//! so pinning is always enabled here. +use super::transaction::Transaction; +use super::{ + Result, + backend::Backend, + store::{Store, now}, +}; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +/// The worker capability that lets a changed definition replay a pinned run. +pub(super) const ENTRY_VALIDATION_CAPABILITY: &str = "workflow_entry_arguments_validation"; + +/// PHP's `stringValue`: a trimmed, non-empty string. +fn trimmed(value: Option<&str>) -> Option { + value.map(str::trim).filter(|value| !value.is_empty()).map(str::to_owned) +} + +fn has_capability(capabilities: &Value, capability: &str) -> bool { + capabilities.as_array().is_some_and(|items| items.iter().any(|item| item == capability)) +} + +/// `context`: the run's recorded definition and where a decision applies. +pub(super) struct DefinitionContext { + pub(super) worker_pinned: bool, + pub(super) recorded: Option, + pub(super) namespace: String, + pub(super) workflow_type: String, + pub(super) task_queue: String, + pub(super) required_build_id: Option, +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + pub(super) async fn definition_context( + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + task_queue: Option<&str>, + required_build_id: Option, + ) -> Result { + let started = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") + .bind(DB::string(run, "id")?) + .fetch_optional(&mut **tx) + .await? + .map(|row| DB::document_row(&row, "payload")) + .transpose()? + .unwrap_or(Value::Null); + Ok(DefinitionContext { + worker_pinned: started["workflow_definition_fingerprint_source"] == "worker", + recorded: trimmed(started["workflow_definition_fingerprint"].as_str()), + namespace: DB::string(run, "namespace")?, + workflow_type: DB::string(run, "workflow_type")?, + task_queue: match task_queue { + Some(queue) => queue.to_owned(), + None => DB::string(run, "queue")?, + }, + required_build_id: required_build_id.or(trimmed(DB::optional_string(run, "compatibility")?.as_deref())), + }) + } + + /// `forWorker`: whether this worker may take the run's workflow task, and + /// PHP's status for the decision. + pub(super) async fn definition_allows( + &self, + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + worker: &DB::Row, + context: &DefinitionContext, + ) -> Result<(bool, &'static str)> { + if !context.worker_pinned { + return Ok((true, "not_worker_pinned")); + } + let Some(recorded) = &context.recorded else { + return Ok((false, "recorded_definition_missing")); + }; + let advertised = DB::optional_document_row(worker, "workflow_definition_fingerprints")? + .and_then(|fingerprints| trimmed(fingerprints[&context.workflow_type].as_str())); + let Some(advertised) = advertised else { + return Ok((false, "worker_definition_missing")); + }; + if &advertised == recorded { + return Ok((true, "matching_definition")); + } + for (_, fingerprint, _) in self.definition_peers(tx, run, context).await? { + if fingerprint.as_deref() == Some(recorded.as_str()) { + return Ok((false, "matching_definition_preferred")); + } + } + // Match the embedded resolver: prefer the recorded source when it is + // available, otherwise let the current definition validate replay. + let capabilities = DB::optional_document_row(worker, "capabilities")?.unwrap_or(Value::Null); + Ok(if has_capability(&capabilities, ENTRY_VALIDATION_CAPABILITY) { + (true, "replay_required") + } else { + (false, "entry_validation_unavailable") + }) + } + + /// `eligibleWorkers`: the live workers on the run's queue (and build) that + /// handle its type, take workflow tasks and can replay its history. Each + /// is its id, advertised fingerprint and capabilities. + pub(super) async fn definition_peers( + &self, + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + context: &DefinitionContext, + ) -> Result, Value)>> { + let rows = Self::query("SELECT worker_id,build_id,status,last_heartbeat_at,workflow_definition_fingerprints,supported_workflow_types,max_concurrent_workflow_tasks,capabilities,process_metrics FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2 AND status='active' ORDER BY worker_id") + .bind(&context.namespace) + .bind(&context.task_queue) + .fetch_all(&mut **tx) + .await?; + let mut peers = Vec::new(); + for row in rows { + if context + .required_build_id + .as_deref() + .is_some_and(|build| DB::optional_string(&row, "build_id").ok().flatten().as_deref() != Some(build)) + { + continue; + } + let types = DB::optional_document_row(&row, "supported_workflow_types")?.unwrap_or(Value::Null); + if !types.as_array().is_some_and(|types| types.iter().any(|kind| kind == context.workflow_type.as_str())) + || DB::optional_number(&row, "max_concurrent_workflow_tasks")?.is_some_and(|max| max < 1) + { + continue; + } + if !self.replay_eligible(tx, run, &row).await? { + continue; + } + let fingerprint = DB::optional_document_row(&row, "workflow_definition_fingerprints")? + .and_then(|fingerprints| trimmed(fingerprints[&context.workflow_type].as_str())); + let capabilities = DB::optional_document_row(&row, "capabilities")?.unwrap_or(Value::Null); + peers.push((DB::string(&row, "worker_id")?, fingerprint, capabilities)); + } + Ok(peers) + } + + /// `WorkflowReplayEligibility::allows` for a registered worker, with the + /// protocol version it last registered or heartbeated with. + async fn replay_eligible(&self, tx: &mut Transaction<'_, DB>, run: &DB::Row, worker: &DB::Row) -> Result { + // `WorkerPollFence::isFresh`: active, with a heartbeat inside the + // stale window. + let fresh = DB::string(worker, "status")? == "active" + && DB::optional_instant(worker, "last_heartbeat_at")?.is_some_and(|heartbeat| { + heartbeat >= now() - chrono::Duration::seconds(self.worker_stale_after_seconds) + }); + if !fresh { + return Ok(false); + } + let capabilities = DB::optional_document_row(worker, "capabilities")?.unwrap_or(Value::Null); + let protocol = DB::optional_document_row(worker, "process_metrics")? + .and_then(|metrics| trimmed(metrics[super::worker_management::PROTOCOL_METRIC].as_str())); + if Self::cancellation_request(run)?.is_some() + && !super::cooperative::supports(&capabilities, protocol.as_deref().unwrap_or("")) + { + return Ok(false); + } + // `WorkflowMetadataCapabilityPolicy::canReplayRun`: memo history needs + // `memo_upserts`, typed search attribute history needs typed upserts. + let run_id = DB::string(run, "id")?; + if !has_capability(&capabilities, "memo_upserts") + && Self::scalar("SELECT COUNT(*) FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='MemoUpserted'") + .bind(&run_id) + .fetch_one(&mut **tx) + .await? + > 0 + { + return Ok(false); + } + if !has_capability(&capabilities, super::search_attribute_upserts::CAPABILITY) { + let typed = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SearchAttributesUpserted' ORDER BY sequence LIMIT 1001") + .bind(&run_id) + .fetch_all(&mut **tx) + .await?; + for event in &typed { + if DB::document_row(event, "payload")?.get("attribute_types").is_some() { + return Ok(false); + } + } + } + Ok(true) + } + + /// `inspect`: how the run's next ready workflow task would be routed, + /// with counts of the eligible workers by definition, for describe, + /// repair and debug. + pub(super) async fn definition_inspect( + &self, + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + task_queue: Option<&str>, + ) -> Result { + let run_id = DB::string(run, "id")?; + let task = Self::query("SELECT queue,compatibility FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status='ready' AND (available_at IS NULL OR available_at<=$2) ORDER BY id") + .bind(&run_id) + .bind(DB::bind_time(now())) + .fetch_all(&mut **tx) + .await? + .into_iter() + .find(|task| task_queue.is_none_or(|queue| DB::optional_string(task, "queue").ok().flatten().as_deref() == Some(queue))); + let (queue, required) = match &task { + Some(task) => ( + task_queue.map(str::to_owned).or(DB::optional_string(task, "queue")?), + trimmed(DB::optional_string(task, "compatibility")?.as_deref()), + ), + None => (task_queue.map(str::to_owned), None), + }; + let context = Self::definition_context(tx, run, queue.as_deref(), required).await?; + let (mut matching, mut changed, mut missing, mut unsafe_replacement) = (0, 0, 0, 0); + let mut examples = Vec::new(); + let peers = self.definition_peers(tx, run, &context).await?; + for (worker_id, fingerprint, capabilities) in &peers { + match fingerprint { + None => missing += 1, + Some(fingerprint) if Some(fingerprint) == context.recorded.as_ref() => matching += 1, + Some(_) => { + changed += 1; + if !has_capability(capabilities, ENTRY_VALIDATION_CAPABILITY) { + unsafe_replacement += 1; + } + } + } + if examples.len() < 10 { + examples.push(json!({"worker_id": worker_id, "fingerprint": fingerprint})); + } + } + let eligible = matching + changed + missing; + let status = match () { + _ if !context.worker_pinned => "not_worker_pinned", + _ if context.recorded.is_none() => "recorded_definition_missing", + _ if matching > 0 => "matching_definition_available", + _ if changed > unsafe_replacement => "replay_required", + _ if unsafe_replacement > 0 => "entry_validation_unavailable", + _ if missing > 0 => "worker_definition_missing", + _ => "no_eligible_worker", + }; + let terminal = !matches!(DB::string(run, "status")?.as_str(), "pending" | "running" | "waiting"); + let runnable = !terminal && task.is_some(); + let blocked = runnable + && context.worker_pinned + && matches!(status, "recorded_definition_missing" | "worker_definition_missing" | "no_eligible_worker" | "entry_validation_unavailable"); + let resolution = match () { + _ if blocked && status == "entry_validation_unavailable" => "Upgrade the replacement worker SDK to an implementation advertising workflow_entry_arguments_validation, or restore the recorded definition. The current replacement cannot safely refuse incompatible entry input.", + _ if blocked => "Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.", + _ if status == "replay_required" => "Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.", + _ => "Recorded build, namespace, type and replay-capability guards remain in force.", + }; + Ok(json!({ + "pinning_enabled": true, "worker_pinned": context.worker_pinned, + "recorded_fingerprint": context.recorded, "workflow_type": context.workflow_type, + "task_queue": context.task_queue, "required_build_id": context.required_build_id, + "status": status, "blocked": blocked, "runnable_task": runnable, + "eligible_worker_count": eligible, "matching_worker_count": matching, + "changed_worker_count": changed, "missing_fingerprint_count": missing, + "entry_validation_unavailable_count": unsafe_replacement, + "workers": examples, "workers_truncated": eligible > examples.len(), + "resolution": resolution, + })) + } + + /// `forQueue`: bounded examples of the queue's runs with a due ready + /// workflow task, never a repair or a payload read. + pub(super) async fn definition_for_queue(&self, tx: &mut Transaction<'_, DB>, task_queue: &str) -> Result { + let runs = Self::query("SELECT * FROM workflow_runs WHERE namespace=$1 AND queue=$2 AND id IN (SELECT workflow_run_id FROM workflow_tasks WHERE task_type='workflow' AND status='ready' AND queue=$3 AND (available_at IS NULL OR available_at<=$4)) ORDER BY id LIMIT 11") + .bind(self.namespace.as_ref()) + .bind(task_queue) + .bind(task_queue) + .bind(DB::bind_time(now())) + .fetch_all(&mut **tx) + .await?; + let mut examples = Vec::new(); + for run in runs.iter().take(10) { + if !matches!(DB::string(run, "status")?.as_str(), "pending" | "running" | "waiting") { + continue; + } + let mut example = json!({"workflow_id": DB::string(run, "workflow_instance_id")?, "run_id": DB::string(run, "id")?}); + if let (Some(example), Value::Object(inspected)) = + (example.as_object_mut(), self.definition_inspect(tx, run, Some(task_queue)).await?) + { + example.extend(inspected); + } + examples.push(example); + } + let blocked = examples.iter().filter(|row| row["blocked"] == true).count(); + Ok(json!({"sample_limit": 10, "has_more_ready_runs": runs.len() > 10, + "blocked_sample_count": blocked, "runs": examples})) + } +} diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 7fd36531f..a658f84bd 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -149,6 +149,11 @@ impl Runtime { max: self.queries.history_page_size_max, } } + /// The wait of a workflow or activity poll that names none + /// (DW_WORKER_POLL_TIMEOUT). + pub(super) fn worker_poll_wait(&self) -> u64 { + u64::try_from(self.queries.worker_poll_timeout_seconds).unwrap_or(0) + } /// Immutable configured-token cohort. Database transitions and scheduler /// stay shared; configuring credentials does not mutate process state. pub fn with_role_tokens( @@ -359,6 +364,7 @@ impl Runtime { delegate!(delete_search_attribute(name: &str) -> Result); delegate!(set_external_policy(name: &str, body: Value) -> Result); delegate!(update_namespace(name: &str, body: Value) -> Result); + delegate!(delete_namespace(name: &str) -> Result); delegate!( upload_external_local( local: crate::external_payload::LocalStorage, @@ -513,6 +519,7 @@ impl Runtime { ) -> Result { let body = super::start_request::normalized(body); super::input_envelope::transport(&body)?; + super::laravel_validation::operation_name(name, "signal")?; super::command_requests::signal(&body)?; let arguments = self.command_arguments(&body).await?; let prepared = super::store::prepare_control_arguments( @@ -629,6 +636,7 @@ impl Runtime { ) -> Result<(StatusCode, Value)> { let body = super::start_request::normalized(body); super::input_envelope::transport(&body)?; + super::laravel_validation::operation_name(name, "update")?; super::command_requests::update(&body)?; let arguments = self.command_arguments(&body).await?; if arguments.len() > super::queries::ARGUMENT_BYTES { @@ -684,9 +692,7 @@ impl Runtime { } error }; - if name.len() > 256 { - return Err(invalid("query_name", "The query name exceeds the maximum length of 256 bytes.")); - } + super::laravel_validation::operation_name(name, "query")?; self.query_target(workflow_id, run_id).await?; let input = super::laravel_validation::normalized(body.get("input")).filter(|value| !value.is_null()); if input.as_ref().is_some_and(|value| !value.is_object() && !value.is_array()) { @@ -883,18 +889,22 @@ impl Runtime { page_size: i64, ) -> Result ); - pub(crate) async fn register(&self, body: Value) -> Result { + #[cfg(test)] + pub(crate) async fn register_unversioned(&self, body: Value) -> Result { + self.register(body, None).await + } + pub(crate) async fn register(&self, body: Value, protocol: Option<&str>) -> Result { match &*self.storage { - Storage::Sqlite(store) => store.register(body, &self.queries).await, - Storage::Postgres(store) => store.register(body, &self.queries).await, - Storage::MySql(store) => store.register(body, &self.queries).await, + Storage::Sqlite(store) => store.register(body, &self.queries, protocol).await, + Storage::Postgres(store) => store.register(body, &self.queries, protocol).await, + Storage::MySql(store) => store.register(body, &self.queries, protocol).await, } } - pub(crate) async fn worker_heartbeat(&self, body: Value) -> Result { + pub(crate) async fn worker_heartbeat(&self, body: Value, protocol: Option<&str>) -> Result { match &*self.storage { - Storage::Sqlite(store) => store.worker_heartbeat(body, &self.queries).await, - Storage::Postgres(store) => store.worker_heartbeat(body, &self.queries).await, - Storage::MySql(store) => store.worker_heartbeat(body, &self.queries).await, + Storage::Sqlite(store) => store.worker_heartbeat(body, &self.queries, protocol).await, + Storage::Postgres(store) => store.worker_heartbeat(body, &self.queries, protocol).await, + Storage::MySql(store) => store.worker_heartbeat(body, &self.queries, protocol).await, } } pub(crate) async fn list_workers( @@ -945,8 +955,9 @@ impl Runtime { Storage::MySql(store) => store.deregister_fenced(worker_id, body, codec).await, } } + #[cfg(test)] delegate!(poll(body: Value, kind: &'static str, protocol_version: &str) -> Result); - delegate!(poll_with_pages(body: Value, kind: &'static str, protocol_version: &str, pages: super::history_pages::HistoryPages) -> Result); + delegate!(poll_with_pages(body: Value, kind: &'static str, protocol_version: &str, pages: super::history_pages::HistoryPages, default_wait: u64) -> Result); delegate!(heartbeat_task(task_id: &str, body: Value) -> Result); pub(crate) async fn deliver_cancellation(&self, task_id: &str, body: Value, protocol: &str) -> Result { let codec = self.transition_codec.clone(); @@ -1120,6 +1131,7 @@ impl Runtime { } delegate!(heartbeat_activity(task_id: &str, body: Value, protocol: &str) -> Result); delegate!(activity_status(task_id: &str, body: Value, protocol: &str) -> Result); + delegate!(renew_activity_lease(task_id: &str, body: Value) -> Result); pub(crate) async fn fail_activity(&self, task_id: &str, body: Value) -> Result { let (failure, blob) = super::activity_failures::prepare_failure(&body, self.signal_codec.clone()).await?; diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index b6bb73eef..8511adf76 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -32,7 +32,7 @@ pub fn router(runtime: Runtime) -> Router { ) .route( "/api/namespaces/{namespace}", - get(describe_namespace).put(update_namespace), + get(describe_namespace).put(update_namespace).delete(delete_namespace), ) .route( "/api/namespaces/{namespace}/external-storage", @@ -216,6 +216,8 @@ pub fn router(runtime: Runtime) -> Router { .route("/api/system/activity-timeouts/pass", post(activity_timeout_pass)) .route("/api/workers", get(list_workers)) .route("/api/task-queues", get(list_task_queues)) + .route("/api/service-catalog/{catalog}", get(service_catalog_list)) + .route("/api/service-catalog/{catalog}/{id}", get(service_catalog_show)) .route("/api/task-queues/{task_queue}", get(describe_task_queue)) .route("/api/task-queues/{task_queue}/build-ids", get(task_queue_build_ids)) .route( @@ -260,6 +262,10 @@ pub fn router(runtime: Runtime) -> Router { "/api/worker/activity-tasks/{task_id}/heartbeat", post(heartbeat_activity), ) + .route( + "/api/worker/activity-tasks/{task_id}/renew-lease", + post(renew_activity_lease), + ) .route( "/api/worker/activity-tasks/{task_id}/fail", post(fail_activity), @@ -411,8 +417,8 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) } else if path.starts_with("/api/workers/") && request.method() != Method::GET || path == "/api/namespaces" && request.method() == Method::POST || path == "/api/storage/test" - // Updating a namespace or its external storage needs an admin. - || path.starts_with("/api/namespaces/") && request.method() == Method::PUT + // Updating or deleting a namespace, or its external storage, needs an admin. + || path.starts_with("/api/namespaces/") && matches!(*request.method(), Method::PUT | Method::DELETE) // PHP's system routes are admin-only. || path.starts_with("/api/system/") || path == "/api/runtime-credentials" @@ -570,6 +576,10 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { if path.starts_with("/api/external-payloads/v1/") { return method == "GET"; } + // The service catalog is namespace-scoped and, in native, always empty. + if path.starts_with("/api/service-catalog/") { + return method == "GET"; + } // Schedules are namespace-scoped rows, as in PHP. if path == "/api/schedules" || path.starts_with("/api/schedules/") { return true; @@ -698,7 +708,7 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { | [ "activity-tasks", _, - "complete" | "fail" | "heartbeat" | "status" + "complete" | "fail" | "heartbeat" | "status" | "renew-lease" ] ); } @@ -721,6 +731,12 @@ async fn update_namespace( ) -> Result> { Ok(Json(runtime.update_namespace(&namespace, body).await?)) } +async fn delete_namespace( + Extension(runtime): Extension, + Path(namespace): Path, +) -> Result> { + Ok(Json(runtime.delete_namespace(&namespace).await?)) +} async fn create_namespace( Extension(runtime): Extension, Json(body): Json, @@ -1319,29 +1335,46 @@ async fn register( headers: HeaderMap, Json(body): Json, ) -> Result<(StatusCode, Json)> { - if body["capabilities"] - .as_array() - .is_some_and(|flags| flags.iter().any(|flag| flag == "memo_upserts")) - { - memo_protocol(&headers)?; + let protocol = request_protocol(&headers)?; + // Laravel trims every string and turns empty ones into null first. + let mut body = super::laravel_rules::normalize(&body); + super::worker_admission::admit(&body, Some(protocol.trim())).map_err(worker_refusal)?; + // PHP names an unnamed worker with a fresh ULID and stores the + // normalized capability manifest. + if body["worker_id"].is_null() { + body["worker_id"] = json!(ulid::Ulid::new().to_string()); } - if body["capabilities"].as_array().is_some_and(|flags| { - flags - .iter() - .any(|flag| flag == super::search_attribute_upserts::CAPABILITY) - }) && !protocol_at_least(&headers, 16)? - { - return Err(super::RuntimeError::Protocol { - status: StatusCode::CONFLICT, - response: json!({"reason":"workflow_metadata_capability_protocol_mismatch", - "worker_capability":super::search_attribute_upserts::CAPABILITY, - "minimum_protocol_version":super::search_attribute_upserts::MINIMUM_PROTOCOL_VERSION,"recorded":false}), - }); + let manifest = super::worker_admission::manifest(&body); + body["capability_manifest"] = manifest; + // Missing type lists are empty; capabilities keep their non-empty + // strings (`nonEmptyStringArray`). + for field in ["supported_workflow_types", "supported_activity_types"] { + if body[field].is_null() { + body[field] = json!([]); + } + } + let flags: Vec = body["capabilities"] + .as_array() + .into_iter() + .flatten() + .filter(|flag| flag.as_str().is_some_and(|flag| !flag.is_empty())) + .cloned() + .collect(); + body["capabilities"] = json!(flags); + let registered = runtime.register(body, Some(protocol)).await.map_err(worker_refusal)?; + Ok((StatusCode::CREATED, Json(worker_response(registered)))) +} + +/// `WorkerProtocol::json` names the protocol and server capabilities on +/// every worker response, refusals included. +fn worker_refusal(error: super::RuntimeError) -> super::RuntimeError { + match error { + super::RuntimeError::Protocol { status, response } => super::RuntimeError::Protocol { + status, + response: worker_response(response), + }, + other => other, } - Ok(( - StatusCode::CREATED, - Json(worker_response(runtime.register(body).await?)), - )) } fn worker_response(mut body: Value) -> Value { @@ -1351,9 +1384,10 @@ fn worker_response(mut body: Value) -> Value { } async fn worker_heartbeat( Extension(runtime): Extension, + headers: HeaderMap, Json(body): Json, ) -> Result> { - runtime.worker_heartbeat(body).await.map(Json) + runtime.worker_heartbeat(body, Some(request_protocol(&headers)?)).await.map(Json) } fn metrics_config(runtime: &Runtime) -> super::operator_metrics::MetricsConfig { super::operator_metrics::MetricsConfig { @@ -1539,8 +1573,9 @@ async fn poll_workflow( Json(body): Json, ) -> Result> { let pages = runtime.history_pages(); + let wait = runtime.worker_poll_wait(); runtime - .poll_with_pages(body, "workflow", request_protocol(&headers)?, pages) + .poll_with_pages(body, "workflow", request_protocol(&headers)?, pages, wait) .await .map(Json) } @@ -1549,8 +1584,9 @@ async fn poll_activity( headers: HeaderMap, Json(body): Json, ) -> Result> { + let wait = runtime.worker_poll_wait(); runtime - .poll(body, "activity", request_protocol(&headers)?) + .poll_with_pages(body, "activity", request_protocol(&headers)?, Default::default(), wait) .await .map(Json) } @@ -1730,6 +1766,14 @@ async fn heartbeat_activity( .map(Json) } +async fn renew_activity_lease( + Extension(runtime): Extension, + Path(task_id): Path, + Json(body): Json, +) -> Result> { + runtime.renew_activity_lease(&task_id, body).await.map(Json) +} + async fn activity_status( Extension(runtime): Extension, Path(task_id): Path, @@ -1838,6 +1882,29 @@ async fn fail_query( ) -> Result> { runtime.finish_query(&task_id, body, true).await.map(Json) } +/// `GET /api/service-catalog/{catalog}`: endpoints, services, operations or calls. +async fn service_catalog_list( + Extension(runtime): Extension, + Path(catalog): Path, + Query(query): Query>, +) -> Response { + match super::service_catalog::Catalog::from_path(&catalog) { + Some(catalog) => super::service_catalog::list(&query, catalog, runtime.namespace()).map(Json).into_response(), + None => unavailable().await, + } +} + +/// `GET /api/service-catalog/{catalog}/{id}`. +async fn service_catalog_show( + Path((catalog, _id)): Path<(String, String)>, + Query(query): Query>, +) -> Response { + match super::service_catalog::Catalog::from_path(&catalog) { + Some(catalog) => super::service_catalog::show(&query, catalog).map(Json).into_response(), + None => unavailable().await, + } +} + async fn unavailable() -> Response { refuse( StatusCode::NOT_IMPLEMENTED, diff --git a/rust/src/runtime/laravel_rules.rs b/rust/src/runtime/laravel_rules.rs index 66cc2a192..28b3b471e 100644 --- a/rust/src/runtime/laravel_rules.rs +++ b/rust/src/runtime/laravel_rules.rs @@ -25,10 +25,16 @@ pub(crate) enum Rule { String, Array, Integer, + /// `numeric`: PHP's `is_numeric`. + Numeric, Boolean, + /// `required_array_keys`: an array holding each named key. + RequiredArrayKeys(&'static [&'static str]), /// A size floor: a number, string length or item count, by type rule. Min(i64), Max(i64), + /// `between:min,max`: a size within both bounds, inclusive. + Between(i64, i64), In(&'static [&'static str]), /// `date`, for the ISO-8601 shapes `schedule_backfill::php_date` reads. Date, @@ -97,7 +103,12 @@ fn expand(data: &Value, pattern: &str) -> Vec { Some(Value::Object(fields)) => next.extend(fields.keys().map(|key| joined(key))), _ => {} } - } else { + } else if prefix.is_empty() + || !pattern_has_wildcard_before(pattern, &prefix) + || lookup(data, &prefix).is_some_and(|item| item.is_array() || item.is_object()) + { + // Laravel initializes `item.field` only under array items + // (`data_set` with `*`), so a scalar item has no such field. next.push(joined(segment)); } } @@ -106,6 +117,12 @@ fn expand(data: &Value, pattern: &str) -> Vec { paths } +/// Whether `prefix`, a path built from `pattern`, already passed a `*`. +fn pattern_has_wildcard_before(pattern: &str, prefix: &str) -> bool { + let depth = prefix.split('.').count(); + pattern.split('.').take(depth).any(|segment| segment == "*") +} + /// Laravel's displayable attribute: expanded `*` attributes keep their raw /// path; others read with spaces for underscores. fn display(attribute: &str, expanded: bool) -> String { @@ -163,7 +180,7 @@ fn php_string(value: &Value) -> Option { } fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) -> String { - let kind = if rules.iter().any(|rule| matches!(rule, Rule::Integer)) { + let kind = if rules.iter().any(|rule| matches!(rule, Rule::Integer | Rule::Numeric)) { "numeric" } else if rules.iter().any(|rule| matches!(rule, Rule::Array)) || value.is_some_and(Value::is_array) { "array" @@ -178,6 +195,10 @@ fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) - Rule::String => format!("The {attribute} field must be a string."), Rule::Array => format!("The {attribute} field must be an array."), Rule::Integer => format!("The {attribute} field must be an integer."), + Rule::Numeric => format!("The {attribute} field must be a number."), + Rule::RequiredArrayKeys(keys) => { + format!("The {attribute} field must contain entries for: {}.", keys.join(", ")) + } Rule::Boolean => format!("The {attribute} field must be true or false."), Rule::In(_) => format!("The selected {attribute} is invalid."), Rule::Min(limit) => match kind { @@ -190,6 +211,11 @@ fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) - "array" => format!("The {attribute} field must not have more than {limit} items."), _ => format!("The {attribute} field must not be greater than {limit} characters."), }, + Rule::Between(min, max) => match kind { + "numeric" => format!("The {attribute} field must be between {min} and {max}."), + "array" => format!("The {attribute} field must have between {min} and {max} items."), + _ => format!("The {attribute} field must be between {min} and {max} characters."), + }, Rule::NotBlank | Rule::Pattern(_) => format!("The {attribute} field format is invalid."), Rule::Date => format!("The {attribute} field must be a valid date."), Rule::Nullable | Rule::Sometimes => String::new(), @@ -199,7 +225,7 @@ fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) - /// Laravel's `getSize`: a numeric value under a numeric rule, otherwise /// an item count or the string length. fn size(value: &Value, rules: &[Rule]) -> Option { - if rules.iter().any(|rule| matches!(rule, Rule::Integer)) { + if rules.iter().any(|rule| matches!(rule, Rule::Integer | Rule::Numeric)) { match value { Value::Number(number) => return number.as_f64(), Value::String(text) if super::schedule_cron::php_is_numeric(text) => { @@ -231,6 +257,14 @@ fn passes(rule: Rule, data: &Value, value: Option<&Value>, rules: &[Rule]) -> bo .is_some_and(|text| super::schedule_backfill::php_date(text).is_some()), Rule::Array => value.is_some_and(|value| value.is_array() || value.is_object()), Rule::Integer => value.and_then(integer).is_some(), + Rule::Numeric => value.is_some_and(|value| match value { + Value::Number(_) => true, + Value::String(text) => super::schedule_cron::php_is_numeric(text), + _ => false, + }), + Rule::RequiredArrayKeys(keys) => value + .and_then(Value::as_object) + .is_some_and(|fields| keys.iter().all(|key| fields.contains_key(*key))), Rule::Boolean => matches!( value, Some(Value::Bool(_)) @@ -247,6 +281,9 @@ fn passes(rule: Rule, data: &Value, value: Option<&Value>, rules: &[Rule]) -> bo Rule::Max(limit) => value .and_then(|value| size(value, rules)) .is_some_and(|size| size <= limit as f64), + Rule::Between(min, max) => value + .and_then(|value| size(value, rules)) + .is_some_and(|size| size >= min as f64 && size <= max as f64), } } diff --git a/rust/src/runtime/laravel_validation.rs b/rust/src/runtime/laravel_validation.rs index f3623fad6..064aaa6e7 100644 --- a/rust/src/runtime/laravel_validation.rs +++ b/rust/src/runtime/laravel_validation.rs @@ -248,3 +248,20 @@ fn messages( } found } + +/// `WorkflowController::validateOperationName`, which a signal, update or +/// query request runs first: the name is one to 256 bytes +/// (DW_MAX_OPERATION_NAME_LENGTH, which native runs only at its default) +/// without ASCII control characters. +pub(super) fn operation_name(name: &str, kind: &str) -> super::Result<()> { + let message = if name.is_empty() { + format!("The {kind} name must not be empty.") + } else if name.len() > 256 { + format!("The {kind} name exceeds the maximum length of 256 bytes.") + } else if name.bytes().any(|byte| byte < 0x20 || byte == 0x7f) { + format!("The {kind} name must not contain control characters.") + } else { + return Ok(()); + }; + Err(failure_messages(vec![(format!("{kind}_name"), vec![message])]).expect("one message")) +} diff --git a/rust/src/runtime/memos.rs b/rust/src/runtime/memos.rs index 27cd0119a..0418763fe 100644 --- a/rust/src/runtime/memos.rs +++ b/rust/src/runtime/memos.rs @@ -454,7 +454,7 @@ mod tests { .unwrap(), ); store.register(json!({"worker_id":"memo-unit-worker", "task_queue":"test", "runtime":"php", - "supported_workflow_types":["echo"], "supported_activity_types":[], "capabilities":["memo_upserts"]}), &crate::runtime::QueryTaskConfig::default()).await.unwrap(); + "supported_workflow_types":["echo"], "supported_activity_types":[], "capabilities":["memo_upserts"]}), &crate::runtime::QueryTaskConfig::default(), None).await.unwrap(); let original = store .start( json!({"workflow_id":"memo-unit", "workflow_type":"echo", "task_queue":"test", diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 8723ac1ff..88948c1c4 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -2,11 +2,33 @@ //! arguments use the official codec on a bounded blocking worker. //! Typed database adapters share transitions; existing PHP data requires qualification. +/// Store methods for every backend: the bounds the shared SQL helpers need, +/// written once rather than in each module. +macro_rules! store_methods { + ($($body:tt)*) => { + impl $crate::runtime::store::Store + where + for<'c> &'c mut DB::Connection: ::sqlx::Executor<'c, Database = DB>, + DB::Arguments: ::sqlx::IntoArguments, + for<'q> &'q str: ::sqlx::Encode<'q, DB> + ::sqlx::Type, + for<'q> Option<&'q str>: ::sqlx::Encode<'q, DB>, + for<'q> Option: ::sqlx::Encode<'q, DB>, + for<'q> String: ::sqlx::Encode<'q, DB> + ::sqlx::Type, + for<'q> i64: ::sqlx::Encode<'q, DB> + ::sqlx::Type, + for<'r> i64: ::sqlx::Decode<'r, DB>, + usize: ::sqlx::ColumnIndex, + { + $($body)* + } + }; +} + mod active_summary; mod activity_completion_uploads; mod activity_deadlines; mod activity_failures; mod activity_heartbeats; +mod activity_lease_renewal; mod activity_metadata; mod activity_metadata_read; mod activity_results; @@ -21,6 +43,7 @@ mod children; mod command_context; mod command_requests; mod continuations; +mod definition_routing; mod control_plane; mod cooperative; mod cross_kind_poll; @@ -42,6 +65,7 @@ mod memo_json; mod memo_limits; mod memos; pub mod mysql; +mod namespace_cleanup; mod namespaces; mod operator_dashboard; mod operator_metrics; @@ -78,6 +102,7 @@ mod search_attribute_upserts; mod search_attributes; mod selection_summary; mod selections; +mod service_catalog; mod sqlite; mod start_definition; mod start_request; @@ -101,6 +126,7 @@ mod wait_timeouts; mod worker_contracts; mod worker_incarnations; mod worker_management; +mod worker_admission; mod workflow_failures; mod workflow_task_guard; diff --git a/rust/src/runtime/namespace_cleanup.rs b/rust/src/runtime/namespace_cleanup.rs new file mode 100644 index 000000000..3fbd980f1 --- /dev/null +++ b/rust/src/runtime/namespace_cleanup.rs @@ -0,0 +1,215 @@ +//! `DELETE /api/namespaces/{namespace}`: PHP's NamespaceController::destroy +//! over NamespaceLifecycleCleanup. One transaction deletes every row the +//! namespace owns in PHP's order and reports the non-zero counts under PHP's +//! keys. PHP checks each table's columns first; the statements here use the +//! columns the verified schema has, so a table without a run or instance +//! column is matched by the other, as PHP's column check does. +//! +//! Native does not delete external payload objects yet. A namespace that +//! still references any refuses before changing anything, with PHP's answer +//! for unavailable external payload storage. +use super::retention::reference; +use super::{Result, RuntimeError}; +use axum::http::StatusCode; +use serde_json::{Map, Value, json}; + +/// The namespace's runs, instances and tasks, as subqueries. Each takes the +/// namespace in its own numbered placeholder. +macro_rules! runs { + ($n:literal) => { + concat!("SELECT id FROM workflow_runs WHERE namespace=", $n) + }; +} +macro_rules! instances { + ($a:literal, $b:literal) => { + concat!( + "SELECT id FROM workflow_instances WHERE namespace=", $a, + " UNION SELECT workflow_instance_id FROM workflow_runs WHERE namespace=", $b + ) + }; +} +macro_rules! tasks { + ($a:literal, $b:literal) => { + concat!( + "SELECT id FROM workflow_tasks WHERE namespace=", $a, + " UNION SELECT id FROM workflow_tasks WHERE workflow_run_id IN (", runs!($b), ")" + ) + }; +} +/// A table matched by its run and instance columns. +macro_rules! by_run_or_instance { + ($table:literal) => { + ( + $table, + concat!( + "DELETE FROM ", $table, " WHERE workflow_run_id IN (", runs!("$1"), + ") OR workflow_instance_id IN (", instances!("$2", "$3"), ")" + ), + 3, + ) + }; +} +/// A table with only a run column. +macro_rules! by_run { + ($table:literal) => { + ($table, concat!("DELETE FROM ", $table, " WHERE workflow_run_id IN (", runs!("$1"), ")"), 1) + }; +} +macro_rules! by_instance_or_run { + ($table:literal, $instance:literal, $run:literal) => { + ( + $table, + concat!( + "DELETE FROM ", $table, " WHERE ", $instance, " IN (", instances!("$1", "$2"), + ") OR ", $run, " IN (", runs!("$3"), ")" + ), + 3, + ) + }; +} +macro_rules! by_namespace { + ($table:literal) => { + ($table, concat!("DELETE FROM ", $table, " WHERE namespace=$1"), 1) + }; +} + +/// PHP's report key, statement and namespace binds, in PHP's order. +const CLEANUP: &[(&str, &str, usize)] = &[ + by_namespace!("runtime_external_payload_cleanup_stats"), + ( + "activity_attempts", + concat!( + "DELETE FROM activity_attempts WHERE workflow_run_id IN (", runs!("$1"), + ") OR workflow_task_id IN (", tasks!("$2", "$3"), ")" + ), + 3, + ), + by_instance_or_run!("workflow_updates", "workflow_instance_id", "workflow_run_id"), + by_instance_or_run!("workflow_signal_records", "workflow_instance_id", "workflow_run_id"), + by_instance_or_run!("workflow_messages", "workflow_instance_id", "workflow_run_id"), + by_instance_or_run!("workflow_child_calls", "parent_workflow_instance_id", "parent_workflow_run_id"), + by_instance_or_run!("workflow_links", "parent_workflow_instance_id", "parent_workflow_run_id"), + ( + "workflow_service_calls", + "DELETE FROM workflow_service_calls WHERE namespace=$1 OR target_namespace=$2", + 2, + ), + // A caller namespace's context is cleared from the calls that remain, so + // a namespace recreated with the same name does not inherit them. + ( + "workflow_service_call_caller_contexts", + concat!( + "UPDATE workflow_service_calls SET caller_namespace=NULL,caller_workflow_instance_id=NULL,", + "caller_workflow_run_id=NULL,caller_principal_subject=NULL,caller_principal_method=NULL,", + "caller_principal_roles=NULL,caller_principal_tenant=NULL,caller_principal_claims=NULL ", + "WHERE caller_namespace=$1 OR caller_workflow_instance_id IN (", instances!("$2", "$3"), + ") OR caller_workflow_run_id IN (", runs!("$4"), ")" + ), + 4, + ), + by_run!("activity_executions"), + by_run_or_instance!("workflow_commands"), + by_run!("workflow_failures"), + by_run!("workflow_history_events"), + by_run_or_instance!("workflow_memos"), + by_run_or_instance!("workflow_run_lineage_entries"), + by_run_or_instance!("workflow_run_timer_entries"), + by_run_or_instance!("workflow_run_timeline_entries"), + by_run_or_instance!("workflow_run_waits"), + by_run!("workflow_run_timers"), + by_run_or_instance!("workflow_search_attributes"), + by_run!("workflow_tasks"), + by_namespace!("workflow_schedule_history_events"), + by_namespace!("workflow_schedules"), + by_namespace!("workflow_service_operations"), + by_namespace!("workflow_services"), + by_namespace!("workflow_service_endpoints"), + by_namespace!("workflow_worker_compatibility_heartbeats"), + by_namespace!("workflow_worker_sessions"), + by_namespace!("workflow_worker_build_id_rollouts"), + by_namespace!("workflow_worker_registrations"), + by_namespace!("search_attribute_definitions"), + by_namespace!("workflow_durable_stream_items"), + by_namespace!("workflow_durable_streams"), + by_namespace!("workflow_inbound_stream_items"), + by_namespace!("workflow_inbound_streams"), + // Run summaries have no run column; PHP matches them by instance. + ( + "workflow_run_summaries", + concat!("DELETE FROM workflow_run_summaries WHERE workflow_instance_id IN (", instances!("$1", "$2"), ")"), + 2, + ), + by_namespace!("workflow_runs"), + by_namespace!("workflow_instances"), +]; + +/// Native's own rows for the namespace. PHP has no such tables, so they are +/// not reported. Task-keyed receipts cascade with their tasks. +const NATIVE_CLEANUP: &[&str] = &[ + "DELETE FROM dw_poll_receipts WHERE namespace=$1", + "DELETE FROM dw_worker_registration_incarnations WHERE namespace=$1", +]; + +/// Registered external payloads and payload columns holding a reference. +const EXTERNAL_PAYLOADS: &str = concat!( + "SELECT (SELECT COUNT(*) FROM runtime_external_payloads WHERE namespace=$1)", + "+(SELECT COUNT(*) FROM workflow_runs WHERE namespace=$2 AND (", reference!("arguments"), " OR ", reference!("output"), "))", + "+(SELECT COUNT(*) FROM workflow_commands WHERE workflow_run_id IN (", runs!("$3"), ") AND ", reference!("payload"), ")", + "+(SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id IN (", runs!("$4"), ") AND ", reference!("arguments"), ")", + "+(SELECT COUNT(*) FROM workflow_updates WHERE workflow_run_id IN (", runs!("$5"), ") AND (", reference!("arguments"), " OR ", reference!("result"), "))", + "+(SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id IN (", runs!("$6"), ") AND (", reference!("arguments"), " OR ", reference!("result"), "))", +); + +pub(super) fn not_found(name: &str) -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message": format!("Namespace [{name}] not found."), "reason": "namespace_not_found", "namespace": name}), + } +} + +store_methods! { + pub(super) async fn delete_namespace(&self, name: &str) -> Result { + let name = name.to_ascii_lowercase(); + let mut tx = self.begin().await?; + if Self::scalar("SELECT COUNT(*) FROM workflow_namespaces WHERE name=$1") + .bind(&name) + .fetch_one(&mut *tx) + .await? + == 0 + { + return Err(not_found(&name)); + } + let mut external = Self::scalar(EXTERNAL_PAYLOADS); + for _ in 0..6 { + external = external.bind(&name); + } + if external.fetch_one(&mut *tx).await? > 0 { + return Err(RuntimeError::Protocol { + status: StatusCode::SERVICE_UNAVAILABLE, + response: json!({ + "message": "Namespace cleanup requires deleting external payloads, but external payload storage is unavailable.", + "reason": "external_payload_storage_driver_unavailable", + "namespace": name, + "error": "The native server does not delete external payload objects yet.", + }), + }); + } + let mut deleted = Map::new(); + for (key, sql, binds) in CLEANUP { + let mut query = Self::query(sql); + for _ in 0..*binds { + query = query.bind(&name); + } + let count = DB::affected(query.execute(&mut *tx).await?); + if count > 0 { + deleted.insert((*key).into(), json!(count)); + } + } + for sql in NATIVE_CLEANUP { + Self::query(sql).bind(&name).execute(&mut *tx).await?; + } + Self::query("DELETE FROM workflow_namespaces WHERE name=$1").bind(&name).execute(&mut *tx).await?; + tx.commit().await?; + Ok(json!({"name": name, "status": "deleted", "deleted": deleted})) + } +} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index b2245e017..eee15364b 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -553,11 +553,15 @@ where pub(super) async fn poll_query(&self, body: Value, config: &QueryTaskConfig) -> Result { let owner = text(&body, "worker_id")?; text(&body, "task_queue")?; - let timeout = body - .get("timeout_seconds") - .map_or(Some(5), Value::as_u64) - .filter(|v| *v <= 60) - .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_poll_timeout"))?; + // `queryPollTimeoutSeconds`: the requested wait, or the worker poll + // wait (DW_WORKER_POLL_TIMEOUT), capped by DW_QUERY_TASK_POLL_TIMEOUT. + let timeout = match body.get("timeout_seconds") { + None | Some(Value::Null) => u64::try_from(config.worker_poll_timeout_seconds).unwrap_or(0), + Some(value) => super::laravel_validation::integer(value) + .filter(|v| (0..=60).contains(v)) + .and_then(|v| u64::try_from(v).ok()) + .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_poll_timeout"))?, + }; let timeout = timeout.min(config.poll_timeout_seconds.max(1) as u64); let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout); // One request belongs to the registration observed before its wait. @@ -788,7 +792,8 @@ where return Err(refuse(StatusCode::CONFLICT, "task_queue_mismatch")); } let types = DB::document_row(&worker, "supported_workflow_types")?; - let capabilities = DB::document_row(&worker, "capabilities")?; + // A registration without capabilities has none (`stringArray`). + let capabilities = DB::optional_document_row(&worker, "capabilities")?.unwrap_or_default(); if !capabilities .as_array() .is_some_and(|values| values.iter().any(|v| v == "query_tasks")) diff --git a/rust/src/runtime/query_authority.rs b/rust/src/runtime/query_authority.rs index f23b5315f..11b0bffeb 100644 --- a/rust/src/runtime/query_authority.rs +++ b/rust/src/runtime/query_authority.rs @@ -27,6 +27,9 @@ pub struct QueryTaskConfig { pub history_page_size_default: i64, pub history_page_size_max: i64, pub poll_timeout_seconds: i64, + /// How long a workflow or activity poll that names no `timeout_seconds` + /// waits, as PHP's `LongPoller`: `max(0, DW_WORKER_POLL_TIMEOUT)`. + pub worker_poll_timeout_seconds: i64, } impl Default for QueryTaskConfig { @@ -43,6 +46,7 @@ impl Default for QueryTaskConfig { history_page_size_default: 500, history_page_size_max: 1000, poll_timeout_seconds: 5, + worker_poll_timeout_seconds: 30, } } } @@ -118,6 +122,7 @@ impl QueryTaskConfig { 5, )? .max(1), + worker_poll_timeout_seconds: poll.max(0), }) } diff --git a/rust/src/runtime/query_codec_tests.rs b/rust/src/runtime/query_codec_tests.rs index 4fc813a87..c91ed219f 100644 --- a/rust/src/runtime/query_codec_tests.rs +++ b/rust/src/runtime/query_codec_tests.rs @@ -33,7 +33,7 @@ async fn original(timeout_seconds: i64) -> Original { lease_timeout_seconds: timeout_seconds, ..Default::default() }); - runtime.register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + runtime.register_unversioned(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", "supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":["query_tasks"], "workflow_command_contracts":{"echo":{"queries":["state"],"query_contracts":[{"name":"state","parameters":[]}]}}})).await.unwrap(); let started = runtime diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index d331102c1..9e888fde5 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -61,6 +61,8 @@ macro_rules! reference { }; } +pub(super) use reference; + /// The run's payload columns that can hold an external reference. History /// events repeat these payloads, so they are not scanned again. const REFERENCES_SQL: &str = concat!( diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 51c1fe383..55aebc489 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -268,6 +268,11 @@ where fields.insert("compatibility_status".into(), json!(if supported { "compatible" } else { "no_compatible_worker" })); fields.insert("compatibility_supported_in_fleet".into(), json!(supported)); fields.insert("compatibility_fleet_reason".into(), json!(fleet_reason)); + // `ServerWorkflowControlPlane::withDefinitionRouting`. + let mut tx = self.begin_read().await?; + let routing = self.definition_inspect(&mut tx, run, None).await?; + tx.commit().await?; + fields.insert("definition_routing".into(), routing); fields.insert("payload_previews".into(), json!({"input_omitted":input_omitted,"output_omitted":output_omitted, "max_encoded_bytes":max_payload_bytes()})); fields.insert("last_progress_at".into(), iso8601(DB::optional_instant(run, "last_progress_at")?)); diff --git a/rust/src/runtime/run_diagnostics.rs b/rust/src/runtime/run_diagnostics.rs index 6f4c1656e..03c4bca78 100644 --- a/rust/src/runtime/run_diagnostics.rs +++ b/rust/src/runtime/run_diagnostics.rs @@ -835,6 +835,18 @@ where findings.extend(self.routing_findings(&run, &rows, at, &fleet, config).await?); } payload["cancellation_cascade"] = cascade; + // `WorkflowRunDiagnostics`: the definition route of the next ready + // task, and a warning when no eligible worker can take it. + let mut tx = self.begin_read().await?; + let routing = self.definition_inspect(&mut tx, &run, None).await?; + tx.commit().await?; + if routing["blocked"] == true { + findings.push(json!({"severity": "warning", "code": "workflow_definition_routing_blocked", + "message": "A ready workflow task has no eligible definition route.", + "routing_status": routing["status"], "expected_resolution": routing["resolution"]})); + payload["diagnostic_status"] = json!("needs_attention"); + } + payload["definition_routing"] = routing; payload["findings"] = json!(findings); Ok(payload) } diff --git a/rust/src/runtime/service_catalog.rs b/rust/src/runtime/service_catalog.rs new file mode 100644 index 000000000..8596c3ffa --- /dev/null +++ b/rust/src/runtime/service_catalog.rs @@ -0,0 +1,195 @@ +//! Service catalog reads, as published Server 2.11 serves them +//! (`ServiceCatalogReadController`): endpoints, services, operations and +//! service calls, each listed by page or read by identity. +//! +//! Native never holds catalog rows. Takeover refuses a PHP database with +//! services, endpoints, operations or service calls, and native has no path +//! that writes them. Every list is therefore PHP's empty page and every +//! identity PHP's not-found answer, after PHP's validation of the filters. +use super::laravel_rules::{self, Rule}; +use super::laravel_validation; +use super::{Result, RuntimeError}; +use axum::http::StatusCode; +use serde_json::{Map, Value, json}; + +const DEFAULT_PAGE_SIZE: i64 = 50; +const SCOPES: &[&str] = &["relevant", "owned", "caller", "target"]; +const STATUSES: &[&str] = &["pending", "accepted", "started", "completed", "failed", "cancelled"]; +const OUTCOMES: &[&str] = &[ + "accepted", "completed", "cancelled", "timed_out", "rejected_not_found", "rejected_forbidden", + "rejected_throttled", "rejected_concurrency_limited", "rejected_circuit_open", "degraded", "handler_failed", +]; +const STATUS_BUCKETS: &[&str] = &["open", "completed", "failed", "cancelled"]; +const OUTCOME_BUCKETS: &[&str] = &["open", "completed", "failed", "cancelled", "policy"]; + +const NAMESPACE: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::Max(191)]; +const PAGE: &[Rule] = &[Rule::Sometimes, Rule::Integer, Rule::Min(1)]; +const PER_PAGE: &[Rule] = &[Rule::Sometimes, Rule::Integer, Rule::Between(1, 100)]; +const IDENTITY: &[Rule] = &[Rule::Sometimes, Rule::Required, Rule::String, Rule::Max(191)]; +const SCOPE: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::In(SCOPES)]; +const STATUS: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::In(STATUSES)]; +const STATUS_BUCKET: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::In(STATUS_BUCKETS)]; +const OUTCOME: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::In(OUTCOMES)]; +const OUTCOME_BUCKET: &[Rule] = &[Rule::Sometimes, Rule::String, Rule::In(OUTCOME_BUCKETS)]; + +/// The catalog a route reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Catalog { + Endpoints, + Services, + Operations, + Calls, +} + +impl Catalog { + pub(crate) fn from_path(segment: &str) -> Option { + match segment { + "endpoints" => Some(Self::Endpoints), + "services" => Some(Self::Services), + "operations" => Some(Self::Operations), + "calls" => Some(Self::Calls), + _ => None, + } + } + + /// The identity filters a list accepts, in PHP's order. + fn identities(self) -> &'static [&'static str] { + match self { + Self::Endpoints => &[], + Self::Services => &["endpoint_id"], + Self::Operations => &["endpoint_id", "service_id"], + Self::Calls => &["endpoint_id", "service_id", "operation_id"], + } + } +} + +/// PHP's `filters`: unknown keys first, then each rule in its order, then +/// the bucket agreement. Returns the normalized query. +fn filters(query: &[(String, String)], catalog: Catalog, detail: bool) -> Result { + let mut rules: Vec<(&'static str, &'static [Rule])> = vec![("namespace", NAMESPACE)]; + if !detail { + rules.push(("page", PAGE)); + rules.push(("per_page", PER_PAGE)); + rules.extend(catalog.identities().iter().map(|id| (*id, IDENTITY))); + if catalog == Catalog::Calls { + rules.extend([("scope", SCOPE), ("status", STATUS), ("status_bucket", STATUS_BUCKET), + ("bucket", STATUS_BUCKET), ("outcome", OUTCOME), ("outcome_bucket", OUTCOME_BUCKET)]); + } + } + let mut unknown: Vec<(String, Vec)> = Vec::new(); + for (key, _) in query { + if !rules.iter().any(|(name, _)| name == key) && !unknown.iter().any(|(seen, _)| seen == key) { + unknown.push((key.clone(), vec!["Unsupported catalog filter.".into()])); + } + } + if let Some(error) = laravel_validation::failure_messages(unknown) { + return Err(error); + } + // A repeated key keeps its last value, as PHP's query parser does. + let mut body = Map::new(); + for (key, value) in query { + body.insert(key.clone(), json!(value)); + } + let body = laravel_rules::normalize(&Value::Object(body)); + laravel_rules::validate(&body, &rules)?; + if let (Some(bucket), Some(status_bucket)) = (body["bucket"].as_str(), body["status_bucket"].as_str()) + && bucket != status_bucket + { + return Err(laravel_validation::failure(vec![("bucket", vec!["Must match status_bucket when both are supplied."])]) + .expect("a message")); + } + Ok(body) +} + +fn whole(value: &Value, default: i64) -> i64 { + value.as_str().and_then(|text| text.parse().ok()).unwrap_or(default) +} + +/// A page of the catalog: always empty in native. +pub(crate) fn list(query: &[(String, String)], catalog: Catalog, namespace: &str) -> Result { + let filters = filters(query, catalog, false)?; + let mut page = json!({"data": [], "current_page": whole(&filters["page"], 1), + "per_page": whole(&filters["per_page"], DEFAULT_PAGE_SIZE), "total": 0, "last_page": 1, "namespace": namespace}); + if catalog == Catalog::Calls { + let status_bucket = match &filters["status_bucket"] { + Value::Null => filters["bucket"].clone(), + bucket => bucket.clone(), + }; + let buckets = |names: &[&str], members: &[&[&str]]| -> Value { + Value::Object(names.iter().zip(members).map(|(name, values)| ((*name).to_owned(), json!(values))).collect()) + }; + let extra = json!({ + "scope": if filters["scope"].is_null() { json!("relevant") } else { filters["scope"].clone() }, + "bucket": status_bucket, "status_bucket": status_bucket, + "outcome": filters["outcome"], "outcome_bucket": filters["outcome_bucket"], + "statuses": STATUSES, "outcomes": OUTCOMES, + "status_buckets": buckets(STATUS_BUCKETS, &[&["pending", "accepted", "started"], &["completed"], &["failed"], &["cancelled"]]), + "outcome_buckets": buckets(OUTCOME_BUCKETS, &[&["accepted"], &["completed", "degraded"], &["timed_out", "handler_failed"], + &["cancelled"], &["rejected_not_found", "rejected_forbidden", "rejected_throttled", + "rejected_concurrency_limited", "rejected_circuit_open"]]), + }); + page.as_object_mut().expect("page").extend(extra.as_object().expect("extra").clone()); + } + Ok(page) +} + +/// One catalog object by identity: never present in native. +pub(crate) fn show(query: &[(String, String)], catalog: Catalog) -> Result { + filters(query, catalog, true)?; + Err(RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message": "Catalog object not found.", "reason": "catalog_object_not_found"}), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn query(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs.iter().map(|(key, value)| ((*key).to_owned(), (*value).to_owned())).collect() + } + + fn refusal(result: Result) -> Value { + match result { + Err(RuntimeError::Protocol { status, response }) => json!([status.as_u16(), response]), + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn lists_are_empty_pages_with_the_requested_paging() { + assert_eq!(list(&query(&[("page", "2"), ("per_page", "2")]), Catalog::Endpoints, "default").unwrap(), + json!({"data":[],"current_page":2,"per_page":2,"total":0,"last_page":1,"namespace":"default"})); + let calls = list(&query(&[("bucket", "open")]), Catalog::Calls, "second").unwrap(); + assert_eq!([&calls["scope"], &calls["bucket"], &calls["status_bucket"], &calls["outcome"], &calls["namespace"]], + [&json!("relevant"), &json!("open"), &json!("open"), &Value::Null, &json!("second")]); + assert_eq!(calls["outcome_buckets"]["policy"].as_array().unwrap().len(), 5); + } + + #[test] + fn filters_fail_like_laravel() { + assert_eq!(refusal(list(&query(&[("per_page", "0")]), Catalog::Endpoints, "default"))[1]["message"], + "The per page field must be between 1 and 100."); + assert_eq!(refusal(list(&query(&[("page", "x")]), Catalog::Endpoints, "default"))[1]["message"], + "The page field must be an integer."); + let unknown = refusal(list(&query(&[("bogus", "1"), ("other", "2")]), Catalog::Endpoints, "default")); + assert_eq!(unknown[1]["message"], "Unsupported catalog filter. (and 1 more error)"); + assert_eq!(refusal(list(&query(&[("endpoint_id", "x")]), Catalog::Endpoints, "default"))[1]["errors"], + json!({"endpoint_id":["Unsupported catalog filter."]})); + assert_eq!(refusal(list(&query(&[("endpoint_id", "")]), Catalog::Services, "default"))[1]["message"], + "The endpoint id field is required."); + assert_eq!(refusal(list(&query(&[("bucket", "open"), ("status_bucket", "failed")]), Catalog::Calls, "default"))[1], + json!({"message":"Must match status_bucket when both are supplied.","reason":"validation_failed", + "errors":{"bucket":["Must match status_bucket when both are supplied."]}, + "validation_errors":{"bucket":["Must match status_bucket when both are supplied."]}})); + assert_eq!(refusal(show(&query(&[("page", "2")]), Catalog::Endpoints))[1]["errors"], + json!({"page":["Unsupported catalog filter."]})); + } + + #[test] + fn identities_are_not_found() { + assert_eq!(refusal(show(&query(&[("namespace", "default")]), Catalog::Calls)), + json!([404, {"message":"Catalog object not found.","reason":"catalog_object_not_found"}])); + } +} diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 5f1a66539..b798a252a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -567,6 +567,8 @@ where // The authenticated request's context; a direct call records only // the request id and signal name. let context = context.unwrap_or_else(|| json!({"request":{"request_id":body["request_id"]},"signal_name":name})); + // PHP's signal_name column holds 255 bytes, one fewer than its + // validation allows; native refuses what it cannot store. if name.trim().is_empty() || name.len() > 255 { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, @@ -1125,6 +1127,7 @@ where &self, body: Value, config: &super::QueryTaskConfig, + protocol: Option<&str>, ) -> Result { let worker_id = text(&body, "worker_id")?; let queue = text(&body, "task_queue")?; @@ -1142,7 +1145,8 @@ where } } let runtime = text(&body, "runtime")?; - let registration = super::worker_management::Registration::parse(&body)?; + let mut registration = super::worker_management::Registration::parse(&body)?; + registration.record_protocol(protocol); let supplied_contracts = body .get("workflow_command_contracts") .map(|contracts| { @@ -1252,31 +1256,40 @@ where ) } + /// A poll with the default page sizes and PHP's default 30-second wait. + #[cfg(test)] pub(crate) async fn poll( &self, body: Value, kind: &'static str, protocol_version: &str, ) -> Result { - self.poll_with_pages(body, kind, protocol_version, Default::default()) + self.poll_with_pages(body, kind, protocol_version, Default::default(), 30) .await } /// A workflow or activity poll, with the configured history page sizes. + /// A poll that names no `timeout_seconds` waits `default_wait` seconds, + /// as PHP's `LongPoller` waits DW_WORKER_POLL_TIMEOUT. pub(crate) async fn poll_with_pages( &self, body: Value, kind: &'static str, protocol_version: &str, pages: super::history_pages::HistoryPages, + default_wait: u64, ) -> Result { text(&body, "worker_id")?; text(&body, "task_queue")?; - let timeout = body - .get("timeout_seconds") - .map_or(Some(5), Value::as_u64) - .filter(|t| *t <= 60) - .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_poll_timeout"))?; + // Laravel's `nullable|integer|min:0|max:60` bounds a requested wait; + // the configured default is not capped. + let timeout = match body.get("timeout_seconds") { + None | Some(Value::Null) => default_wait, + Some(value) => super::laravel_validation::integer(value) + .filter(|t| (0..=60).contains(t)) + .and_then(|t| u64::try_from(t).ok()) + .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_poll_timeout"))?, + }; let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout); let requested = super::history_pages::requested_size(&body)?; let encoding = if kind == "workflow" { @@ -1479,6 +1492,17 @@ where { continue; } + // `matchesRecordedWorkflowDefinition`: a run pinned to its recorded + // definition goes to a worker that has it, while one is live. + if kind == "workflow" { + let required = DB::optional_string(&task, "compatibility")? + .map(|build| build.trim().to_owned()) + .filter(|build| !build.is_empty()); + let context = Self::definition_context(&mut tx, &run, Some(queue), required).await?; + if !self.definition_allows(&mut tx, &run, &worker, &context).await?.0 { + continue; + } + } if kind == "workflow" && let Some(update_id) = task_payload["workflow_update_id"].as_str() { @@ -1565,7 +1589,7 @@ where super::activity_deadlines::validate_policy(&policy)?; let attempt_id = id(); let started_at = now(); - Self::query("UPDATE activity_attempts SET status='expired',closed_at=$1 WHERE activity_execution_id=$2 AND status='running'") + Self::query("UPDATE activity_attempts SET status='expired',lease_expires_at=NULL,closed_at=$1 WHERE activity_execution_id=$2 AND status='running'") .bind(DB::bind_time(started_at)).bind(&activity_id).execute(&mut *tx).await?; Self::query("INSERT INTO activity_attempts(id,activity_execution_id,workflow_run_id,workflow_task_id,attempt_number,status,lease_owner,lease_expires_at,started_at) VALUES ($1,$2,$3,$4,$5,'running',$6,$7,$8)") .bind(&attempt_id).bind(&activity_id).bind(&run_id).bind(&task_id).bind(attempt) @@ -2784,12 +2808,12 @@ where "ActivityCompleted", ) .await?; - Self::query("UPDATE activity_attempts SET status='completed',closed_at=$1 WHERE id=$2") + Self::query("UPDATE activity_attempts SET status='completed',lease_expires_at=NULL,closed_at=$1 WHERE id=$2") .bind(DB::bind_time(now())) .bind(attempt_id) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_tasks SET status='completed' WHERE id=$1") + Self::query("UPDATE workflow_tasks SET status='completed',lease_expires_at=NULL WHERE id=$1") .bind(task_id) .execute(&mut *tx) .await?; @@ -3792,7 +3816,8 @@ pub(crate) fn capabilities() -> Value { "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, - "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true}); + "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, + "activity_lease_renewal": true}); value["supported_workflow_task_commands"] .as_array_mut() .expect("commands") diff --git a/rust/src/runtime/task_queues.rs b/rust/src/runtime/task_queues.rs index d4359f316..68d9129bd 100644 --- a/rust/src/runtime/task_queues.rs +++ b/rust/src/runtime/task_queues.rs @@ -425,7 +425,17 @@ where /// `GET /api/task-queues/{name}`. pub(crate) async fn describe_task_queue(&self, queue: &str, config: &QueryTaskConfig) -> Result { - self.queue_view(queue, config, true).await + let mut payload = self.queue_view(queue, config, true).await?; + // `TaskQueueController::show`: sampled definition routes of the + // queue's ready runs; a blocked one needs attention. + let mut tx = self.begin_read().await?; + let routing = self.definition_for_queue(&mut tx, queue).await?; + tx.commit().await?; + if routing["blocked_sample_count"].as_u64().unwrap_or(0) > 0 { + payload["repair"]["needs_attention"] = json!(true); + } + payload["definition_routing"] = routing; + Ok(payload) } } diff --git a/rust/src/runtime/task_repair.rs b/rust/src/runtime/task_repair.rs index a4db66df1..d82373fab 100644 --- a/rust/src/runtime/task_repair.rs +++ b/rust/src/runtime/task_repair.rs @@ -90,6 +90,31 @@ where body: Value, context: Value, codec: Arc, + ) -> Result<(StatusCode, Value)> { + let (status, mut response) = self.repair_run(workflow_id, requested_run_id, body, context, codec).await?; + // `ServerWorkflowControlPlane::withDefinitionRouting`: the route of the + // repaired run's next ready task. + let mut tx = self.begin_read().await?; + let run = match response["run_id"].as_str() { + Some(run_id) => Self::query("SELECT * FROM workflow_runs WHERE id=$1 AND namespace=$2") + .bind(run_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?, + None => Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1 AND i.namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?, + }; + if let Some(run) = run { + response["definition_routing"] = self.definition_inspect(&mut tx, &run, None).await?; + } + tx.commit().await?; + Ok((status, response)) + } + + async fn repair_run( + &self, + workflow_id: &str, + requested_run_id: Option<&str>, + body: Value, + context: Value, + codec: Arc, ) -> Result<(StatusCode, Value)> { if !body.is_object() || body.get("request_id").is_some_and(|value| { diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs index 90a5d1f8a..7ab514a78 100644 --- a/rust/src/runtime/transition_codec_tests.rs +++ b/rust/src/runtime/transition_codec_tests.rs @@ -21,7 +21,7 @@ async fn runtime(directory: &tempfile::TempDir) -> Runtime { .await .unwrap(); runtime - .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + .register_unversioned(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", "supported_workflow_types":["echo"],"supported_activity_types":[], "workflow_command_contracts":{"echo":{"signals":["payload"],"signal_contracts":[{"name":"payload", "parameters":[{"name":"value","position":0,"type":"array","required":true,"variadic":false, @@ -135,7 +135,7 @@ async fn fenced_deregistration_repair_does_not_wait_for_read_codec_capacity() { .await .unwrap(); let registration = runtime - .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + .register_unversioned(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", "supported_workflow_types":["echo"],"supported_activity_types":[]})) .await .unwrap(); diff --git a/rust/src/runtime/updates.rs b/rust/src/runtime/updates.rs index 6bbdb5f10..f92870ba2 100644 --- a/rust/src/runtime/updates.rs +++ b/rust/src/runtime/updates.rs @@ -304,6 +304,8 @@ where context: Option, codec: Arc, ) -> Result<(StatusCode, Value)> { + // PHP's update_name column holds 255 bytes, one fewer than its + // validation allows; native refuses what it cannot store. if name.trim().is_empty() || name.len() > 255 { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, diff --git a/rust/src/runtime/worker_admission.rs b/rust/src/runtime/worker_admission.rs new file mode 100644 index 000000000..e8ce3f6ae --- /dev/null +++ b/rust/src/runtime/worker_admission.rs @@ -0,0 +1,228 @@ +//! Server 2.10.0 `WorkerController::register` admission: Laravel's request +//! rules, then the capability guards, in PHP's order. A passing request +//! continues to the store with the normalized capability manifest. +use super::laravel_rules::{self, Rule, Rule::*}; +use super::{Result, RuntimeError}; +use axum::http::StatusCode; +use serde_json::{Map, Value, json}; + +/// `WorkerProtocol::PORTABLE_WORKER_AFFINITY_CAPABILITIES`. +const AFFINITY: &[&str] = &["local_activities", "worker_sessions", "sticky_execution"]; +/// `WorkerProtocol::PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION`. +const AFFINITY_FLOOR: &str = "1.18"; +const COOPERATIVE: &str = "cooperative_cancellation"; +/// `CooperativeCancellationPolicy` and `PreparedLocalActivityPolicy`. +const COOPERATIVE_FLOOR: &str = "1.20"; +const PREPARED: &str = "prepared_local_activities"; +const PREPARED_GROUPS: &str = "prepared_local_activity_groups"; +const PREPARED_POLICIES: &str = "prepared_local_activity_cancellation_policies"; +const NEXUS_COOPERATIVE: &str = "nexus_cooperative_cancellation"; +/// `WorkflowMetadataCapabilityPolicy::definitions`, in order. +const METADATA: &[(&str, &str)] = &[("memo_upserts", "1.14"), ("typed_search_attributes", "1.16")]; +/// `MessageStreamsContract`. +const MESSAGE_STREAMS: (&str, &str) = ("message_streams", "1.15"); + +const RUNTIMES: &[&str] = &["php", "python", "rust", "typescript", "go", "java", "external"]; +const MANIFEST_REQUIRED: &[Rule] = &[Required, Array, RequiredArrayKeys(AFFINITY)]; +const MANIFEST_NULLABLE: &[Rule] = &[Nullable, Array, RequiredArrayKeys(AFFINITY)]; + +/// The request rules, with the manifest required from protocol 1.18 +/// (`WorkerProtocol::portableWorkerAffinitySupported`). +fn rules(manifest: &'static [Rule]) -> Vec<(&'static str, &'static [Rule])> { + vec![ + ("worker_id", &[Nullable, String, Max(255)]), + ("task_queue", &[Required, String, Max(255)]), + ("runtime", &[Required, String, In(RUNTIMES)]), + ("sdk_version", &[Nullable, String, Max(64)]), + ("build_id", &[Nullable, String, Max(255)]), + ("supported_workflow_types", &[Nullable, Array]), + ("supported_workflow_types.*", &[String]), + ("workflow_definition_fingerprints", &[Nullable, Array]), + ("workflow_definition_fingerprints.*", &[String, Max(255)]), + ("workflow_command_contracts", &[Nullable, Array]), + ("supported_activity_types", &[Nullable, Array]), + ("supported_activity_types.*", &[String]), + ("capabilities", &[Nullable, Array]), + ("capabilities.*", &[String, Max(255)]), + ("capability_manifest", manifest), + ("capability_manifest.*", &[Array]), + ("capability_manifest.*.supported", &[Required, Boolean]), + ("capability_manifest.*.minimum_protocol_version", &[Required, String, Max(32)]), + ("capability_manifest.*.implementation", &[Nullable, String, Max(255)]), + ("capability_manifest.*.reason", &[Nullable, String, Max(255)]), + ("max_concurrent_workflow_tasks", &[Nullable, Integer, Min(0)]), + ("max_concurrent_activity_tasks", &[Nullable, Integer, Min(0)]), + ("max_concurrent_worker_sessions", &[Nullable, Integer, Min(1)]), + ("task_slots", &[Nullable, Array]), + ("task_slots.workflow_available", &[Nullable, Integer, Min(0)]), + ("task_slots.activity_available", &[Nullable, Integer, Min(0)]), + ("task_slots.session_available", &[Nullable, Integer, Min(0)]), + ("process_metrics", &[Nullable, Array]), + ("process_metrics.cpu_percent", &[Nullable, Numeric, Min(0)]), + ("process_metrics.memory_bytes", &[Nullable, Integer, Min(0)]), + ("process_metrics.process_uptime_seconds", &[Nullable, Integer, Min(0)]), + ("process_metrics.process_id", &[Nullable, Integer, Min(0)]), + ("process_metrics.host", &[Nullable, String, Max(255)]), + ("process_metrics.process_started_at", &[Nullable, String, Max(64)]), + ("heartbeat_interval_seconds", &[Nullable, Integer, Min(1), Max(3600)]), + ] +} + +/// PHP's `version_compare($version, $minimum, '>=')` on dotted numbers; +/// a missing version meets nothing (`WorkerProtocol::versionMeetsMinimum`). +pub(super) fn at_least(version: Option<&str>, minimum: &str) -> bool { + let parts = |text: &str| -> Vec { text.trim().split('.').map(|part| part.parse().unwrap_or(0)).collect() }; + let Some(version) = version.filter(|version| !version.trim().is_empty()) else { + return false; + }; + let (version, minimum) = (parts(version), parts(minimum)); + let length = version.len().max(minimum.len()); + let padded = |list: &[u64]| (0..length).map(|index| list.get(index).copied().unwrap_or(0)).collect::>(); + padded(&version) >= padded(&minimum) +} + +fn refused(mut response: Value) -> RuntimeError { + response["registered"] = json!(false); + RuntimeError::Protocol { status: StatusCode::CONFLICT, response } +} + +/// The registration's flat capabilities, as PHP's `nonEmptyStringArray`. +fn capabilities(body: &Value) -> Vec<&str> { + body["capabilities"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|capability| !capability.is_empty()) + .collect() +} + +/// `portableWorkerCapabilityManifest`: the three affinity entries that are +/// arrays, with `supported` and a trimmed floor, plus non-empty +/// `implementation` and `reason`. +pub(super) fn manifest(body: &Value) -> Value { + let mut normalized = Map::new(); + for capability in AFFINITY { + let Some(entry) = body["capability_manifest"].get(*capability).filter(|entry| entry.is_object()) else { + continue; + }; + let supported = match &entry["supported"] { + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0), + Value::String(text) => !text.is_empty() && text != "0", + _ => false, + }; + let floor = match &entry["minimum_protocol_version"] { + Value::String(text) => text.trim().to_owned(), + Value::Number(number) => number.to_string(), + _ => std::string::String::new(), + }; + let mut item = json!({"supported": supported, "minimum_protocol_version": floor}); + for field in ["implementation", "reason"] { + if let Some(text) = entry[field].as_str().map(str::trim).filter(|text| !text.is_empty()) { + item[field] = json!(text); + } + } + normalized.insert((*capability).to_owned(), item); + } + Value::Object(normalized) +} + +/// The admission checks before any registration state is read. `body` is +/// the request after Laravel's trimming; `version` is the request's +/// worker protocol header. +pub(super) fn admit(body: &Value, version: Option<&str>) -> Result<()> { + let manifest_rule = if at_least(Some(version.unwrap_or(super::WORKER_PROTOCOL_VERSION)), AFFINITY_FLOOR) { + MANIFEST_REQUIRED + } else { + MANIFEST_NULLABLE + }; + laravel_rules::validate(body, &rules(manifest_rule))?; + let flags = capabilities(body); + let has = |capability: &str| flags.contains(&capability); + let cooperative = has(COOPERATIVE) && at_least(version, COOPERATIVE_FLOOR); + let requested = json!(version); + let pair = json!([COOPERATIVE, PREPARED]); + if has(PREPARED_POLICIES) && !(cooperative && has(PREPARED) && has(PREPARED_POLICIES)) { + return Err(refused(json!({"reason": "prepared_local_activity_cancellation_policy_capability_mismatch", + "required_capabilities": pair, "minimum_protocol_version": COOPERATIVE_FLOOR, "requested_version": requested}))); + } + if has(PREPARED_GROUPS) && (!has(PREPARED) || !cooperative) { + return Err(refused(json!({"reason": "prepared_local_activity_group_capability_mismatch", + "required_capabilities": pair, "minimum_protocol_version": COOPERATIVE_FLOOR, "requested_version": requested}))); + } + if has(PREPARED) && !cooperative { + return Err(refused(json!({"reason": "prepared_local_activity_capability_mismatch", + "required_capability": COOPERATIVE, "minimum_protocol_version": COOPERATIVE_FLOOR, "requested_version": requested}))); + } + if has(NEXUS_COOPERATIVE) && !cooperative { + return Err(refused(json!({"reason": "nexus_cooperative_cancellation_capability_mismatch", + "required_capabilities": [COOPERATIVE, NEXUS_COOPERATIVE], "minimum_protocol_version": COOPERATIVE_FLOOR, + "requested_version": requested}))); + } + if has(COOPERATIVE) && !cooperative { + return Err(refused(json!({"reason": "cooperative_cancellation_protocol_mismatch", + "minimum_protocol_version": COOPERATIVE_FLOOR, "requested_version": requested}))); + } + // `guardPortableWorkerCapabilityManifest`: the structured flag and the + // flat capability agree, and an advertised capability meets its floor. + let normalized = manifest(body); + for capability in AFFINITY { + let advertised = has(capability); + let entry = normalized.get(*capability); + if entry.is_none() && !advertised { + continue; + } + if entry.is_none_or(|entry| entry["supported"].as_bool() != Some(advertised)) { + return Err(refused(json!({"reason": "worker_capability_manifest_mismatch", "capability": capability, + "requested_version": requested, "minimum_protocol_version": AFFINITY_FLOOR, + "remediation": "Keep the structured supported flag and the flat routing capability in exact agreement."}))); + } + let floor = entry.and_then(|entry| entry["minimum_protocol_version"].as_str()).unwrap_or_default(); + if floor.is_empty() || !at_least(Some(floor), AFFINITY_FLOOR) || (advertised && !at_least(version, floor)) { + return Err(refused(json!({"reason": "worker_capability_protocol_floor_mismatch", "capability": capability, + "requested_version": requested, "minimum_protocol_version": AFFINITY_FLOOR, + "remediation": format!("Advertise {capability} only from a worker targeting protocol {AFFINITY_FLOOR} or newer.")}))); + } + } + if let Some((capability, minimum)) = + METADATA.iter().find(|(capability, minimum)| has(capability) && !at_least(version, minimum)) + { + return Err(refused(json!({"reason": "workflow_metadata_capability_protocol_mismatch", + "capability": capability, "requested_version": requested, "minimum_protocol_version": minimum, + "remediation": format!("Advertise {capability} only while sending worker protocol {minimum} or newer.")}))); + } + let (streams, minimum) = MESSAGE_STREAMS; + if has(streams) && !at_least(version, minimum) { + return Err(refused(json!({"reason": "message_streams_unavailable", "requested_version": requested, + "minimum_protocol_version": minimum, + "remediation": format!("Advertise message_streams only while sending worker protocol {minimum} or newer.")}))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn protocol_versions_compare_like_php() { + assert!(at_least(Some("1.20"), "1.18")); + assert!(at_least(Some("1.18"), "1.18")); + assert!(!at_least(Some("1.9"), "1.18")); + assert!(at_least(Some("2.0"), "1.20")); + assert!(!at_least(None, "1.0")); + } + + #[test] + fn the_manifest_keeps_the_affinity_entries_that_are_arrays() { + let body = json!({"capability_manifest": { + "local_activities": {"supported": 1, "minimum_protocol_version": " 1.18 ", "implementation": " sdk ", "reason": ""}, + "worker_sessions": "unsupported", + "sticky_execution": {"supported": false, "minimum_protocol_version": "1.18"}, + "other": {"supported": true, "minimum_protocol_version": "1.18"}}}); + assert_eq!(manifest(&body), json!({ + "local_activities": {"supported": true, "minimum_protocol_version": "1.18", "implementation": "sdk"}, + "sticky_execution": {"supported": false, "minimum_protocol_version": "1.18"}})); + } +} diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 968bdc729..45480584f 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -129,7 +129,6 @@ pub(super) async fn token() -> Result { #[cfg(test)] mod tests { - use super::*; use crate::runtime::{Runtime, RuntimeError}; use serde_json::json; @@ -142,7 +141,7 @@ mod tests { .unwrap(); let definition = json!({"worker_id":"original-worker","task_queue":"original-queue", "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]}); - runtime.register(definition.clone()).await.unwrap(); + runtime.register_unversioned(definition.clone()).await.unwrap(); let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", path.display())) .await .unwrap(); @@ -154,7 +153,7 @@ mod tests { let mut replacement = definition; replacement["task_queue"] = json!("replacement-queue"); assert!(matches!( - runtime.register(replacement).await, + runtime.register_unversioned(replacement).await, Err(RuntimeError::Database(_)) )); let after: Vec<(String,String,String,String,Option,Option,String,String)> = diff --git a/rust/src/runtime/worker_management.rs b/rust/src/runtime/worker_management.rs index a80d77aa7..32d5418a1 100644 --- a/rust/src/runtime/worker_management.rs +++ b/rust/src/runtime/worker_management.rs @@ -91,6 +91,20 @@ fn slot(value: &Value, capacity: i64) -> Option { Some(count.max(0).min(capacity.max(0))) } +/// The key under which PHP records a worker's protocol version. +pub(super) const PROTOCOL_METRIC: &str = "_worker_protocol_version"; + +/// Process metrics with the request's protocol version added, as PHP's +/// `[...$metrics, PROTOCOL_METRIC => requestVersion]`. +pub(super) fn with_protocol(metrics: Option, protocol: Option<&str>) -> Value { + let mut metrics = match metrics { + Some(Value::Object(metrics)) => metrics, + _ => Map::new(), + }; + metrics.insert(PROTOCOL_METRIC.into(), json!(protocol)); + Value::Object(metrics) +} + /// Validates the process metrics a worker reports and keeps PHP's normalized /// subset. Absent means "not reported"; the outer Option is presence. fn process_metrics(body: &Value) -> Result>> { @@ -209,6 +223,13 @@ pub(super) struct Registration { } impl Registration { + /// `WorkflowReplayEligibility::PROTOCOL_METRIC`: PHP keeps the protocol + /// version of each registration and heartbeat in the worker's process + /// metrics, after the metrics the worker reported. + pub(super) fn record_protocol(&mut self, protocol: Option<&str>) { + self.process_metrics = Some(with_protocol(self.process_metrics.take(), protocol)); + } + pub(super) fn parse(body: &Value) -> Result { let fingerprints = match body.get("workflow_definition_fingerprints") { None => None, @@ -439,12 +460,13 @@ where &self, body: Value, config: &QueryTaskConfig, + protocol: Option<&str>, ) -> Result { let worker_id = super::text(&body, "worker_id")?; let metrics = process_metrics(&body)?; let interval = bounded(&body, "heartbeat_interval_seconds", 1, 3600)?; let mut tx = self.begin().await?; - let Some(worker) = Self::query("SELECT max_concurrent_workflow_tasks,max_concurrent_activity_tasks,max_concurrent_worker_sessions FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + let Some(worker) = Self::query("SELECT max_concurrent_workflow_tasks,max_concurrent_activity_tasks,max_concurrent_worker_sessions,process_metrics FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") .bind(worker_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? else { return Err(RuntimeError::Protocol { @@ -471,11 +493,15 @@ where } } } - if let Some(metrics) = metrics { - Self::query("UPDATE workflow_worker_registrations SET process_metrics=$1 WHERE worker_id=$2 AND namespace=$3") - .bind(metrics.as_ref().map(DB::document)).bind(worker_id).bind(namespace) - .execute(&mut *tx).await?; - } + // PHP stores the reported metrics, else keeps the stored ones, and adds + // this heartbeat's protocol version either way. + let reported = match metrics { + Some(metrics) => metrics, + None => DB::optional_document_row(&worker, "process_metrics").ok().flatten(), + }; + Self::query("UPDATE workflow_worker_registrations SET process_metrics=$1 WHERE worker_id=$2 AND namespace=$3") + .bind(DB::document(&with_protocol(reported, protocol))).bind(worker_id).bind(namespace) + .execute(&mut *tx).await?; if interval.is_some() { Self::set_count(&mut tx, &INTERVAL, interval, worker_id, namespace).await?; } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 746707a9a..2f60a1c43 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -5,7 +5,7 @@ use axum::{ }; use durable_workflow_server::{ codec::{Value as Payload, ValueCodec}, - runtime::{Runtime, router}, + runtime::Runtime, }; use serde_json::{Value, json}; use tower::ServiceExt; @@ -97,12 +97,20 @@ mod signal_rejection; mod cooperative_waits; #[path = "execution/child_policies.rs"] mod child_policies; +#[path = "execution/definition_routing.rs"] +mod definition_routing; +#[path = "execution/worker_admission.rs"] +mod worker_admission; +/// `WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY`. +const ENTRY_VALIDATION: &str = "workflow_entry_arguments_validation"; #[path = "execution/parent_close_policies.rs"] mod parent_close_policies; #[path = "execution/history_pages.rs"] mod history_pages; #[path = "execution/declared_signals.rs"] mod declared_signals; +#[path = "execution/poll_waits.rs"] +mod poll_waits; #[path = "execution/child_cascade.rs"] mod child_cascade; #[path = "execution/completion_open_work.rs"] @@ -122,6 +130,9 @@ mod worker_pressure; #[path = "execution/version_markers.rs"] mod version_markers; +#[path = "execution/activity_lease_renewal.rs"] +mod activity_lease_renewal; + #[path = "execution/activity_deadlines.rs"] mod activity_deadlines; @@ -3188,8 +3199,10 @@ async fn cooperative_root_request_requires_original_capable_claim_not_later_flee let registration = json!({"worker_id":"cooperative-owner","task_queue":"test","runtime":"php", "supported_workflow_types":["echo"],"supported_activity_types":[], "capabilities":if original_capable {json!(["cooperative_cancellation"])} else {json!([])}}); + // Registration needs protocol 1.20 to advertise cooperative + // cancellation; the claim records the protocol of the poll. assert_eq!( - request(&app, "POST", "/api/worker/register", registration.clone()) + request_protocol(&app, "POST", "/api/worker/register", registration.clone(), "1.20") .await .0, StatusCode::CREATED @@ -3299,7 +3312,7 @@ async fn cooperative_requested_root_ignores_protocol_spoofing_and_requires_capab let mut capable = registration; capable["capabilities"] = json!(["cooperative_cancellation"]); assert_eq!( - request(&app, "POST", "/api/worker/register", capable) + request_protocol(&app, "POST", "/api/worker/register", capable, "1.20") .await .0, StatusCode::CREATED @@ -5087,3 +5100,42 @@ async fn authentication_precedes_protocol_and_namespace_checks() { runtime.close().await; database.remove().await; } + +/// The server router, behind an SDK's registration habit: from protocol +/// 1.18 an SDK sends a `capability_manifest` that agrees with its flat +/// capabilities, so one is added where a test registration omits it. +/// Tests of the manifest rules use `durable_workflow_server::runtime::router`. +fn router(runtime: Runtime) -> Router { + durable_workflow_server::runtime::router(runtime).layer(axum::middleware::from_fn(sdk_capability_manifest)) +} + +async fn sdk_capability_manifest(request: Request, next: axum::middleware::Next) -> axum::response::Response { + let floor = request + .headers() + .get("x-durable-workflow-protocol-version") + .and_then(|version| version.to_str().ok()) + .and_then(|version| version.trim().strip_prefix("1.")) + .and_then(|minor| minor.parse::().ok()) + .is_some_and(|minor| minor >= 18); + if request.uri().path() != "/api/worker/register" || !floor { + return next.run(request).await; + } + let (parts, body) = request.into_parts(); + let bytes = to_bytes(body, usize::MAX).await.unwrap(); + let bytes = match serde_json::from_slice::(&bytes) { + Ok(Value::Object(mut fields)) if !fields.contains_key("capability_manifest") => { + let flags = fields.get("capabilities").cloned().unwrap_or(Value::Null); + let advertised = |capability: &str| flags.as_array().is_some_and(|flags| flags.iter().any(|flag| flag == capability)); + let manifest: serde_json::Map = ["local_activities", "worker_sessions", "sticky_execution"] + .into_iter() + .map(|capability| { + (capability.to_owned(), json!({"supported": advertised(capability), "minimum_protocol_version": "1.18"})) + }) + .collect(); + fields.insert("capability_manifest".into(), Value::Object(manifest)); + Value::Object(fields).to_string().into_bytes() + } + _ => bytes.to_vec(), + }; + next.run(Request::from_parts(parts, Body::from(bytes))).await +} diff --git a/rust/tests/execution/activity_lease_renewal.rs b/rust/tests/execution/activity_lease_renewal.rs new file mode 100644 index 000000000..85849e2b0 --- /dev/null +++ b/rust/tests/execution/activity_lease_renewal.rs @@ -0,0 +1,173 @@ +use super::*; + +// Expectations measured against published Server 2.11.1 (Workflow 2.10.2): +// POST /api/worker/activity-tasks/{task}/renew-lease. + +fn lease_activity() -> Value { + json!({"type":"schedule_activity","activity_type":"lease_activity", + "arguments":envelope(Payload::Array(vec![])), + "retry_policy":{"max_attempts":1,"backoff_seconds":[]}, + "heartbeat_timeout":60,"start_to_close_timeout":300}) +} + +/// Starts a run and claims its single activity. +async fn claim(app: &Router, workflow_id: &str) -> (Value, Value, Value) { + let started = start(app, workflow_id).await; + let workflow = poll(app, "lease-worker", "workflow").await; + assert_eq!(workflow["run_id"], started["run_id"], "{workflow}"); + assert_eq!(finish_task(app, &workflow, json!([lease_activity()])).await.0, StatusCode::OK); + let activity = poll(app, "lease-worker", "activity").await; + assert_eq!(activity["run_id"], started["run_id"], "{activity}"); + (started, workflow, activity) +} + +fn own(activity: &Value) -> Value { + json!({"activity_attempt_id":activity["activity_attempt_id"],"lease_owner":activity["lease_owner"]}) +} + +fn path(activity: &Value, verb: &str) -> String { + format!("/api/worker/activity-tasks/{}/{verb}", activity["task_id"].as_str().unwrap()) +} + +async fn renew(app: &Router, activity: &Value, body: Value) -> (StatusCode, Value) { + request_protocol(app, "POST", &path(activity, "renew-lease"), body, "1.20").await +} + +fn without_protocol(mut body: Value) -> Value { + let map = body.as_object_mut().unwrap(); + assert_eq!(map.remove("protocol_version"), Some(json!("1.20"))); + assert_eq!(map.remove("server_capabilities").unwrap()["activity_lease_renewal"], true); + body +} + +#[tokio::test] +async fn renewal_extends_only_the_live_lease() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "lease-worker", json!(["echo"]), json!(["lease_activity"])).await; + let (started, workflow, activity) = claim(&app, "lease-live").await; + let status_before = request_protocol(&app, "POST", &path(&activity, "status"), own(&activity), "1.20").await; + assert_eq!(status_before.0, StatusCode::OK, "{}", status_before.1); + let history_before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + let (code, renewed) = renew(&app, &activity, own(&activity)).await; + assert_eq!(code, StatusCode::OK, "{renewed}"); + let mut renewed = without_protocol(renewed); + let lease = renewed.as_object_mut().unwrap().remove("lease_expires_at").unwrap(); + let server_time = renewed.as_object_mut().unwrap().remove("server_time").unwrap(); + let heartbeat_deadline = renewed.as_object_mut().unwrap().remove("heartbeat_deadline_at").unwrap(); + assert_eq!(renewed, json!({ + "can_continue":true,"cancel_requested":false,"reason":null,"heartbeat_recorded":false, + "workflow_instance_id":"lease-live","workflow_run_id":started["run_id"],"workflow_task_id":activity["task_id"], + "activity_execution_id":activity["activity_execution_id"],"activity_attempt_id":activity["activity_attempt_id"], + "attempt_number":1,"run_status":"waiting","run_closed_reason":null,"run_closed_at":null, + "activity_status":"running","attempt_status":"running","task_status":"leased", + "lease_owner":activity["lease_owner"],"last_heartbeat_at":null, + "lease_renewed":true,"task_id":activity["task_id"],"worker_session":null})); + let micros = |value: &Value| { + let text = value.as_str().unwrap(); + assert!(text.ends_with('Z') && text.len() == 27, "{text}"); + chrono::DateTime::parse_from_rfc3339(text).unwrap() + }; + let time = micros(&server_time); + assert_eq!(micros(&lease), time + chrono::Duration::seconds(300), "the lease runs the configured 300 s from now"); + assert!(lease.as_str().unwrap() > status_before.1["lease_expires_at"].as_str().unwrap()); + assert_eq!(heartbeat_deadline, status_before.1["deadlines"]["heartbeat"], "progress deadlines never move"); + + let status_after = request_protocol(&app, "POST", &path(&activity, "status"), own(&activity), "1.20").await; + let instant = |value: &Value| chrono::DateTime::parse_from_rfc3339(value.as_str().unwrap()).unwrap(); + assert_eq!(instant(&status_after.1["lease_expires_at"]), instant(&lease), "status reads the renewed lease"); + assert_eq!(status_after.1["last_heartbeat_at"], Value::Null); + assert_eq!(status_after.1["deadlines"], status_before.1["deadlines"]); + assert_eq!(run_history(&app, &started["workflow_id"], &started["run_id"]).await, history_before, + "renewal adds no history"); + + // Like PHP, a 1.19 request and an unruled field are accepted. + assert_eq!(request(&app, "POST", &path(&activity, "renew-lease"), own(&activity)).await.0, StatusCode::OK); + let mut extra = own(&activity); + extra["message"] = json!("progress"); + assert_eq!(renew(&app, &activity, extra).await.0, StatusCode::OK); + + // Ownership refusals, in PHP's order and shape. + let refusal = |task: &Value, attempt: &Value, error: &str, reason: &str| { + json!({"task_id":task,"activity_attempt_id":attempt,"error":error,"reason":reason}) + }; + let (code, body) = renew(&app, &activity, json!({"activity_attempt_id":activity["activity_attempt_id"],"lease_owner":"someone-else"})).await; + let mut expected = refusal(&activity["task_id"], &activity["activity_attempt_id"], + "Activity attempt lease is owned by another worker.", "lease_owner_mismatch"); + expected["lease_owner"] = activity["lease_owner"].clone(); + assert_eq!((code, without_protocol(body)), (StatusCode::CONFLICT, expected)); + let (code, body) = renew(&app, &activity, json!({"activity_attempt_id":"no-such-attempt","lease_owner":activity["lease_owner"]})).await; + assert_eq!((code, without_protocol(body)), (StatusCode::NOT_FOUND, + refusal(&activity["task_id"], &json!("no-such-attempt"), "Activity attempt not found.", "attempt_not_found"))); + let (code, body) = request_protocol(&app, "POST", "/api/worker/activity-tasks/no-such-task/renew-lease", own(&activity), "1.20").await; + assert_eq!((code, without_protocol(body)), (StatusCode::NOT_FOUND, + refusal(&json!("no-such-task"), &activity["activity_attempt_id"], "Activity task not found.", "task_not_found"))); + // Another task of this namespace, here the run's workflow task. + let (code, body) = request_protocol(&app, "POST", &path(&workflow, "renew-lease"), own(&activity), "1.20").await; + assert_eq!((code, without_protocol(body)), (StatusCode::CONFLICT, + refusal(&workflow["task_id"], &activity["activity_attempt_id"], "Activity attempt is not leased for this task.", "task_mismatch"))); + + // Laravel's validation. + let (code, body) = renew(&app, &activity, json!({})).await; + assert_eq!((code, without_protocol(body)), (StatusCode::UNPROCESSABLE_ENTITY, json!({ + "message":"The activity attempt id field is required. (and 1 more error)","reason":"validation_failed", + "errors":{"activity_attempt_id":["The activity attempt id field is required."],"lease_owner":["The lease owner field is required."]}, + "validation_errors":{"activity_attempt_id":["The activity attempt id field is required."],"lease_owner":["The lease owner field is required."]}}))); + let (code, body) = renew(&app, &activity, json!({"activity_attempt_id":5,"lease_owner":["x"]})).await; + assert_eq!(code, StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(body["errors"], json!({"activity_attempt_id":["The activity attempt id field must be a string."], + "lease_owner":["The lease owner field must be a string."]})); + + // A recorded heartbeat is reported; renewal still records none. + let beat = request_protocol(&app, "POST", &path(&activity, "heartbeat"), own(&activity), "1.20").await; + assert_eq!(beat.1["heartbeat_recorded"], true, "{}", beat.1); + let (_, after_beat) = renew(&app, &activity, own(&activity)).await; + assert_eq!(instant(&after_beat["last_heartbeat_at"]), instant(&beat.1["last_heartbeat_at"])); + assert_ne!(after_beat["heartbeat_deadline_at"], heartbeat_deadline, "the heartbeat moved its own deadline"); + + // A closed attempt reports its status and cannot be renewed. + let completed = request_protocol(&app, "POST", &path(&activity, "complete"), + json!({"activity_attempt_id":activity["activity_attempt_id"],"lease_owner":activity["lease_owner"], + "result":envelope(Payload::String("done".into()))}), "1.20").await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + let (code, closed) = renew(&app, &activity, own(&activity)).await; + assert_eq!(code, StatusCode::CONFLICT, "{closed}"); + let closed = without_protocol(closed); + assert_eq!([&closed["can_continue"], &closed["cancel_requested"], &closed["reason"], &closed["lease_renewed"], + &closed["activity_status"], &closed["attempt_status"], &closed["task_status"], &closed["lease_expires_at"]], + [&json!(false), &json!(false), &json!("attempt_closed"), &json!(false), &json!("completed"), + &json!("completed"), &json!("completed"), &Value::Null]); + for absent in ["task_id", "server_time", "worker_session"] { + assert!(closed.get(absent).is_none(), "a refused status carries no {absent}: {closed}"); + } +} + +#[tokio::test] +async fn renewal_cannot_revive_an_expired_or_closed_run() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "lease-worker", json!(["echo"]), json!(["lease_activity"])).await; + let (started, _, live) = claim(&app, "lease-terminated").await; + let terminated = request(&app, "POST", &format!("/api/workflows/{}/terminate", started["workflow_id"].as_str().unwrap()), + json!({"reason":"measure"})).await; + assert_eq!(terminated.0, StatusCode::OK, "{}", terminated.1); + let (code, body) = renew(&app, &live, own(&live)).await; + assert_eq!(code, StatusCode::CONFLICT, "{body}"); + assert_eq!([&body["can_continue"], &body["cancel_requested"], &body["reason"], &body["run_status"], + &body["run_closed_reason"], &body["attempt_status"], &body["lease_renewed"]], + [&json!(false), &json!(true), &json!("run_terminated"), &json!("terminated"), &json!("terminated"), + &json!("cancelled"), &json!(false)]); + // Last: an expired lease is redelivered as a new attempt. + let (_, _, activity) = claim(&app, "lease-expired").await; + database.execute("UPDATE workflow_tasks SET lease_expires_at='2020-01-02 03:04:05.123456' WHERE task_type='activity'").await; + let (code, expired) = renew(&app, &activity, own(&activity)).await; + assert_eq!(code, StatusCode::CONFLICT, "{expired}"); + assert_eq!([&expired["reason"], &expired["lease_renewed"], &expired["can_continue"]], + [&json!("lease_expired"), &json!(false), &json!(false)]); + let status = request_protocol(&app, "POST", &path(&activity, "status"), own(&activity), "1.20").await; + assert_eq!(status.1["can_continue"], false, "an expired lease stays expired: {}", status.1); +} diff --git a/rust/tests/execution/definition_routing.rs b/rust/tests/execution/definition_routing.rs new file mode 100644 index 000000000..0e93a1fea --- /dev/null +++ b/rust/tests/execution/definition_routing.rs @@ -0,0 +1,133 @@ +use super::*; + +// Server 2.10.0 `WorkflowDefinitionRouting`: a run whose start recorded its +// worker cohort's definition fingerprint goes to a worker advertising that +// definition while one is live. A worker with another definition gets it only +// when none is, and only if it validates entry arguments. + +async fn worker(app: &Router, id: &str, fingerprint: Option<&str>, capabilities: Value) { + let mut body = json!({"worker_id":id,"task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":capabilities}); + if let Some(fingerprint) = fingerprint { + body["workflow_definition_fingerprints"] = json!({"echo": fingerprint}); + } + let (status, registered) = request(app, "POST", "/api/worker/register", body).await; + assert_eq!(status, StatusCode::CREATED, "{registered}"); +} + +async fn deregister(app: &Router, id: &str) { + let (status, body) = request(app, "DELETE", &format!("/api/workers/{id}"), Value::Null).await; + assert!(status.is_success(), "{status} {body}"); +} + +#[tokio::test] +async fn a_pinned_run_goes_to_a_worker_with_its_recorded_definition() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "original", Some("sha256:original"), json!([])).await; + let started = start(&app, "pinned").await; + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let recorded = history.as_array().unwrap().iter().find(|event| event["event_type"] == "WorkflowStarted").unwrap(); + assert_eq!(recorded["payload"]["workflow_definition_fingerprint"], "sha256:original"); + assert_eq!(recorded["payload"]["workflow_definition_fingerprint_source"], "worker"); + + // A changed definition waits while the recorded one is live, even if it + // could validate entry arguments; so does a worker without a fingerprint. + worker(&app, "changed", Some("sha256:changed"), json!([super::ENTRY_VALIDATION])).await; + worker(&app, "unfingerprinted", None, json!([super::ENTRY_VALIDATION])).await; + assert!(poll(&app, "changed", "workflow").await.is_null(), "matching_definition_preferred"); + assert!(poll(&app, "unfingerprinted", "workflow").await.is_null(), "worker_definition_missing"); + let task = poll(&app, "original", "workflow").await; + assert_eq!(task["run_id"], started["run_id"], "{task}"); +} + +#[tokio::test] +async fn without_the_recorded_definition_only_an_entry_validating_worker_replays() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "original", Some("sha256:original"), json!([])).await; + let started = start(&app, "replacement").await; + deregister(&app, "original").await; + worker(&app, "unvalidated", Some("sha256:changed"), json!([])).await; + assert!(poll(&app, "unvalidated", "workflow").await.is_null(), "entry_validation_unavailable"); + worker(&app, "validating", Some("sha256:changed"), json!([super::ENTRY_VALIDATION])).await; + let task = poll(&app, "validating", "workflow").await; + assert_eq!(task["run_id"], started["run_id"], "replay_required: {task}"); +} + +#[tokio::test] +async fn an_unpinned_run_goes_to_any_worker_and_workers_record_their_protocol() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + // No fingerprint when the run starts, so it is not worker-pinned. + worker(&app, "plain", None, json!([])).await; + let started = start(&app, "unpinned").await; + worker(&app, "fingerprinted", Some("sha256:any"), json!([])).await; + let task = poll(&app, "fingerprinted", "workflow").await; + assert_eq!(task["run_id"], started["run_id"], "not_worker_pinned: {task}"); + + // `WorkflowReplayEligibility::PROTOCOL_METRIC`: each registration and + // heartbeat records its protocol version with the process metrics. + let (status, described) = request(&app, "GET", "/api/workers/plain", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!(described["process_metrics"], json!({"_worker_protocol_version": "1.19"}), "{described}"); + let (status, beat) = request(&app, "POST", "/api/worker/heartbeat", + json!({"worker_id":"plain","process_metrics":{"host":"worker-host"}})).await; + assert_eq!(status, StatusCode::OK, "{beat}"); + let (_, described) = request(&app, "GET", "/api/workers/plain", Value::Null).await; + assert_eq!(described["process_metrics"]["host"], "worker-host", "{described}"); + assert_eq!(described["process_metrics"]["_worker_protocol_version"], "1.19", "{described}"); +} + +// Describe, repair, debug and the task queue report the route of a pinned +// run's next ready task as `definition_routing`. A blocked route is a debug +// finding and makes the queue need attention. +#[tokio::test] +async fn reads_report_a_blocked_definition_route() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "original", Some("sha256:original"), json!([])).await; + let started = start(&app, "reported").await; + deregister(&app, "original").await; + worker(&app, "unvalidated", Some("sha256:changed"), json!([])).await; + + let (status, described) = request(&app, "GET", "/api/workflows/reported", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + let routing = &described["definition_routing"]; + assert_eq!(routing["status"], "entry_validation_unavailable", "{routing}"); + assert_eq!(routing["blocked"], true); + assert_eq!(routing["runnable_task"], true); + assert_eq!(routing["worker_pinned"], true); + assert_eq!(routing["pinning_enabled"], true); + assert_eq!(routing["recorded_fingerprint"], "sha256:original"); + assert_eq!(routing["task_queue"], "test"); + assert_eq!((routing["eligible_worker_count"].clone(), routing["changed_worker_count"].clone(), + routing["entry_validation_unavailable_count"].clone()), (json!(1), json!(1), json!(1))); + assert_eq!(routing["workers"], json!([{"worker_id": "unvalidated", "fingerprint": "sha256:changed"}])); + assert!(routing["resolution"].as_str().unwrap().starts_with("Upgrade the replacement worker SDK")); + + let (status, debug) = request(&app, "GET", "/api/workflows/reported/debug", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{debug}"); + assert_eq!(debug["definition_routing"], *routing); + assert_eq!(debug["diagnostic_status"], "needs_attention"); + let finding = debug["findings"].as_array().unwrap().iter() + .find(|finding| finding["code"] == "workflow_definition_routing_blocked").unwrap_or_else(|| panic!("{debug}")); + assert_eq!(finding["routing_status"], "entry_validation_unavailable"); + + let (status, queue) = request(&app, "GET", "/api/task-queues/test", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{queue}"); + assert_eq!(queue["definition_routing"]["blocked_sample_count"], 1, "{queue}"); + assert_eq!(queue["definition_routing"]["runs"][0]["run_id"], started["run_id"]); + assert_eq!(queue["definition_routing"]["runs"][0]["workflow_id"], "reported"); + assert_eq!(queue["repair"]["needs_attention"], true, "{queue}"); + + let (status, repaired) = request(&app, "POST", "/api/workflows/reported/repair", json!({})).await; + assert!(status.is_success(), "{status} {repaired}"); + assert_eq!(repaired["definition_routing"]["status"], "entry_validation_unavailable", "{repaired}"); + + // An entry-validating replacement unblocks the route. + worker(&app, "validating", Some("sha256:changed"), json!([super::ENTRY_VALIDATION])).await; + let (_, described) = request(&app, "GET", "/api/workflows/reported", Value::Null).await; + assert_eq!(described["definition_routing"]["status"], "replay_required", "{described}"); + assert_eq!(described["definition_routing"]["blocked"], false); +} diff --git a/rust/tests/execution/namespaces.rs b/rust/tests/execution/namespaces.rs index c63637d7b..933665b6a 100644 --- a/rust/tests/execution/namespaces.rs +++ b/rust/tests/execution/namespaces.rs @@ -749,3 +749,66 @@ async fn a_named_namespace_child_runs_and_resolves_in_its_parents_namespace() { let (_, described) = scoped(&app, "alpha", "GET", "/api/workflows/named-parent", Value::Null).await; assert_eq!(described["status"], "completed", "{described}"); } + +// `DELETE /api/namespaces/{namespace}` follows PHP's NamespaceController::destroy: +// the namespace's rows go in one transaction, the non-zero counts are reported, +// other namespaces keep theirs, and a missing namespace answers 404. +#[tokio::test] +async fn namespace_delete_removes_only_what_the_namespace_owns() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + create(&app, "doomed").await; + let (status, body) = scoped(&app, "doomed", "POST", "/api/search-attributes", + json!({"name":"CustomerTier","type":"keyword"})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + start_scoped(&app, "doomed", "namespace-delete-run").await; + start_scoped(&app, "default", "namespace-delete-peer").await; + let (status, body) = scoped(&app, "default", "DELETE", "/api/namespaces/DOOMED", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + // What published PHP 2.5.14 reports for one started run and one search + // attribute definition. + assert_eq!(body, json!({"name":"doomed","status":"deleted","deleted":{ + "workflow_commands":1,"workflow_history_events":2,"workflow_tasks":1, + "search_attribute_definitions":1,"workflow_run_summaries":1,"workflow_runs":1,"workflow_instances":1}})); + for (method, uri) in [("GET", "/api/namespaces/doomed"), ("DELETE", "/api/namespaces/doomed")] { + let (status, body) = scoped(&app, "default", method, uri, Value::Null).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{method} {uri}: {body}"); + assert_eq!(body, json!({"message":"Namespace [doomed] not found.","reason":"namespace_not_found","namespace":"doomed"})); + } + let (status, body) = scoped(&app, "default", "GET", "/api/workflows/namespace-delete-peer", Value::Null).await; + assert_eq!(status, StatusCode::OK, "another namespace keeps its run: {body}"); + // The deleted run's workflow id is free again. + let restarted = start_scoped(&app, "default", "namespace-delete-run").await; + assert_eq!(restarted["outcome"], "started_new", "{restarted}"); + runtime.close().await; + database.remove().await; +} + +// Native does not delete external payload objects yet, so a namespace that +// still references one refuses, as PHP does when its storage is unavailable, +// and nothing is deleted. +#[tokio::test] +async fn namespace_delete_refuses_while_external_payloads_remain() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + create(&app, "held").await; + start_scoped(&app, "held", "namespace-delete-held").await; + database + .execute("UPDATE workflow_runs SET output='{\"reference\":\"dw-external-payload:v1:held\"}' WHERE namespace='held'") + .await; + let (status, body) = scoped(&app, "default", "DELETE", "/api/namespaces/held", Value::Null).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["reason"], "external_payload_storage_driver_unavailable"); + assert_eq!(body["namespace"], "held"); + let (status, body) = scoped(&app, "default", "GET", "/api/namespaces/held", Value::Null).await; + assert_eq!(status, StatusCode::OK, "the refused namespace remains: {body}"); + // The planted reference makes describe resolve an external payload, so + // the listing shows the run is still there. + let (status, body) = scoped(&app, "held", "GET", "/api/workflows", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body.to_string().contains("namespace-delete-held"), "its run remains: {body}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/poll_waits.rs b/rust/tests/execution/poll_waits.rs new file mode 100644 index 000000000..89a8c06db --- /dev/null +++ b/rust/tests/execution/poll_waits.rs @@ -0,0 +1,118 @@ +use super::*; +use durable_workflow_server::runtime::QueryTaskConfig; +use std::time::{Duration, Instant}; + +// Server 2.10.0's `LongPoller`: a worker poll that names no `timeout_seconds` +// (or names null) waits DW_WORKER_POLL_TIMEOUT seconds, 30 by default. A +// requested wait is a Laravel integer from 0 to 60; the setting itself is not +// capped. A query-task poll caps either wait by DW_QUERY_TASK_POLL_TIMEOUT. + +async fn timed_poll(app: &Router, path: &str, body: Value) -> (StatusCode, Value, Duration) { + let began = Instant::now(); + let (status, response) = request(app, "POST", path, body).await; + (status, response, began.elapsed()) +} + +const WORKFLOW: &str = "/api/worker/workflow-tasks/poll"; +const ACTIVITY: &str = "/api/worker/activity-tasks/poll"; +const QUERY: &str = "/api/worker/query-tasks/poll"; + +#[tokio::test] +async fn polls_without_a_wait_use_the_configured_worker_poll_wait() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let configured = |seconds| QueryTaskConfig { worker_poll_timeout_seconds: seconds, ..Default::default() }; + let one = router(runtime.clone().with_query_task_config(configured(1))); + let zero = router(runtime.with_query_task_config(configured(0))); + register(&one, "worker", json!(["echo"]), json!(["greet"])).await; + + // One configured second, where native used to wait five. + for (path, body) in [ + (WORKFLOW, json!({"worker_id":"worker","task_queue":"test"})), + (WORKFLOW, json!({"worker_id":"worker","task_queue":"test","timeout_seconds":null})), + (ACTIVITY, json!({"worker_id":"worker","task_queue":"test"})), + (QUERY, json!({"worker_id":"worker","task_queue":"test"})), + ] { + let (status, response, waited) = timed_poll(&one, path, body.clone()).await; + assert_eq!(status, StatusCode::OK, "{path} {body}: {response}"); + assert!(response["task"].is_null(), "{response}"); + assert!(waited >= Duration::from_millis(900) && waited < Duration::from_secs(4), "{path} {body} waited {waited:?}"); + } + + // A zero setting answers at once, as does a requested zero given as a + // string. + for (app, body) in [ + (&zero, json!({"worker_id":"worker","task_queue":"test"})), + (&one, json!({"worker_id":"worker","task_queue":"test","timeout_seconds":"0"})), + ] { + let (status, response, waited) = timed_poll(app, WORKFLOW, body.clone()).await; + assert_eq!(status, StatusCode::OK, "{body}: {response}"); + assert!(waited < Duration::from_millis(800), "{body} waited {waited:?}"); + } + + // A requested wait outside 0 through 60 is refused. + for timeout in [json!(61), json!(-1), json!("soon"), json!(1.5)] { + for path in [WORKFLOW, ACTIVITY, QUERY] { + let (status, response) = + request(&one, "POST", path, json!({"worker_id":"worker","task_queue":"test","timeout_seconds":timeout})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{path} {timeout}: {response}"); + } + } +} + +#[tokio::test] +async fn a_task_that_arrives_during_the_default_wait_is_claimed() { + let database = TestDatabase::new().await; + let runtime = database + .open() + .await + .unwrap() + .with_query_task_config(QueryTaskConfig { worker_poll_timeout_seconds: 10, ..Default::default() }); + let app = router(runtime); + register(&app, "worker", json!(["echo"]), json!([])).await; + let waiting = { + let app = app.clone(); + tokio::spawn(async move { timed_poll(&app, WORKFLOW, json!({"worker_id":"worker","task_queue":"test"})).await }) + }; + tokio::time::sleep(Duration::from_millis(300)).await; + let started = start(&app, "arrives-while-waiting").await; + let (status, response, waited) = waiting.await.unwrap(); + assert_eq!(status, StatusCode::OK, "{response}"); + assert_eq!(response["task"]["run_id"], started["run_id"], "{response}"); + assert!(waited < Duration::from_secs(8), "waited {waited:?}"); +} + +// `WorkflowController::validateOperationName` runs first on a signal, update +// or query: a name is one to 256 bytes without control characters. +#[tokio::test] +async fn operation_names_are_validated_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + start(&app, "named").await; + // PHP stores signal and update names in 255-byte columns, so native + // refuses a 256-byte name its validation would pass. + let (status, body) = request(&app, "POST", &format!("/api/workflows/named/signal/{}", "s".repeat(255)), json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "a 255-byte signal name is accepted: {body}"); + for (kind, reason) in [("signal", "invalid_signal_name"), ("update", "invalid_update_name")] { + let (status, body) = request(&app, "POST", &format!("/api/workflows/named/{kind}/{}", "s".repeat(256)), json!({})).await; + assert_eq!((status, body["reason"].as_str()), (StatusCode::UNPROCESSABLE_ENTITY, Some(reason)), "{body}"); + } + let (status, body) = request(&app, "POST", &format!("/api/workflows/missing/query/{}", "q".repeat(256)), json!({})).await; + assert_eq!((status, body["reason"].as_str()), (StatusCode::NOT_FOUND, Some("instance_not_found")), "a 256-byte query name passes validation: {body}"); + + let too_long = "s".repeat(257); + for (kind, name, message) in [ + ("signal", too_long.as_str(), "The signal name exceeds the maximum length of 256 bytes."), + ("signal", "bad%0Aname", "The signal name must not contain control characters."), + ("update", too_long.as_str(), "The update name exceeds the maximum length of 256 bytes."), + ("update", "bad%7Fname", "The update name must not contain control characters."), + ("query", too_long.as_str(), "The query name exceeds the maximum length of 256 bytes."), + ("query", "bad%09name", "The query name must not contain control characters."), + ] { + let (status, body) = request(&app, "POST", &format!("/api/workflows/named/{kind}/{name}"), json!({})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{kind} {name}: {body}"); + assert_eq!(body["reason"], "validation_failed", "{body}"); + assert_eq!(body["errors"][format!("{kind}_name")], json!([message]), "{body}"); + assert_eq!(body["workflow_id"], "named", "{body}"); + } +} diff --git a/rust/tests/execution/worker_admission.rs b/rust/tests/execution/worker_admission.rs new file mode 100644 index 000000000..8b7ffda77 --- /dev/null +++ b/rust/tests/execution/worker_admission.rs @@ -0,0 +1,161 @@ +use super::*; + +// Server 2.10.0 `WorkerController::register`: Laravel's request rules, with +// `capability_manifest` required from protocol 1.18, then the capability +// guards in PHP's order. These tests use the server's own router, without +// the SDK-style manifest the other tests' router adds. + +async fn app() -> (TestDatabase, Router) { + let database = TestDatabase::new().await; + let app = durable_workflow_server::runtime::router(database.open().await.unwrap()); + (database, app) +} + +fn affinity(local: bool, sessions: bool, sticky: bool) -> Value { + let entry = |supported: bool| json!({"supported": supported, "minimum_protocol_version": "1.18"}); + json!({"local_activities": entry(local), "worker_sessions": entry(sessions), "sticky_execution": entry(sticky)}) +} + +async fn register_at(app: &Router, version: &str, fields: Value) -> (StatusCode, Value) { + let mut body = json!({"worker_id": "admitted", "task_queue": "test", "runtime": "php"}); + for (field, value) in fields.as_object().unwrap() { + body[field] = value.clone(); + } + request_protocol(app, "POST", "/api/worker/register", body, version).await +} + +#[tokio::test] +async fn the_manifest_is_required_from_protocol_1_18() { + let (_database, app) = app().await; + let (status, body) = register_at(&app, "1.19", json!({})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["reason"], "validation_failed"); + assert_eq!(body["errors"]["capability_manifest"], json!(["The capability manifest field is required."]), "{body}"); + // `WorkerProtocol::json` names the protocol on refusals too. + assert_eq!(body["protocol_version"], "1.20"); + assert!(body["server_capabilities"].is_object(), "{body}"); + + let (status, body) = register_at(&app, "1.17", json!({})).await; + assert_eq!(status, StatusCode::CREATED, "an older worker needs no manifest: {body}"); + + let (status, body) = register_at(&app, "1.19", json!({"capability_manifest": {"local_activities": {}}})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!( + body["errors"], + json!({ + "capability_manifest": ["The capability manifest field must contain entries for: local_activities, worker_sessions, sticky_execution."], + "capability_manifest.local_activities.supported": ["The capability_manifest.local_activities.supported field is required."], + "capability_manifest.local_activities.minimum_protocol_version": ["The capability_manifest.local_activities.minimum_protocol_version field is required."], + }), + "{body}" + ); + + // A scalar entry is not an array and has no fields to require. + let mut manifest = affinity(false, false, false); + manifest["worker_sessions"] = json!("no"); + let (status, body) = register_at(&app, "1.19", json!({"capability_manifest": manifest})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"], json!({"capability_manifest.worker_sessions": ["The capability_manifest.worker_sessions field must be an array."]}), "{body}"); +} + +#[tokio::test] +async fn registration_fields_follow_php_rules() { + let (_database, app) = app().await; + let manifest = affinity(false, false, false); + let (status, body) = register_at(&app, "1.19", json!({"capability_manifest": manifest, "runtime": "cobol", + "process_metrics": {"cpu_percent": "high"}, "heartbeat_interval_seconds": 0})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!( + body["errors"], + json!({ + "runtime": ["The selected runtime is invalid."], + "process_metrics.cpu_percent": ["The process metrics.cpu percent field must be a number."], + "heartbeat_interval_seconds": ["The heartbeat interval seconds field must be at least 1."], + }), + "{body}" + ); + + // An unnamed worker gets a ULID; strings are trimmed and the manifest is + // stored normalized. + let mut manifest = affinity(false, false, false); + manifest["local_activities"]["implementation"] = json!(" sdk "); + manifest["local_activities"]["extra"] = json!("dropped"); + let (status, body) = register_at(&app, "1.19", json!({"worker_id": "", "task_queue": " test ", "capability_manifest": manifest})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + let worker_id = body["worker_id"].as_str().unwrap(); + assert_eq!(worker_id.len(), 26, "{body}"); + let (status, described) = request(&app, "GET", &format!("/api/workers/{worker_id}"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!(described["task_queue"], "test", "{described}"); + assert_eq!( + described["capability_manifest"]["local_activities"], + json!({"supported": false, "minimum_protocol_version": "1.18", "implementation": "sdk"}), + "{described}" + ); +} + +#[tokio::test] +async fn capability_guards_refuse_like_php() { + let (_database, app) = app().await; + let manifest = affinity(false, false, false); + let refused = |capabilities: Value, version: &'static str, manifest: Value| { + let app = app.clone(); + async move { + let (status, body) = + register_at(&app, version, json!({"capabilities": capabilities, "capability_manifest": manifest})).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["registered"], false, "{body}"); + assert_eq!(body["protocol_version"], "1.20", "{body}"); + body + } + }; + + // The flat capability and the structured flag must agree. + let body = refused(json!(["worker_sessions"]), "1.19", manifest.clone()).await; + assert_eq!((body["reason"].as_str(), body["capability"].as_str()), (Some("worker_capability_manifest_mismatch"), Some("worker_sessions")), "{body}"); + let body = refused(json!([]), "1.19", affinity(false, false, true)).await; + assert_eq!(body["capability"], "sticky_execution", "{body}"); + + // An advertised capability needs its floor at 1.18 or later and met. + let mut low = affinity(true, false, false); + low["local_activities"]["minimum_protocol_version"] = json!("1.17"); + let body = refused(json!(["local_activities"]), "1.19", low).await; + assert_eq!(body["reason"], "worker_capability_protocol_floor_mismatch", "{body}"); + let mut high = affinity(true, false, false); + high["local_activities"]["minimum_protocol_version"] = json!("1.21"); + let body = refused(json!(["local_activities"]), "1.20", high).await; + assert_eq!(body["reason"], "worker_capability_protocol_floor_mismatch", "{body}"); + assert_eq!(body["remediation"], "Advertise local_activities only from a worker targeting protocol 1.18 or newer."); + + // Cooperative cancellation and what builds on it need protocol 1.20. + let body = refused(json!(["cooperative_cancellation"]), "1.19", manifest.clone()).await; + assert_eq!(body["reason"], "cooperative_cancellation_protocol_mismatch", "{body}"); + assert_eq!(body["requested_version"], "1.19"); + let body = refused(json!(["prepared_local_activities"]), "1.20", manifest.clone()).await; + assert_eq!(body["reason"], "prepared_local_activity_capability_mismatch", "{body}"); + assert_eq!(body["required_capability"], "cooperative_cancellation"); + let body = refused(json!(["cooperative_cancellation", "prepared_local_activity_groups"]), "1.20", manifest.clone()).await; + assert_eq!(body["reason"], "prepared_local_activity_group_capability_mismatch", "{body}"); + let body = refused(json!(["prepared_local_activity_cancellation_policies"]), "1.20", manifest.clone()).await; + assert_eq!(body["reason"], "prepared_local_activity_cancellation_policy_capability_mismatch", "{body}"); + let body = refused(json!(["nexus_cooperative_cancellation"]), "1.20", manifest.clone()).await; + assert_eq!(body["reason"], "nexus_cooperative_cancellation_capability_mismatch", "{body}"); + + // Workflow metadata and message streams have their own floors. + let body = refused(json!(["memo_upserts"]), "1.13", Value::Null).await; + assert_eq!( + (body["reason"].as_str(), body["capability"].as_str(), body["minimum_protocol_version"].as_str()), + (Some("workflow_metadata_capability_protocol_mismatch"), Some("memo_upserts"), Some("1.14")), + "{body}" + ); + assert_eq!(body["remediation"], "Advertise memo_upserts only while sending worker protocol 1.14 or newer."); + let body = refused(json!(["typed_search_attributes"]), "1.15", Value::Null).await; + assert_eq!(body["capability"], "typed_search_attributes", "{body}"); + let body = refused(json!(["message_streams"]), "1.14", Value::Null).await; + assert_eq!(body["reason"], "message_streams_unavailable", "{body}"); + + // At 1.20 a cooperative worker with matching flags registers. + let (status, body) = register_at(&app, "1.20", json!({"capabilities": ["cooperative_cancellation", "local_activities"], + "capability_manifest": affinity(true, false, false)})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} diff --git a/rust/tests/execution/worker_listing.rs b/rust/tests/execution/worker_listing.rs index 1abcbc1e9..034b47201 100644 --- a/rust/tests/execution/worker_listing.rs +++ b/rust/tests/execution/worker_listing.rs @@ -82,7 +82,7 @@ async fn workers_are_listed_with_their_registration_and_heartbeat_facts() { ); assert_eq!( worker["process_metrics"], - json!({"process_uptime_seconds": 0, "process_id": 17}) + json!({"process_uptime_seconds": 0, "process_id": 17, "_worker_protocol_version": "1.19"}) ); assert_eq!(worker["status"], "active"); assert_eq!(worker["heartbeat_interval_seconds"], Value::Null); @@ -94,7 +94,9 @@ async fn workers_are_listed_with_their_registration_and_heartbeat_facts() { json!({"workflow_available": null, "activity_available": null, "session_available": null, "workflow_capacity": 100, "activity_capacity": 100, "session_capacity": 10}) ); - assert_eq!(defaults["process_metrics"], Value::Null); + // Server 2.10.0 records each registration's protocol version with the + // process metrics (`WorkflowReplayEligibility::PROTOCOL_METRIC`). + assert_eq!(defaults["process_metrics"], json!({"_worker_protocol_version": "1.19"})); assert_eq!(ours(&app, "?task_queue=queue-a").await.len(), 1); assert_eq!( ours(&app, "?task_queue=0").await.len(), @@ -162,7 +164,7 @@ async fn workers_are_listed_with_their_registration_and_heartbeat_facts() { json!({"listing": fingerprint}) ); assert_eq!(after["task_slots"]["workflow_available"], Value::Null); - assert_eq!(after["process_metrics"], Value::Null); + assert_eq!(after["process_metrics"], json!({"_worker_protocol_version": "1.19"})); assert_eq!(after["registered_at"], described["registered_at"]); // A heartbeat changes only what it reports. diff --git a/scripts/conformance/server-parity.mjs b/scripts/conformance/server-parity.mjs index 061d3b55b..f0da20bc4 100644 --- a/scripts/conformance/server-parity.mjs +++ b/scripts/conformance/server-parity.mjs @@ -40,7 +40,7 @@ Requires Node 20+ and PHP 8.3+ with the locked adapter installed. Each target must have its own isolated, already bootstrapped database. Set DW_PARITY_TOKEN for the HTTP target; it is never recorded. Use the same --prefix on all targets and a fresh database or prefix for a new run. --artifacts selects the exact -consumer tuple; the default is tests/Fixtures/ServerParityBaselines/php-2.5.14.json. +consumer tuple; the default is tests/Fixtures/ServerParityBaselines/php-2.11.1.json. Repeat --fixture PATH to record or compare an explicitly selected candidate set. --database-resource-limits accepts inspected memory_bytes and memory_swap_bytes as a JSON object; comparisons require equal effective @@ -77,7 +77,7 @@ try { } const record = { schema: 'durable-workflow.server-parity-record/v1', target: values.target, mode: values.mode, - runner_revision: values['runner-revision'], artifacts: json(values.artifacts ?? resolve(root, 'tests/Fixtures/ServerParityBaselines/php-2.5.14.json')), + runner_revision: values['runner-revision'], artifacts: json(values.artifacts ?? resolve(root, 'tests/Fixtures/ServerParityBaselines/php-2.11.1.json')), started_at: new Date().toISOString(), outcome: 'runner-blocked', fixture_hashes: hashes, cases: [], database_resource_limits: parseDatabaseResourceLimits(values['database-resource-limits']), }; diff --git a/scripts/conformance/server-parity/README.md b/scripts/conformance/server-parity/README.md index 498782d62..7ca4e4f67 100644 --- a/scripts/conformance/server-parity/README.md +++ b/scripts/conformance/server-parity/README.md @@ -282,7 +282,7 @@ Docker kill and verifies exit code 137 before restarting the native process. For an adapter installed elsewhere, set `DW_PARITY_SDK_AUTOLOAD` to its locked `vendor/autoload.php`. `--artifacts` selects a frozen tuple manifest. The default -is `tests/Fixtures/ServerParityBaselines/php-2.5.14.json`; its SDK version must match +is `tests/Fixtures/ServerParityBaselines/php-2.11.1.json`; its SDK version must match the installed lock. A frozen tuple does not prove every listed consumer ran. The original `tests/Fixtures/ServerParity/php-baseline.json` remains an immutable 2.5.13 / Workflow 2.5.4 reference, together with its known NUL reproducer outside diff --git a/scripts/conformance/server-parity/activity-lease-renewal-contract.mjs b/scripts/conformance/server-parity/activity-lease-renewal-contract.mjs new file mode 100644 index 000000000..5c7754b54 --- /dev/null +++ b/scripts/conformance/server-parity/activity-lease-renewal-contract.mjs @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; + +const MICROS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; +const RENEWALS = ['renewed', 'renewed_on_protocol_1_19', 'extra_field', 'foreign_owner', 'unknown_attempt', + 'unknown_task', 'task_mismatch', 'missing_fields', 'non_string_fields', 'renewed_after_heartbeat', + 'renewed_after_completion', 'renewed_after_cancel', 'renewed_after_terminate']; + +// Milliseconds since the epoch of a UTC timestamp with any fraction length. +function at(value, label) { + assert.ok(typeof value === 'string', `${label}: a timestamp, got ${JSON.stringify(value)}`); + const millis = Date.parse(value.replace(/\.(\d{3})\d*Z$/, '.$1Z')); + assert.ok(!Number.isNaN(millis), `${label}: ${value}`); + return millis; +} + +// Reviewed runtime activity lease renewal (ActivityTaskController::renewLease +// over Workflow's ActivityLeaseRenewal). Identities, times and leases come +// from the observation; their relations and every other field are exact. +export function checkActivityLeaseRenewal(fixture, observation, workflowId) { + const definition = fixture.activity_lease_renewal; + const state = observation.activity_lease_renewal; + const projection = {scope: 'activity_lease_renewal', lease_seconds: definition.lease_seconds}; + assert.ok(state && typeof state === 'object', 'actual lease renewal observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded lease renewal explicitly inapplicable'); + return projection; + } + const equal = (actual, expected, label) => assert.deepStrictEqual(actual, expected, label); + const live = state.live; + equal([state.applicable, state.namespace, live.workflow_id], [true, `${workflowId}-lease`.toLowerCase(), `${workflowId}-live`], + 'renewal namespace and run'); + for (const key of RENEWALS) { + equal([state[key]?.response?.protocol_version, state[key]?.lease_renewal_capability], ['1.20', true], + `${key} answers in worker protocol 1.20 and advertises renewal`); + } + // Workflow's attemptStatus for a claim, before any field is overridden. + const attempt = (claim, fields = {}) => ({can_continue: true, cancel_requested: false, reason: null, heartbeat_recorded: false, + workflow_instance_id: claim.workflow_id, workflow_run_id: claim.run_id, workflow_task_id: claim.task_id, + activity_execution_id: claim.activity_execution_id, activity_attempt_id: claim.activity_attempt_id, attempt_number: 1, + run_status: 'waiting', run_closed_reason: null, run_closed_at: null, activity_status: 'running', attempt_status: 'running', + task_status: 'leased', lease_owner: claim.lease_owner, ...fields}); + + // A live renewal: only the lease moves. + const before = state.status_before.response; + equal([state.status_before.status, before.can_continue], [200, true], 'the claim is live'); + equal(state.renewed.status, 200, 'renewal accepted'); + const {lease_expires_at: lease, server_time: serverTime, heartbeat_deadline_at: heartbeatDeadline, protocol_version: _p, + ...renewed} = state.renewed.response; + equal(renewed, {...attempt(live), last_heartbeat_at: null, lease_renewed: true, task_id: live.task_id, worker_session: null}, + 'renewal status'); + assert.match(lease, MICROS, 'lease in UTC microseconds'); + assert.match(serverTime, MICROS, 'server time in UTC microseconds'); + const window = (at(lease, 'lease') - at(serverTime, 'server time')) / 1000; + assert.ok(window > definition.lease_seconds - 1 && window <= definition.lease_seconds, + `the lease runs ${definition.lease_seconds} s from the renewal, not ${window}`); + assert.ok(at(lease, 'lease') > at(before.lease_expires_at, 'claim lease'), 'renewal extends the claim lease'); + equal(heartbeatDeadline, before.deadlines.heartbeat, 'renewal leaves the progress deadline'); + equal(at(state.status_after.response.lease_expires_at, 'status lease'), at(lease, 'lease'), 'status reads the renewed lease'); + equal(state.status_after.response.last_heartbeat_at, null, 'renewal records no heartbeat'); + equal(state.history_after, state.history_before, 'renewal adds no history'); + for (const key of ['renewed_on_protocol_1_19', 'extra_field']) { + equal([state[key].status, state[key].response.lease_renewed], [200, true], `${key} renews`); + } + + // Ownership, in PHP's order and shape, and Laravel's validation. + const refusal = (status, task, attempt, error, reason, extra = {}) => ({status, lease_renewal_capability: true, + response: {task_id: task, activity_attempt_id: attempt, error, reason, ...extra, protocol_version: '1.20'}}); + equal(state.foreign_owner, refusal(409, live.task_id, live.activity_attempt_id, 'Activity attempt lease is owned by another worker.', + 'lease_owner_mismatch', {lease_owner: live.lease_owner}), 'another owner'); + equal(state.unknown_attempt, refusal(404, live.task_id, 'no-such-attempt', 'Activity attempt not found.', 'attempt_not_found'), + 'unknown attempt'); + equal(state.unknown_task, refusal(404, 'no-such-task', live.activity_attempt_id, 'Activity task not found.', 'task_not_found'), + 'unknown task'); + equal(state.task_mismatch, refusal(409, live.workflow_task_id, live.activity_attempt_id, + 'Activity attempt is not leased for this task.', 'task_mismatch'), 'another task of the namespace'); + for (const key of ['missing_fields', 'non_string_fields']) { + equal(state[key], {status: 422, lease_renewal_capability: true, + response: {...definition.validation[key], protocol_version: '1.20'}}, key); + } + + // A heartbeat's progress is reported; renewal keeps its deadline. + equal([state.heartbeat.status, state.heartbeat.response.heartbeat_recorded], [200, true], 'heartbeat recorded'); + const beat = at(state.heartbeat.response.last_heartbeat_at, 'heartbeat'); + const afterBeat = state.renewed_after_heartbeat.response; + equal([state.renewed_after_heartbeat.status, afterBeat.lease_renewed, afterBeat.heartbeat_recorded], [200, true, false], + 'renewal after a heartbeat'); + equal(at(afterBeat.last_heartbeat_at, 'reported heartbeat'), beat, 'the recorded heartbeat is reported'); + equal(at(afterBeat.heartbeat_deadline_at, 'heartbeat deadline') - beat, definition.heartbeat_timeout * 1000, + 'the heartbeat keeps its own deadline'); + + // Closed attempts are refused with their status and no lease. + equal(state.completed.status, 200, 'activity completed'); + const {last_heartbeat_at: closedBeat, heartbeat_deadline_at: closedDeadline, protocol_version: _c, ...closed} = + state.renewed_after_completion.response; + equal(state.renewed_after_completion.status, 409, 'a completed attempt is refused'); + equal(closed, {...attempt(live, {can_continue: false, reason: 'attempt_closed', activity_status: 'completed', + attempt_status: 'completed', task_status: 'completed'}), lease_expires_at: null, lease_renewed: false}, 'completed attempt status'); + equal([at(closedBeat, 'closed heartbeat'), closedDeadline], [beat, afterBeat.heartbeat_deadline_at], 'closed attempt keeps its heartbeat'); + const stopped = (key, claim, command, runStatus, reason) => { + equal(state[command], 200, `${command} accepted`); + const {run_closed_at: closedAt, heartbeat_deadline_at: deadline, protocol_version: _s, ...body} = state[key].response; + equal(state[key].status, 409, `${key} refused`); + assert.match(closedAt, MICROS, `${key} close time in UTC microseconds`); + at(deadline, `${key} heartbeat deadline`); + const {run_closed_at: _r, ...expected} = attempt(claim, {can_continue: false, cancel_requested: true, reason, run_status: runStatus, + run_closed_reason: runStatus, activity_status: 'cancelled', attempt_status: 'cancelled', task_status: 'cancelled'}); + equal(body, {...expected, lease_expires_at: null, last_heartbeat_at: null, lease_renewed: false}, `${key} status`); + }; + stopped('renewed_after_cancel', state.cancelled_claim, 'cancel', 'cancelled', 'run_cancelled'); + stopped('renewed_after_terminate', state.terminated_claim, 'terminate', 'terminated', 'run_terminated'); + return projection; +} diff --git a/scripts/conformance/server-parity/activity-lease-renewal-probe.php b/scripts/conformance/server-parity/activity-lease-renewal-probe.php new file mode 100644 index 000000000..08a9bf8b0 --- /dev/null +++ b/scripts/conformance/server-parity/activity-lease-renewal-probe.php @@ -0,0 +1,111 @@ +createNamespace($namespace, 'activity lease renewal λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $http = new LeaseRenewalHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30]); + $raw = static function (string $method, string $path, ?array $body = [], string $protocol = '1.20') use ($http, $token, $namespace): array { + $options = ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => $protocol, + ])]; + if ($body !== null) { + $options['json'] = $body === [] ? (object) [] : $body; + } + $response = $http->request($method, $path, $options); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + $capability = is_array($decoded) && array_key_exists('server_capabilities', $decoded) + ? ($decoded['server_capabilities']['activity_lease_renewal'] ?? null) : 'absent'; + unset($decoded['server_capabilities'], $decoded['control_plane']); + + return ['status' => $response->getStatusCode(), 'response' => $decoded, 'lease_renewal_capability' => $capability]; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Activity lease renewal '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + // Avro payloads precomputed by the published serializer: [] and 'done'. + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $done = ['codec' => 'avro', 'blob' => $definition['result_blob']]; + // Starts a run on its own queue with its own worker and claims its single + // activity. + $claim = static function (string $suffix) use ($raw, $ok, $empty, $definition, $workflowId): array { + $id = $workflowId.'-'.$suffix; + $queue = $id.'-queue'; + $worker = $id.'-worker'; + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => [], + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])]), 201, 'registration'); + $poll = ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 0]; + $started = $ok($raw('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => $definition['workflow_type'], + 'task_queue' => $queue, 'input' => $empty]), 201, 'start'); + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $poll), 200, 'workflow claim')['task']; + $ok($raw('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $empty, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], + 'heartbeat_timeout' => $definition['heartbeat_timeout'], 'start_to_close_timeout' => 300]]]), 200, 'schedule activity'); + $activity = $ok($raw('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; + + return ['workflow_id' => $id, 'run_id' => $started['run_id'], 'workflow_task_id' => $task['task_id'], + 'task_id' => $activity['task_id'], 'activity_attempt_id' => $activity['activity_attempt_id'], + 'activity_execution_id' => $activity['activity_execution_id'] ?? null, 'lease_owner' => $activity['lease_owner']]; + }; + $own = static fn (array $claimed): array => ['activity_attempt_id' => $claimed['activity_attempt_id'], 'lease_owner' => $claimed['lease_owner']]; + $renew = static fn (array $claimed, ?array $body = null, string $protocol = '1.20'): array => $raw('POST', + '/api/worker/activity-tasks/'.$claimed['task_id'].'/renew-lease', $body ?? $own($claimed), $protocol); + $status = static fn (array $claimed): array => $raw('POST', '/api/worker/activity-tasks/'.$claimed['task_id'].'/status', $own($claimed)); + $history = static fn (array $claimed): array => array_column($ok($raw('GET', + '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/history', null), 200, 'history')['events'] ?? [], 'event_type'); + + $state = ['applicable' => true, 'namespace' => $namespace]; + $live = $state['live'] = $claim('live'); + $state['status_before'] = $status($live); + $state['history_before'] = $history($live); + usleep(20000); + $state['renewed'] = $renew($live); + $state['status_after'] = $status($live); + $state['history_after'] = $history($live); + $state['renewed_on_protocol_1_19'] = $renew($live, null, '1.19'); + $state['extra_field'] = $renew($live, $own($live) + ['message' => 'progress']); + $state['foreign_owner'] = $renew($live, ['activity_attempt_id' => $live['activity_attempt_id'], 'lease_owner' => 'someone-else']); + $state['unknown_attempt'] = $renew($live, ['activity_attempt_id' => 'no-such-attempt', 'lease_owner' => $live['lease_owner']]); + $state['unknown_task'] = $raw('POST', '/api/worker/activity-tasks/no-such-task/renew-lease', $own($live)); + $state['task_mismatch'] = $raw('POST', '/api/worker/activity-tasks/'.$live['workflow_task_id'].'/renew-lease', $own($live)); + $state['missing_fields'] = $renew($live, []); + $state['non_string_fields'] = $renew($live, ['activity_attempt_id' => 5, 'lease_owner' => ['x']]); + $state['heartbeat'] = $raw('POST', '/api/worker/activity-tasks/'.$live['task_id'].'/heartbeat', $own($live) + ['message' => 'half']); + $state['renewed_after_heartbeat'] = $renew($live); + $state['completed'] = $raw('POST', '/api/worker/activity-tasks/'.$live['task_id'].'/complete', $own($live) + ['result' => $done]); + $state['renewed_after_completion'] = $renew($live); + + $cancelled = $state['cancelled_claim'] = $claim('cancelled'); + $state['cancel'] = $raw('POST', '/api/workflows/'.$cancelled['workflow_id'].'/cancel', ['reason' => 'lease renewal fixture'])['status']; + $state['renewed_after_cancel'] = $renew($cancelled); + $terminated = $state['terminated_claim'] = $claim('terminated'); + $state['terminate'] = $raw('POST', '/api/workflows/'.$terminated['workflow_id'].'/terminate', ['reason' => 'lease renewal fixture'])['status']; + $state['renewed_after_terminate'] = $renew($terminated); + + return $state; +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index a181ab10d..794ad66a9 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -12,6 +12,9 @@ import {checkWorkerDeregistration} from './worker-deregistration-contract.mjs'; import {checkLateWorkflowCompletion} from './late-workflow-completion-contract.mjs'; import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; +import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; +import {checkServiceCatalog} from './service-catalog-contract.mjs'; +import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -141,6 +144,9 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const lateWorkflowCompletion = fixture.late_workflow_completion ? checkLateWorkflowCompletion(fixture, observation, workflowId) : null; const searchAttributes = fixture.search_attributes ? checkSearchAttributes(fixture, observation, workflowId) : null; const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; + const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; + const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; + const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -687,6 +693,9 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(lateWorkflowCompletion ? {late_workflow_completion: lateWorkflowCompletion} : {}), ...(searchAttributes ? {search_attributes: searchAttributes} : {}), ...(storageTest ? {storage_test: storageTest} : {}), + ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), + ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), + ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/lock-pressure-transport.php b/scripts/conformance/server-parity/lock-pressure-transport.php index bdb4b696e..a0bc0d3bb 100644 --- a/scripts/conformance/server-parity/lock-pressure-transport.php +++ b/scripts/conformance/server-parity/lock-pressure-transport.php @@ -9,8 +9,11 @@ require_once __DIR__.'/lock-pressure-retry.php'; /** - * Applies the documented worker-poll lock-pressure retry to an SDK transport and - * records each retry so evidence shows when a reference backend was contended. + * Applies the documented lock-pressure retries to an SDK transport and records + * each retry so evidence shows when a reference backend was contended. A poll + * is retried only when no task was handed out. Any other request is retried + * only when the server marked its refusal retryable, so nothing was committed + * (see lockPressureOperationRetryDelay). */ final class LockPressurePollRetryTransport implements Transport { @@ -31,7 +34,7 @@ public function send(string $method, string $uri, array $headers, ?array $body = try { return $this->inner->send($method, $uri, $headers, $body); } catch (TransportException $exception) { - $delay = lockPressurePollRetryDelay($method, $uri, $exception->status, $exception->response, + $delay = lockPressureOperationRetryDelay($method, $uri, $exception->status, $exception->response, count($this->retries), $waited); if ($delay === null) { throw $exception; diff --git a/scripts/conformance/server-parity/namespace-delete-contract.mjs b/scripts/conformance/server-parity/namespace-delete-contract.mjs new file mode 100644 index 000000000..b7cb741e1 --- /dev/null +++ b/scripts/conformance/server-parity/namespace-delete-contract.mjs @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; + +// Reviewed namespace deletion (NamespaceController::destroy over +// NamespaceLifecycleCleanup). The namespace holds one search attribute +// definition and one started, unclaimed run. The deleted counts are exact, +// except PHP's timeline, timer and lineage projections, which PHP fills +// asynchronously: they may be absent or positive. Embedded applications have +// no HTTP namespaces, so the projection is the definition alone. +export function checkNamespaceDelete(fixture, observation, workflowId) { + const definition = fixture.namespace_delete; + const state = observation.namespace_delete; + const projection = {scope: 'namespace_delete', deleted: definition.deleted}; + assert.ok(state && typeof state === 'object', 'actual namespace delete observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_namespaces'}, + 'embedded namespace delete explicitly inapplicable'); + return projection; + } + const equal = (actual, expected, label) => assert.deepStrictEqual(actual, expected, label); + const namespace = `${workflowId}-delete`.toLowerCase(); + equal([state.applicable, state.namespace, state.created?.name, state.created?.status], [true, namespace, namespace, 'active'], + 'namespace created'); + equal([state.search_attribute, state.started], [201, 'pending'], 'definition and run in the namespace'); + + equal(state.deleted.status, 200, 'namespace deleted'); + const {deleted, ...response} = state.deleted.response; + equal(response, {name: namespace, status: 'deleted'}, 'delete response'); + const exact = Object.fromEntries(Object.entries(deleted).filter(([key]) => !definition.asynchronous_projections.includes(key))); + equal(exact, definition.deleted, 'each owned table deleted with its count'); + for (const key of definition.asynchronous_projections) { + assert.ok(!(key in deleted) || (Number.isInteger(deleted[key]) && deleted[key] > 0), `${key} absent or positive`); + } + + const gone = {status: 404, response: {message: `Namespace [${namespace}] not found.`, reason: 'namespace_not_found', namespace}}; + equal(state.described_after, gone, 'a deleted namespace reads as not found'); + equal(state.deleted_again, gone, 'a deleted namespace cannot be deleted again'); + equal(state.sdk_delete_again, {class: 'DurableWorkflow\\Exception\\ServerException', status: 404, reason: 'namespace_not_found'}, + 'the published SDK refuses a second delete'); + equal(state.own_run, workflowId, 'a run in another namespace remains'); + equal(state.restarted, `${workflowId}-doomed`, "the deleted run's workflow ID starts a new run"); + return projection; +} diff --git a/scripts/conformance/server-parity/namespace-delete-probe.php b/scripts/conformance/server-parity/namespace-delete-probe.php new file mode 100644 index 000000000..6de16961f --- /dev/null +++ b/scripts/conformance/server-parity/namespace-delete-probe.php @@ -0,0 +1,58 @@ +push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new NamespaceDeleteHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, string $scope, ?array $body = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $scope, + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), ...($body === null ? [] : ['json' => $body])]); + + return ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + $state = ['applicable' => true, 'namespace' => $namespace]; + $state['created'] = $admin->createNamespace($namespace, $definition['description'], 30)->raw; + $state['search_attribute'] = $raw('POST', 'search-attributes', $namespace, $definition['search_attribute'])['status']; + $scoped = parityClient($url, namespace: $namespace, token: $token); + $state['started'] = $scoped->startWorkflow($fixture['workflow_type'], $doomed, $queue.'-delete', [$fixture['input']]) + ->describeSelectedRun()->raw['status'] ?? null; + // The delete itself over HTTP, so its status is recorded too. + $state['deleted'] = $raw('DELETE', 'namespaces/'.$namespace, 'default'); + $state['described_after'] = $raw('GET', 'namespaces/'.$namespace, 'default'); + $state['deleted_again'] = $raw('DELETE', 'namespaces/'.$namespace, 'default'); + // The published SDK's typed refusal for a namespace that is gone. + try { + $admin->deleteNamespace($namespace); + $state['sdk_delete_again'] = ['accepted' => true]; + } catch (ServerException $error) { + $state['sdk_delete_again'] = ['class' => get_class($error), 'status' => $error->status, 'reason' => $error->reason]; + } + $state['own_run'] = $admin->describeWorkflow($workflowId)->raw['workflow_id'] ?? null; + $state['restarted'] = $admin->startWorkflow($fixture['workflow_type'], $doomed, $queue.'-delete', [$fixture['input']]) + ->describeSelectedRun()->raw['workflow_id'] ?? null; + + return $state; +} diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index df4481841..69df4275e 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import {isDeepStrictEqual} from 'node:util'; // Reviewed operator metrics and health of one namespace, as published PHP // 2.5.14 reports them through a run's stages. Counts must match exactly, with @@ -75,6 +76,12 @@ const EXPECTED = { const REPAIRED = {...EXPECTED.waiting_after_redispatch, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1}}; const REPAIRED_WARNINGS = ['command_contract_snapshots']; +// PHP's TaskWatchdog repairs the task and projects the run before +// TaskDispatcher redispatches it and projects the run again. Until then the +// task is still overdue and the run reads as needing repair. +const REPAIRING = {...EXPECTED.waiting_after_redispatch, runs: {...waitingRun, repair_needed: 1}, + backlog: {...EXPECTED.waiting_after_redispatch.backlog, repair_needed_runs: 1}}; +const REPAIRING_WARNINGS = ['command_contract_snapshots', 'task_transport', 'routing_health', 'durable_resume_paths']; // Checks other than ok at each stage (the uncompared check aside). const WARNINGS = { empty: [], @@ -86,6 +93,16 @@ const WARNINGS = { completed: [], }; +// Past the redispatch window published PHP is in one of these states, in +// order: overdue, repairing, or redispatched. Metrics and health are separate +// requests, so its health may be at the state its metrics show or a later one. +// Native runs no repair loop and is held to the state its metrics show. +const REPAIR_STATES = [ + {counts: EXPECTED.waiting_after_redispatch, warnings: WARNINGS.waiting_after_redispatch}, + {counts: REPAIRING, warnings: REPAIRING_WARNINGS}, + {counts: REPAIRED, warnings: REPAIRED_WARNINGS}, +]; + function normalized(sections) { const counts = {}; for (const [name, section] of Object.entries(sections)) { @@ -134,14 +151,19 @@ function checkStage(name, stage, namespace, family) { assert.deepStrictEqual([raw.tasks.counts.ready_due, raw.starts.counts.ready_tasks], [1, 1], `${label} due task`); } const counts = normalized(raw); - const repaired = name === 'waiting_after_redispatch' && counts.tasks.max_repair_count > 0; + const redispatch = name === 'waiting_after_redispatch'; + const php = redispatch && stage.runtime === 'php'; + const repaired = redispatch && counts.tasks.max_repair_count > 0; if (repaired) delete counts.tasks.max_repair_count; - const expectedCounts = repaired ? REPAIRED : EXPECTED[name]; - const expectedWarnings = repaired ? REPAIRED_WARNINGS : WARNINGS[name]; + const states = redispatch ? REPAIR_STATES : [{counts: EXPECTED[name], warnings: WARNINGS[name]}]; + // A repair count means PHP repaired the task: mid-repair (PHP only) or redispatched. + const possible = !repaired ? [0] : php ? [1, 2] : [2]; + const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); + const matches = index => Object.entries(counts) + .every(([section, values]) => isDeepStrictEqual(nonzero(values), states[index].counts[section] ?? {})); + const state = possible.find(matches) ?? possible[possible.length - 1]; for (const [section, values] of Object.entries(counts)) { - const expected = expectedCounts[section] ?? {}; - const nonzero = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); - assert.deepStrictEqual(nonzero, expected, `${label} ${section} counts`); + assert.deepStrictEqual(nonzero(values), states[state].counts[section] ?? {}, `${label} ${section} counts`); } assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, `${label} active workers`); @@ -149,9 +171,11 @@ function checkStage(name, stage, namespace, family) { assert.deepStrictEqual(stage.checks.map(check => [check.name, check.category]), CHECKS, `${label} check names`); const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); - assert.deepStrictEqual(statuses.filter(check => check.status !== 'ok').map(check => check.name), expectedWarnings, - `${label} warnings`); - const warning = expectedWarnings.length > 0 || stage.checks.some(check => check.status === 'warning'); + const warnings = statuses.filter(check => check.status !== 'ok').map(check => check.name); + const allowed = (php ? states.slice(state) : [states[state]]).map(candidate => candidate.warnings); + const settled = allowed.find(list => isDeepStrictEqual(list, warnings)) ?? states[state].warnings; + assert.deepStrictEqual(warnings, settled, `${label} warnings`); + const warning = settled.length > 0 || stage.checks.some(check => check.status === 'warning'); assert.deepStrictEqual([stage.health_status, stage.health.status, stage.health.healthy], [200, warning ? 'warning' : 'ok', true], `${label} health`); assert.deepStrictEqual(stage.retention, ['bounded', 30], `${label} namespace retention`); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 030b34773..e89a0cd8b 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -192,8 +192,11 @@ function patchPublishedSdkPhase(array $fixture, array $options): array if (isset($fixture['patch_deployment']['legacy_marker_history'])) { $proxyArguments[] = '--forwarding-state'; } + // The published worker deregisters at shutdown without retrying; let the + // observer retry the SQLite reference's documented retryable refusal. $proxy = proc_open($proxyArguments, - [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $proxyPipes); + [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $proxyPipes, null, + [...getenv(), 'DW_PARITY_PROXY_RETRY_LOCK_PRESSURE' => '1']); if (! is_resource($proxy)) { unlink($journal); throw new RuntimeException('Could not start the SDK HTTP observer.'); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 0b40d67a2..5ccb00c47 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -430,6 +430,9 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $lateWorkflowCompletion = isset($fixture['late_workflow_completion']) ? finishHttpLateWorkflowCompletion($client, $fixture, $workflowId) : null; $searchAttributes = isset($fixture['search_attributes']) ? finishHttpSearchAttributes($client, $fixture, $workflowId, $url) : null; $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; + $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; + $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; + $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -529,6 +532,9 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'late_workflow_completion' => $lateWorkflowCompletion, 'search_attributes' => $searchAttributes, 'storage_test' => $storageTest, + 'activity_lease_renewal' => $activityLeaseRenewal, + 'service_catalog' => $serviceCatalog, + 'namespace_delete' => $namespaceDelete, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -832,6 +838,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/namespace-probe.php'; require __DIR__.'/search-attribute-probe.php'; require __DIR__.'/storage-test-probe.php'; + require __DIR__.'/activity-lease-renewal-probe.php'; + require __DIR__.'/service-catalog-probe.php'; + require __DIR__.'/namespace-delete-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -990,6 +999,15 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['storage_test'])) { $observation['storage_test'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_storage_diagnostic']; } + if ($mode === 'embedded' && isset($fixture['activity_lease_renewal'])) { + $observation['activity_lease_renewal'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } + if ($mode === 'embedded' && isset($fixture['service_catalog'])) { + $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; + } + if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { + $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/published-sdk-patch/observe-http.mjs b/scripts/conformance/server-parity/published-sdk-patch/observe-http.mjs index 1e6d81984..a21d33b70 100644 --- a/scripts/conformance/server-parity/published-sdk-patch/observe-http.mjs +++ b/scripts/conformance/server-parity/published-sdk-patch/observe-http.mjs @@ -13,6 +13,23 @@ if (upstream.protocol !== 'http:') throw new Error('Isolated fixture target must const arrivalJournal = process.env.DW_PARITY_HTTP_REQUEST_ARRIVALS; const captureNamespace = process.env.DW_PARITY_HTTP_CAPTURE_NAMESPACE === '1'; const captureTiming = process.env.DW_PARITY_HTTP_CAPTURE_TIMING === '1'; +// Opt-in: retry the SQLite reference's documented retryable operation refusal +// (503 backend_lock_pressure, retryable, nothing recorded) below the SDK, as +// lockPressureOperationRetryDelay does for PHP clients. Polls are never +// retried here, so their pressure stays visible. Only the final reply is +// recorded and returned. +const retryLockPressure = process.env.DW_PARITY_PROXY_RETRY_LOCK_PRESSURE === '1'; +const lockPressureRetryDelay = (method, path, status, text, retries, waited) => { + if (!retryLockPressure || status !== 503 || path.endsWith('/poll') || retries >= 15) return null; + let reply; + try { reply = JSON.parse(text); } catch { return null; } + if (!reply || reply.reason !== 'backend_lock_pressure' || reply.retryable !== true + || (Object.hasOwn(reply, 'recorded') && reply.recorded !== false)) return null; + const hint = reply.retry_after_seconds ?? 1; + if (typeof hint !== 'number') return null; + const delay = Math.min(Math.max(hint, 0.1), 5); + return waited + delay <= 30 ? delay : null; +}; if (arrivalJournal !== undefined && (!arrivalJournal || observationMode !== '--forwarding-state')) { throw new Error('Request-arrival capture requires a journal and forwarding-state mode'); } @@ -65,7 +82,7 @@ const server = http.createServer((request, response) => { if (arrivalLine && (arrivalCount >= 512 || arrivalBytes + arrivalSize > 4 * 1024 * 1024)) { throw new Error('Request-arrival journal exceeds its bounded fixture budget'); } - forwarded = proxyHttpRequest(request.url, upstream, { + const send = (retries, waited) => { forwarded = proxyHttpRequest(request.url, upstream, { method: request.method, headers: {...request.headers, host: upstream.host}, }, incoming => { @@ -78,6 +95,12 @@ const server = http.createServer((request, response) => { const decoders = {identity: value => value, gzip: gunzipSync, deflate: inflateSync, br: brotliDecompressSync}; if (!decoders[encoding]) throw new Error(`Unsupported observed content encoding: ${encoding}`); const decoded = decoders[encoding](bytes).toString('utf8'); + const delay = lockPressureRetryDelay(request.method, proxyTargetPath(request.url).pathname, incoming.statusCode, + decoded, retries, waited); + if (delay !== null) { + setTimeout(() => { if (!recorded) send(retries + 1, waited + delay); }, delay * 1000); + return; + } record({status: incoming.statusCode, response_encoding: encoding, response_retry_after: incoming.headers['retry-after'] ?? null, response_bytes_base64: bytes.toString('base64'), response_body: decoded}); @@ -98,7 +121,8 @@ const server = http.createServer((request, response) => { incoming.on('error', failed); }); forwarded.on('error', failed); - forwarded.end(body); + forwarded.end(body); }; + send(0, 0); if (arrivalLine) { appendFileSync(arrivalJournal, arrivalLine); arrivalCount++; diff --git a/scripts/conformance/server-parity/role-profile.sh b/scripts/conformance/server-parity/role-profile.sh index 43119caaf..0fa79101c 100644 --- a/scripts/conformance/server-parity/role-profile.sh +++ b/scripts/conformance/server-parity/role-profile.sh @@ -8,7 +8,7 @@ test -n "$RESOURCE_SCOPE" scope_identity="$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$GITHUB_JOB" case "$RESOURCE_SCOPE" in *"$scope_identity"*) ;; *) exit 2 ;; esac backend="$1" -image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json) +image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) rust_image=mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 fixture=tests/Fixtures/ServerParityProfiles/role-tokens/admission-role-tokens.json credentials=(-e DW_AUTH_TOKEN=parity-role-legacy -e DW_WORKER_TOKEN=parity-role-worker diff --git a/scripts/conformance/server-parity/schedule-catalog-contract.mjs b/scripts/conformance/server-parity/schedule-catalog-contract.mjs index 88e1d7abd..1a82da941 100644 --- a/scripts/conformance/server-parity/schedule-catalog-contract.mjs +++ b/scripts/conformance/server-parity/schedule-catalog-contract.mjs @@ -34,7 +34,7 @@ export function checkScheduleCatalog(fixture, observation, workflowId) { 'embedded schedule surface explicitly inapplicable'); return {scope: 'schedule_catalog'}; } - assert.ok(Number.isInteger(state.attempts) && state.attempts >= 1 && state.attempts <= 3, 'recorded sequence attempts'); + assert.ok(Number.isInteger(state.attempts) && state.attempts >= 1 && state.attempts <= 5, 'recorded sequence attempts'); // A sequence replayed after PHP's SQLite lock defect runs in a new namespace. const namespace = `${workflowId}-catalog`.toLowerCase() + (state.attempts === 1 ? '' : `-r${state.attempts}`); assert.deepStrictEqual([state.applicable, state.namespace], [true, namespace], 'schedule catalog namespace'); diff --git a/scripts/conformance/server-parity/schedule-policies-contract.mjs b/scripts/conformance/server-parity/schedule-policies-contract.mjs index 3f8e42905..3fbfa6898 100644 --- a/scripts/conformance/server-parity/schedule-policies-contract.mjs +++ b/scripts/conformance/server-parity/schedule-policies-contract.mjs @@ -41,6 +41,13 @@ const POLICY_OUTCOMES = { terminate_other: ['triggered', 'triggered', 'triggered'], }; +// Each step whose status differs, with its response, so a failed run names +// the cause instead of only the position. +function statusDifferences(steps) { + return steps.flatMap((step, index) => step.status === STATUSES[index] ? [] : + [`step ${index} ${step.request}: ${step.status}, expected ${STATUSES[index]}: ${JSON.stringify(step.response).slice(0, 400)}`]); +} + export function checkSchedulePolicies(fixture, observation, workflowId) { const state = observation.schedule_policies; assert.ok(state && typeof state === 'object', 'actual schedule policy observation'); @@ -49,7 +56,7 @@ export function checkSchedulePolicies(fixture, observation, workflowId) { 'embedded schedule surface explicitly inapplicable'); return {scope: 'schedule_policies'}; } - assert.ok(Number.isInteger(state.attempts) && state.attempts >= 1 && state.attempts <= 3, 'recorded sequence attempts'); + assert.ok(Number.isInteger(state.attempts) && state.attempts >= 1 && state.attempts <= 5, 'recorded sequence attempts'); // A sequence replayed after PHP's SQLite lock defect runs in a new namespace. const namespace = `${workflowId}-schedules`.toLowerCase() + (state.attempts === 1 ? '' : `-r${state.attempts}`); assert.deepStrictEqual([state.applicable, state.namespace], [true, namespace], 'schedule policy namespace'); @@ -57,7 +64,8 @@ export function checkSchedulePolicies(fixture, observation, workflowId) { const raw = state.steps; const names = {[namespace]: '@namespace', [state.queue]: '@queue'}; const steps = raw.map(step => ({request: step.request, status: step.status, response: normalized(step.response, names)})); - assert.deepStrictEqual(steps.map(step => step.status), STATUSES, 'schedule step statuses'); + assert.deepStrictEqual(steps.map(step => step.status), STATUSES, + `schedule step statuses (${steps.length} of ${STATUSES.length} steps)\n${statusDifferences(steps).join('\n')}`); const at = request => steps.filter(step => step.request === request); // Laravel names expanded attributes raw and others with spaces. assert.equal(steps[1].response.message, 'The spec.timezone field must be a string. (and 9 more errors)', 'validation summary'); @@ -79,8 +87,9 @@ export function checkSchedulePolicies(fixture, observation, workflowId) { execution_timeout_seconds: 90, run_timeout_seconds: 30}, 'action timeout aliases'); for (const [policy, outcomes] of Object.entries(POLICY_OUTCOMES)) { const id = `policy-${policy.replaceAll('_', '-')}`; - assert.deepStrictEqual(at(`POST schedules/${id}/trigger`).slice(0, 3).map(step => step.response.outcome), outcomes, - `${policy} trigger outcomes`); + const triggers = at(`POST schedules/${id}/trigger`).slice(0, 3).map(step => step.response); + assert.deepStrictEqual(triggers.map(response => response.outcome), outcomes, + `${policy} trigger outcomes ${JSON.stringify(triggers.map(response => response.outcome))}, expected ${JSON.stringify(outcomes)}: ${JSON.stringify(triggers).slice(0, 600)}`); } const quota = at('POST schedules/quota/trigger'); assert.deepStrictEqual(quota.map(step => step.response.outcome), ['triggered', 'triggered'], 'two quota runs'); diff --git a/scripts/conformance/server-parity/schedule-policies-probe.php b/scripts/conformance/server-parity/schedule-policies-probe.php index 511ecfbe6..e511fbf0f 100644 --- a/scripts/conformance/server-parity/schedule-policies-probe.php +++ b/scripts/conformance/server-parity/schedule-policies-probe.php @@ -12,10 +12,13 @@ // the contract normalizes generated identities and times. // // Published PHP on SQLite can lose a write race with its own maintenance -// roles ("database is locked"). A 500 for that rolls the whole operation -// back, so the step is sent again. A backfill instead records the lost -// occurrence as a failure and goes on, which changes the schedule, so the -// whole sequence is replayed once more in a new namespace. Each is recorded. +// roles: "database is locked", or from Server 2.10 the structured, retryable +// 503 backend_lock_pressure refusal. A read is simply sent again. A write may +// already have committed when it reports the lock, so it is sent again only +// when its schedule reads the same as before it; otherwise, and when a +// backfill records a lost occurrence as a failure, the whole sequence is +// replayed in a new namespace. Native never reports this lock, so it never +// retries or replays. Each is recorded. function finishHttpSchedulePolicies(Client $admin, array $fixture, string $workflowId, string $url, string $key = 'schedule_policies'): array { $definition = $fixture[$key]; @@ -40,16 +43,47 @@ function finishHttpSchedulePolicies(Client $admin, array $fixture, string $workf $namespace = $attempt === 1 ? $base : "$base-r$attempt"; $admin->createNamespace($namespace, 'schedule policies λ', 30); $pageToken = ''; - $send = static function (string $method, string $path, mixed $body = null) use ($http, $token, $namespace, &$retries, &$pageToken): array { + $request = static function (string $method, string $target, mixed $body = null) use ($http, $token, $namespace): array { + $response = $http->request($method, $target, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), ...($body === null ? [] : ['json' => $body === [] ? (object) [] : $body])]); + + return [$response->getStatusCode(), (string) $response->getBody()]; + }; + $lockFailure = static fn (int $status, string $text): bool => $status === 500 && str_contains($text, 'database is locked') + || $status === 503 && str_contains($text, '"reason":"backend_lock_pressure"'); + // The schedule as it reads now, to tell a lock failure that rolled + // back (unchanged) from one reported after its commit. + $snapshot = static function (?string $schedule) use ($request, $lockFailure, &$retries): ?string { + for ($try = 1; $schedule !== null && $try <= 5; $try++) { + [$status, $text] = $request('GET', 'schedules/'.rawurlencode($schedule)); + if (! $lockFailure($status, $text)) { + return "$status $text"; + } + $retries++; + usleep(200000 * $try); + } + + return null; + }; + $send = static function (string $method, string $path, mixed $body = null) use ($request, $lockFailure, $snapshot, &$retries, &$pageToken): array { $target = str_replace('{token}', rawurlencode($pageToken), $path); + $schedule = null; + if ($method !== 'GET') { + $schedule = preg_match('#^schedules/([^/?]+)#', $path, $match) === 1 + ? rawurldecode($match[1]) + : (is_array($body) && is_string($body['schedule_id'] ?? null) ? $body['schedule_id'] : null); + } + $before = $method === 'GET' ? null : $snapshot($schedule); for ($try = 1; ; $try++) { - $response = $http->request($method, $target, ['headers' => array_filter([ - 'Authorization' => $token === null ? null : 'Bearer '.$token, - 'Accept' => 'application/json', 'X-Namespace' => $namespace, - 'X-Durable-Workflow-Control-Plane-Version' => '2', - ]), ...($body === null ? [] : ['json' => $body === [] ? (object) [] : $body])]); - $text = (string) $response->getBody(); - if ($response->getStatusCode() === 500 && str_contains($text, 'database is locked') && $try < 5) { + [$status, $text] = $request($method, $target, $body); + // A read is sent again; a write only when the schedule shows + // it rolled back. Otherwise the failure is recorded, and the + // recorded lock replays the sequence. + if ($lockFailure($status, $text) && $try < 5 + && ($method === 'GET' || ($before !== null && $snapshot($schedule) === $before))) { $retries++; usleep(200000 * $try); @@ -60,34 +94,43 @@ function finishHttpSchedulePolicies(Client $admin, array $fixture, string $workf $pageToken = $decoded['next_page_token']; } - return ['request' => "$method $path", 'status' => $response->getStatusCode(), 'response' => $decoded]; + return ['request' => "$method $path", 'status' => $status, 'response' => $decoded]; } }; $steps = []; $locked = false; + // Records a step; true once a response reports the lock. + $record = static function (array $step) use (&$steps, &$locked): bool { + $steps[] = $step; + $text = json_encode($step['response']); + $locked = str_contains($text, 'database is locked') || str_contains($text, '"reason":"backend_lock_pressure"'); + + return $locked; + }; foreach ($definition['steps'] as $step) { if ($step[0] === '@policy') { $policy = $step[1]; $id = 'policy-'.str_replace('_', '-', $policy); - $steps[] = $send('POST', 'schedules', ['schedule_id' => $id, 'spec' => ['cron_expressions' => ['0 0 1 1 *']], + $requests = [['POST', 'schedules', ['schedule_id' => $id, 'spec' => ['cron_expressions' => ['0 0 1 1 *']], 'action' => ['workflow_type' => $fixture['workflow_type'], 'task_queue' => $queue, 'input' => $input], - 'overlap_policy' => $policy]); + 'overlap_policy' => $policy]]]; for ($trigger = 0; $trigger < 3; $trigger++) { - $steps[] = $send('POST', "schedules/$id/trigger", []); + $requests[] = ['POST', "schedules/$id/trigger", []]; + } + $requests[] = ['GET', "schedules/$id", null]; + foreach ($requests as [$method, $path, $body]) { + if ($record($send($method, $path, $body))) { + break 2; + } } - $steps[] = $send('GET', "schedules/$id"); continue; } - $steps[] = $send($step[0], $step[1], array_key_exists(2, $step) ? $fill($step[2]) : null); - $last = end($steps); - if ($last['status'] !== 500 && str_contains(json_encode($last['response']), 'database is locked')) { - $locked = true; - + if ($record($send($step[0], $step[1], array_key_exists(2, $step) ? $fill($step[2]) : null))) { break; } } - if (! $locked || $attempt === 3) { + if (! $locked || $attempt === 5) { return ['applicable' => true, 'namespace' => $namespace, 'queue' => $queue, 'retries' => $retries, 'attempts' => $attempt, 'steps' => $steps]; } diff --git a/scripts/conformance/server-parity/sdk-query-direction-contract.mjs b/scripts/conformance/server-parity/sdk-query-direction-contract.mjs index fbb8fabe1..731b6051e 100644 --- a/scripts/conformance/server-parity/sdk-query-direction-contract.mjs +++ b/scripts/conformance/server-parity/sdk-query-direction-contract.mjs @@ -42,11 +42,16 @@ function unchangedDurableSnapshot(before, after, message) { const hasAvailability = snapshot => Object.hasOwn(snapshot.execution?.actions ?? {}, 'can_query'); assert.equal(hasAvailability(after), hasAvailability(before), 'computed query availability field stays present'); const durable = snapshot => { - if (!hasAvailability(snapshot)) return snapshot; - assert.equal(typeof snapshot.execution.actions.can_query, 'boolean', 'actual computed query availability'); - const actions = {...snapshot.execution.actions}; + // definition_routing (Server 2.10.0 and native) reports how the next ready + // workflow task would route now, so worker presence changes it without + // touching the run or its history. + const {definition_routing: _routing, ...execution} = snapshot.execution ?? {}; + const stable = snapshot.execution ? {...snapshot, execution} : snapshot; + if (!hasAvailability(stable)) return stable; + assert.equal(typeof stable.execution.actions.can_query, 'boolean', 'actual computed query availability'); + const actions = {...stable.execution.actions}; delete actions.can_query; // Only live worker availability; complete raw value stays recorded. - return {...snapshot, execution: {...snapshot.execution, actions}}; + return {...stable, execution: {...stable.execution, actions}}; }; unchanged(durable(before), durable(after), message); } diff --git a/scripts/conformance/server-parity/service-catalog-contract.mjs b/scripts/conformance/server-parity/service-catalog-contract.mjs new file mode 100644 index 000000000..d53d72bb6 --- /dev/null +++ b/scripts/conformance/server-parity/service-catalog-contract.mjs @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; + +// Reviewed service catalog reads (Server 2.11's ServiceCatalogReadController) +// in a namespace with no catalog objects. Every response is exact; "@own" +// stands for the fixture's own namespace. Embedded applications have no HTTP +// catalog. +export function checkServiceCatalog(fixture, observation, workflowId) { + const definition = fixture.service_catalog; + const state = observation.service_catalog; + const projection = {scope: 'service_catalog', cases: definition.requests.length}; + assert.ok(state && typeof state === 'object', 'actual service catalog observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_service_catalog'}, + 'embedded service catalog explicitly inapplicable'); + return projection; + } + const own = `${workflowId}-catalog`.toLowerCase(); + assert.deepStrictEqual([state.applicable, state.namespace], [true, own], 'catalog namespace'); + const labels = definition.requests.map(request => request.label); + assert.deepStrictEqual(Object.keys(state.responses ?? {}).sort(), [...labels].sort(), 'every catalog request answered'); + const substitute = value => JSON.parse(JSON.stringify(value).split('"@own"').join(JSON.stringify(own))); + for (const label of labels) { + assert.deepStrictEqual(state.responses[label], substitute(definition.expected[label]), label); + } + return projection; +} diff --git a/scripts/conformance/server-parity/service-catalog-probe.php b/scripts/conformance/server-parity/service-catalog-probe.php new file mode 100644 index 000000000..e46ee0942 --- /dev/null +++ b/scripts/conformance/server-parity/service-catalog-probe.php @@ -0,0 +1,36 @@ +createNamespace($own, 'service catalog λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $http = new ServiceCatalogHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30]); + $state = ['applicable' => true, 'namespace' => $own, 'responses' => []]; + foreach ($definition['requests'] as $request) { + $namespace = match ($request['namespace']) { + '@own' => $own, + '@OWN' => strtoupper($own), + default => $request['namespace'], + }; + $response = $http->get(str_replace('@own', $own, $request['path']), ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => $request['version'] ? '2' : null, + ], static fn (?string $value): bool => $value !== null)]); + $state['responses'][$request['label']] = [$response->getStatusCode(), + json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + } + + return $state; +} diff --git a/scripts/conformance/server-parity/signals-before-worker-contract.mjs b/scripts/conformance/server-parity/signals-before-worker-contract.mjs index a27235d75..957b5d84d 100644 --- a/scripts/conformance/server-parity/signals-before-worker-contract.mjs +++ b/scripts/conformance/server-parity/signals-before-worker-contract.mjs @@ -19,8 +19,10 @@ export function normalized(value, names = {}) { } export const STATUSES = [201, 202, 202, 202, 202, 422, 200, 409, 422, 404, 200]; +// From Server 2.10, signals that a run closed before applying are recorded as +// rejected after its terminal event. const EVENTS = ['StartAccepted', 'WorkflowStarted', 'SignalReceived', 'SignalReceived', 'SignalReceived', 'SignalReceived', - 'TerminateRequested', 'WorkflowTerminated']; + 'TerminateRequested', 'WorkflowTerminated', 'SignalRejected', 'SignalRejected', 'SignalRejected', 'SignalRejected']; export function checkSignalsBeforeWorker(fixture, observation, workflowId) { const state = observation.signals_before_worker; diff --git a/scripts/conformance/server-parity/worker-deregistration-contract.mjs b/scripts/conformance/server-parity/worker-deregistration-contract.mjs index 0c684209e..6cddf091b 100644 --- a/scripts/conformance/server-parity/worker-deregistration-contract.mjs +++ b/scripts/conformance/server-parity/worker-deregistration-contract.mjs @@ -2,6 +2,16 @@ import assert from 'node:assert/strict'; import {checkSdkDatabasePressure} from './worker-pressure-contract.mjs'; import {checkSdkPersistentPressure} from './worker-persistent-contract.mjs'; +// A run read without `definition_routing`: that block (Server 2.10.0 and +// native) reports how the next ready workflow task would route now, so a +// lease release or a registration change alters it without touching the +// run or its history. +const durable = run => { + if (!run?.execution?.definition_routing) return run; + const {definition_routing: _routing, ...execution} = run.execution; + return {...run, execution}; +}; + export function checkWorkerDeregistration(fixture, observation, workflowId) { const definition = fixture.worker_deregistration; const state = observation.worker_deregistration; @@ -131,7 +141,7 @@ export function checkWorkerDeregistration(fixture, observation, workflowId) { assert.equal(initial.workflow_task_attempt, 1); assert.equal(task.workflow_task_attempt, initial.workflow_task_attempt+1); assert.deepStrictEqual(live.before_replacement.events, history.slice(0,2)); - assert.deepStrictEqual(live.after_replacement, live.before_replacement, 'registration release preserves full original run/history'); + assert.deepStrictEqual(durable(live.after_replacement), durable(live.before_replacement), 'registration release preserves full original run/history'); for (const key of ['before_replacement','after_replacement','after_superseded','after_stale_before_poll','before_stale_after_poll','after_stale_after_poll']) { const run = live[key]; assert.equal(run.workflow_id, peerId); @@ -141,10 +151,10 @@ export function checkWorkerDeregistration(fixture, observation, workflowId) { assert.equal(run.status, state.before_unknown.status); assert.deepStrictEqual(run.typed_input, {type:'list',value:[fixture.typed_value]}); } - assert.deepStrictEqual(live.after_superseded, live.after_replacement); - assert.deepStrictEqual(live.after_stale_before_poll, live.after_replacement); - assert.deepStrictEqual(live.after_stale_after_poll, live.before_stale_after_poll); - assert.deepStrictEqual(live.after_stale_after_poll, state.before_unknown); + assert.deepStrictEqual(durable(live.after_superseded), durable(live.after_replacement)); + assert.deepStrictEqual(durable(live.after_stale_before_poll), durable(live.after_replacement)); + assert.deepStrictEqual(durable(live.after_stale_after_poll), durable(live.before_stale_after_poll)); + assert.deepStrictEqual(durable(live.after_stale_after_poll), durable(state.before_unknown)); assert.equal(live.superseded.status, 409); assert.equal(live.superseded.response.reason, 'worker_registration_lost_authority'); assert.equal(live.superseded.response.worker_id, workerId); diff --git a/scripts/conformance/server-parity/worker-listing-contract.mjs b/scripts/conformance/server-parity/worker-listing-contract.mjs index f57dcf083..e419be26e 100644 --- a/scripts/conformance/server-parity/worker-listing-contract.mjs +++ b/scripts/conformance/server-parity/worker-listing-contract.mjs @@ -9,6 +9,10 @@ const SUMMARY_FIELDS = ['worker_id', 'namespace', 'task_queue', 'runtime', 'sdk_ // Reviewed worker management contract (WorkerManagementController and the // worker registration and heartbeat facts it reports). Embedded applications // have no HTTP worker registration, so the projection is the definition alone. +// From Server 2.10 every registration stores its protocol version in its +// process metrics, including one that reports no metrics. +const PROTOCOL = {_worker_protocol_version: '1.19'}; + export function checkWorkerListing(fixture, observation, workflowId) { const definition = fixture.worker_listing; const state = observation.worker_listing; @@ -50,7 +54,7 @@ export function checkWorkerListing(fixture, observation, workflowId) { max_concurrent_worker_sessions: capacity.sessions, task_slots: {...definition.stored_task_slots, workflow_capacity: capacity.workflow, activity_capacity: capacity.activity, session_capacity: capacity.sessions}, - process_metrics: {process_uptime_seconds: 0, process_id: state.probe_pid}, + process_metrics: {process_uptime_seconds: 0, process_id: state.probe_pid, ...PROTOCOL}, heartbeat_interval_seconds: null, status: 'active'}; // The published SDK's registration defaults. const secondFacts = {worker_id: second, namespace: 'default', task_queue: queues[1], runtime: 'php', build_id: null, @@ -59,7 +63,7 @@ export function checkWorkerListing(fixture, observation, workflowId) { max_concurrent_activity_tasks: 1, max_concurrent_worker_sessions: 10, task_slots: {workflow_available: null, activity_available: null, session_available: null, workflow_capacity: 1, activity_capacity: 1, session_capacity: 10}, - process_metrics: null, heartbeat_interval_seconds: null, status: 'active'}; + process_metrics: PROTOCOL, heartbeat_interval_seconds: null, status: 'active'}; const registered = {[first]: state.first_registration, [second]: state.second_registration}; const summary = (worker, facts, label) => { equal(Object.keys(worker).sort(), [...SUMMARY_FIELDS].sort(), `${label} summary fields`); @@ -106,7 +110,7 @@ export function checkWorkerListing(fixture, observation, workflowId) { // task slots and process metrics it did not report. described(state.after_reregistration, {...firstFacts, task_slots: {...firstFacts.task_slots, workflow_available: null, activity_available: null, session_available: null}, - process_metrics: null}, 'after re-registration'); + process_metrics: PROTOCOL}, 'after re-registration'); equal(state.after_reregistration.registered_at, state.described.registered_at, 're-registration keeps the registration time'); return projection; } diff --git a/tests/Fixtures/ServerParity/activity-lease-renewal.json b/tests/Fixtures/ServerParity/activity-lease-renewal.json new file mode 100644 index 000000000..e6e569ce0 --- /dev/null +++ b/tests/Fixtures/ServerParity/activity-lease-renewal.json @@ -0,0 +1,37 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "activity-lease-renewal", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "lease renewal λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "lease renewal λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "activity_lease_renewal": { + "workflow_type": "parity.v1.lease", + "activity_type": "parity.v1.lease_activity", + "heartbeat_timeout": 60, + "lease_seconds": 300, + "empty_blob": "wwHioz3/VYAiNwwA", + "result_blob": "wwHioz3/VYAiNwoIZG9uZQ==", + "validation": { + "missing_fields": { + "message": "The activity attempt id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."]}, + "validation_errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."]} + }, + "non_string_fields": { + "message": "The activity attempt id field must be a string. (and 1 more error)", + "reason": "validation_failed", + "errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."]}, + "validation_errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."]} + } + } + }, + "contract": "POST /api/worker/activity-tasks/{task}/renew-lease (published Server 2.11, advertised as server_capabilities.activity_lease_renewal) keeps a live attempt's lease for its owner. It answers 200 with the attempt's status: can_continue true, no heartbeat recorded, the run, task, execution and attempt identities, the stored heartbeat and its deadline unchanged, lease_renewed true, server_time, task_id and worker_session null. The lease then runs the configured 300 seconds from the renewal, never less than before, and the run's history is unchanged. A 1.19 request and an unruled body field are accepted. Ownership is checked in PHP's order with its bodies: an unknown task or attempt is 404, another task or another owner 409. Missing or non-string fields fail Laravel's validation. A heartbeat's progress is reported and its own deadline kept. A completed, cancelled or terminated attempt is refused with its status and lease_renewed false, with no lease. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParity/namespace-delete.json b/tests/Fixtures/ServerParity/namespace-delete.json new file mode 100644 index 000000000..37fdd283f --- /dev/null +++ b/tests/Fixtures/ServerParity/namespace-delete.json @@ -0,0 +1,29 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "namespace-delete", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "namespace delete λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "namespace delete λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "namespace_delete": { + "description": "delete me λ", + "search_attribute": {"name": "CustomerTier", "type": "keyword"}, + "deleted": { + "workflow_commands": 1, + "workflow_history_events": 2, + "workflow_tasks": 1, + "search_attribute_definitions": 1, + "workflow_run_summaries": 1, + "workflow_runs": 1, + "workflow_instances": 1 + }, + "asynchronous_projections": ["workflow_run_timeline_entries", "workflow_run_timer_entries", "workflow_run_lineage_entries"] + }, + "contract": "DELETE /api/namespaces/{namespace} (published SDK deleteNamespace) removes a namespace and every row it owns in one transaction. Here the namespace holds one search attribute definition and one started, unclaimed run, and the response is 200 {name, status: deleted, deleted} with each table's non-zero count: one command, two history events, one task, one definition, one run summary, one run and one instance. PHP fills its timeline, timer and lineage projections asynchronously, so those counts may be absent or positive. Afterwards the namespace reads and deletes as 404 namespace_not_found with the message Namespace [name] not found. A run in another namespace is untouched, and the deleted run's workflow ID starts a new run. Embedded applications have no HTTP namespaces." +} diff --git a/tests/Fixtures/ServerParity/service-catalog.json b/tests/Fixtures/ServerParity/service-catalog.json new file mode 100644 index 000000000..8d4ed7fb1 --- /dev/null +++ b/tests/Fixtures/ServerParity/service-catalog.json @@ -0,0 +1,1076 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "service-catalog", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "service catalog λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "service catalog λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "service_catalog": { + "requests": [ + { + "label": "endpoints list", + "path": "service-catalog/endpoints", + "namespace": "default", + "version": true + }, + { + "label": "services list", + "path": "service-catalog/services", + "namespace": "default", + "version": true + }, + { + "label": "operations list", + "path": "service-catalog/operations", + "namespace": "default", + "version": true + }, + { + "label": "calls list", + "path": "service-catalog/calls", + "namespace": "default", + "version": true + }, + { + "label": "endpoints show", + "path": "service-catalog/endpoints/missing-id", + "namespace": "default", + "version": true + }, + { + "label": "services show", + "path": "service-catalog/services/missing-id", + "namespace": "default", + "version": true + }, + { + "label": "operations show", + "path": "service-catalog/operations/missing-id", + "namespace": "default", + "version": true + }, + { + "label": "calls show", + "path": "service-catalog/calls/missing-id", + "namespace": "default", + "version": true + }, + { + "label": "endpoints page2 per2", + "path": "service-catalog/endpoints?page=2&per_page=2", + "namespace": "default", + "version": true + }, + { + "label": "endpoints per0", + "path": "service-catalog/endpoints?per_page=0", + "namespace": "default", + "version": true + }, + { + "label": "endpoints per_max+1", + "path": "service-catalog/endpoints?per_page=1000", + "namespace": "default", + "version": true + }, + { + "label": "endpoints page0", + "path": "service-catalog/endpoints?page=0", + "namespace": "default", + "version": true + }, + { + "label": "endpoints pagex", + "path": "service-catalog/endpoints?page=x", + "namespace": "default", + "version": true + }, + { + "label": "endpoints unknown", + "path": "service-catalog/endpoints?bogus=1&other=2", + "namespace": "default", + "version": true + }, + { + "label": "endpoints namespace", + "path": "service-catalog/endpoints?namespace=default", + "namespace": "default", + "version": true + }, + { + "label": "endpoints endpoint_id", + "path": "service-catalog/endpoints?endpoint_id=x", + "namespace": "default", + "version": true + }, + { + "label": "services endpoint_id", + "path": "service-catalog/services?endpoint_id=e1", + "namespace": "default", + "version": true + }, + { + "label": "services endpoint_id empty", + "path": "service-catalog/services?endpoint_id=", + "namespace": "default", + "version": true + }, + { + "label": "operations ids", + "path": "service-catalog/operations?endpoint_id=e&service_id=s", + "namespace": "default", + "version": true + }, + { + "label": "calls filters", + "path": "service-catalog/calls?scope=owned&status=completed&outcome=completed&status_bucket=completed&outcome_bucket=completed", + "namespace": "default", + "version": true + }, + { + "label": "calls bad enums", + "path": "service-catalog/calls?scope=x&status=y&outcome=z&status_bucket=q&outcome_bucket=w", + "namespace": "default", + "version": true + }, + { + "label": "calls bucket mismatch", + "path": "service-catalog/calls?bucket=open&status_bucket=failed", + "namespace": "default", + "version": true + }, + { + "label": "calls bucket", + "path": "service-catalog/calls?bucket=open", + "namespace": "default", + "version": true + }, + { + "label": "show with page", + "path": "service-catalog/endpoints/x?page=2", + "namespace": "default", + "version": true + }, + { + "label": "show with namespace", + "path": "service-catalog/calls/x?namespace=default", + "namespace": "default", + "version": true + }, + { + "label": "no cpv header", + "path": "service-catalog/endpoints", + "namespace": "default", + "version": false + }, + { + "label": "list per_page max", + "path": "service-catalog/calls?per_page=100&page=3", + "namespace": "default", + "version": true + }, + { + "label": "header default, query second", + "path": "service-catalog/endpoints?namespace=@own", + "namespace": "default", + "version": true + }, + { + "label": "header second, query default", + "path": "service-catalog/endpoints?namespace=default", + "namespace": "@own", + "version": true + }, + { + "label": "no header, query second", + "path": "service-catalog/endpoints?namespace=@own", + "namespace": null, + "version": true + }, + { + "label": "no header, no query", + "path": "service-catalog/endpoints", + "namespace": null, + "version": true + }, + { + "label": "header missing", + "path": "service-catalog/endpoints", + "namespace": "missing", + "version": true + }, + { + "label": "header upper", + "path": "service-catalog/calls", + "namespace": "@OWN", + "version": true + }, + { + "label": "namespace too long", + "path": "service-catalog/calls/x?namespace=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "namespace": "default", + "version": true + }, + { + "label": "per_page decimal", + "path": "service-catalog/endpoints?per_page=2.5", + "namespace": "default", + "version": true + }, + { + "label": "page negative", + "path": "service-catalog/endpoints?page=-1", + "namespace": "default", + "version": true + }, + { + "label": "operation show", + "path": "service-catalog/operations/op-1", + "namespace": "@own", + "version": true + } + ], + "expected": { + "endpoints list": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "services list": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "operations list": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "calls list": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default", + "scope": "relevant", + "bucket": null, + "status_bucket": null, + "outcome": null, + "outcome_bucket": null, + "statuses": [ + "pending", + "accepted", + "started", + "completed", + "failed", + "cancelled" + ], + "outcomes": [ + "accepted", + "completed", + "cancelled", + "timed_out", + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open", + "degraded", + "handler_failed" + ], + "status_buckets": { + "open": [ + "pending", + "accepted", + "started" + ], + "completed": [ + "completed" + ], + "failed": [ + "failed" + ], + "cancelled": [ + "cancelled" + ] + }, + "outcome_buckets": { + "open": [ + "accepted" + ], + "completed": [ + "completed", + "degraded" + ], + "failed": [ + "timed_out", + "handler_failed" + ], + "cancelled": [ + "cancelled" + ], + "policy": [ + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open" + ] + } + } + ], + "endpoints show": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ], + "services show": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ], + "operations show": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ], + "calls show": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ], + "endpoints page2 per2": [ + 200, + { + "data": [], + "current_page": 2, + "per_page": 2, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "endpoints per0": [ + 422, + { + "message": "The per page field must be between 1 and 100.", + "reason": "validation_failed", + "errors": { + "per_page": [ + "The per page field must be between 1 and 100." + ] + }, + "validation_errors": { + "per_page": [ + "The per page field must be between 1 and 100." + ] + } + } + ], + "endpoints per_max+1": [ + 422, + { + "message": "The per page field must be between 1 and 100.", + "reason": "validation_failed", + "errors": { + "per_page": [ + "The per page field must be between 1 and 100." + ] + }, + "validation_errors": { + "per_page": [ + "The per page field must be between 1 and 100." + ] + } + } + ], + "endpoints page0": [ + 422, + { + "message": "The page field must be at least 1.", + "reason": "validation_failed", + "errors": { + "page": [ + "The page field must be at least 1." + ] + }, + "validation_errors": { + "page": [ + "The page field must be at least 1." + ] + } + } + ], + "endpoints pagex": [ + 422, + { + "message": "The page field must be an integer.", + "reason": "validation_failed", + "errors": { + "page": [ + "The page field must be an integer." + ] + }, + "validation_errors": { + "page": [ + "The page field must be an integer." + ] + } + } + ], + "endpoints unknown": [ + 422, + { + "message": "Unsupported catalog filter. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "bogus": [ + "Unsupported catalog filter." + ], + "other": [ + "Unsupported catalog filter." + ] + }, + "validation_errors": { + "bogus": [ + "Unsupported catalog filter." + ], + "other": [ + "Unsupported catalog filter." + ] + } + } + ], + "endpoints namespace": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "endpoints endpoint_id": [ + 422, + { + "message": "Unsupported catalog filter.", + "reason": "validation_failed", + "errors": { + "endpoint_id": [ + "Unsupported catalog filter." + ] + }, + "validation_errors": { + "endpoint_id": [ + "Unsupported catalog filter." + ] + } + } + ], + "services endpoint_id": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "services endpoint_id empty": [ + 422, + { + "message": "The endpoint id field is required.", + "reason": "validation_failed", + "errors": { + "endpoint_id": [ + "The endpoint id field is required." + ] + }, + "validation_errors": { + "endpoint_id": [ + "The endpoint id field is required." + ] + } + } + ], + "operations ids": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "calls filters": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default", + "scope": "owned", + "bucket": "completed", + "status_bucket": "completed", + "outcome": "completed", + "outcome_bucket": "completed", + "statuses": [ + "pending", + "accepted", + "started", + "completed", + "failed", + "cancelled" + ], + "outcomes": [ + "accepted", + "completed", + "cancelled", + "timed_out", + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open", + "degraded", + "handler_failed" + ], + "status_buckets": { + "open": [ + "pending", + "accepted", + "started" + ], + "completed": [ + "completed" + ], + "failed": [ + "failed" + ], + "cancelled": [ + "cancelled" + ] + }, + "outcome_buckets": { + "open": [ + "accepted" + ], + "completed": [ + "completed", + "degraded" + ], + "failed": [ + "timed_out", + "handler_failed" + ], + "cancelled": [ + "cancelled" + ], + "policy": [ + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open" + ] + } + } + ], + "calls bad enums": [ + 422, + { + "message": "The selected scope is invalid. (and 4 more errors)", + "reason": "validation_failed", + "errors": { + "scope": [ + "The selected scope is invalid." + ], + "status": [ + "The selected status is invalid." + ], + "status_bucket": [ + "The selected status bucket is invalid." + ], + "outcome": [ + "The selected outcome is invalid." + ], + "outcome_bucket": [ + "The selected outcome bucket is invalid." + ] + }, + "validation_errors": { + "scope": [ + "The selected scope is invalid." + ], + "status": [ + "The selected status is invalid." + ], + "status_bucket": [ + "The selected status bucket is invalid." + ], + "outcome": [ + "The selected outcome is invalid." + ], + "outcome_bucket": [ + "The selected outcome bucket is invalid." + ] + } + } + ], + "calls bucket mismatch": [ + 422, + { + "message": "Must match status_bucket when both are supplied.", + "reason": "validation_failed", + "errors": { + "bucket": [ + "Must match status_bucket when both are supplied." + ] + }, + "validation_errors": { + "bucket": [ + "Must match status_bucket when both are supplied." + ] + } + } + ], + "calls bucket": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default", + "scope": "relevant", + "bucket": "open", + "status_bucket": "open", + "outcome": null, + "outcome_bucket": null, + "statuses": [ + "pending", + "accepted", + "started", + "completed", + "failed", + "cancelled" + ], + "outcomes": [ + "accepted", + "completed", + "cancelled", + "timed_out", + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open", + "degraded", + "handler_failed" + ], + "status_buckets": { + "open": [ + "pending", + "accepted", + "started" + ], + "completed": [ + "completed" + ], + "failed": [ + "failed" + ], + "cancelled": [ + "cancelled" + ] + }, + "outcome_buckets": { + "open": [ + "accepted" + ], + "completed": [ + "completed", + "degraded" + ], + "failed": [ + "timed_out", + "handler_failed" + ], + "cancelled": [ + "cancelled" + ], + "policy": [ + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open" + ] + } + } + ], + "show with page": [ + 422, + { + "message": "Unsupported catalog filter.", + "reason": "validation_failed", + "errors": { + "page": [ + "Unsupported catalog filter." + ] + }, + "validation_errors": { + "page": [ + "Unsupported catalog filter." + ] + } + } + ], + "show with namespace": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ], + "no cpv header": [ + 400, + { + "message": "Missing control-plane version header.", + "reason": "missing_control_plane_version", + "supported_version": "2", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Control-Plane-Version: 2 header on control-plane requests." + } + ], + "list per_page max": [ + 200, + { + "data": [], + "current_page": 3, + "per_page": 100, + "total": 0, + "last_page": 1, + "namespace": "default", + "scope": "relevant", + "bucket": null, + "status_bucket": null, + "outcome": null, + "outcome_bucket": null, + "statuses": [ + "pending", + "accepted", + "started", + "completed", + "failed", + "cancelled" + ], + "outcomes": [ + "accepted", + "completed", + "cancelled", + "timed_out", + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open", + "degraded", + "handler_failed" + ], + "status_buckets": { + "open": [ + "pending", + "accepted", + "started" + ], + "completed": [ + "completed" + ], + "failed": [ + "failed" + ], + "cancelled": [ + "cancelled" + ] + }, + "outcome_buckets": { + "open": [ + "accepted" + ], + "completed": [ + "completed", + "degraded" + ], + "failed": [ + "timed_out", + "handler_failed" + ], + "cancelled": [ + "cancelled" + ], + "policy": [ + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open" + ] + } + } + ], + "header default, query second": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "header second, query default": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "@own" + } + ], + "no header, query second": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "@own" + } + ], + "no header, no query": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "default" + } + ], + "header missing": [ + 404, + { + "message": "Namespace 'missing' does not exist.", + "reason": "namespace_not_found", + "namespace": "missing", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace." + } + ], + "header upper": [ + 200, + { + "data": [], + "current_page": 1, + "per_page": 50, + "total": 0, + "last_page": 1, + "namespace": "@own", + "scope": "relevant", + "bucket": null, + "status_bucket": null, + "outcome": null, + "outcome_bucket": null, + "statuses": [ + "pending", + "accepted", + "started", + "completed", + "failed", + "cancelled" + ], + "outcomes": [ + "accepted", + "completed", + "cancelled", + "timed_out", + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open", + "degraded", + "handler_failed" + ], + "status_buckets": { + "open": [ + "pending", + "accepted", + "started" + ], + "completed": [ + "completed" + ], + "failed": [ + "failed" + ], + "cancelled": [ + "cancelled" + ] + }, + "outcome_buckets": { + "open": [ + "accepted" + ], + "completed": [ + "completed", + "degraded" + ], + "failed": [ + "timed_out", + "handler_failed" + ], + "cancelled": [ + "cancelled" + ], + "policy": [ + "rejected_not_found", + "rejected_forbidden", + "rejected_throttled", + "rejected_concurrency_limited", + "rejected_circuit_open" + ] + } + } + ], + "namespace too long": [ + 422, + { + "message": "The namespace field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "namespace": [ + "The namespace field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "namespace": [ + "The namespace field must not be greater than 191 characters." + ] + } + } + ], + "per_page decimal": [ + 422, + { + "message": "The per page field must be an integer.", + "reason": "validation_failed", + "errors": { + "per_page": [ + "The per page field must be an integer." + ] + }, + "validation_errors": { + "per_page": [ + "The per page field must be an integer." + ] + } + } + ], + "page negative": [ + 422, + { + "message": "The page field must be at least 1.", + "reason": "validation_failed", + "errors": { + "page": [ + "The page field must be at least 1." + ] + }, + "validation_errors": { + "page": [ + "The page field must be at least 1." + ] + } + } + ], + "operation show": [ + 404, + { + "message": "Catalog object not found.", + "reason": "catalog_object_not_found" + } + ] + } + }, + "contract": "GET /api/service-catalog/{endpoints|services|operations|calls}[/{id}] (published Server 2.11) reads a namespace's service catalog. This fixture's namespace holds no catalog objects: every list is a 200 page with empty data, the requested page and page size (default 1 and 50), total 0, last page 1 and the resolved namespace; calls also report their scope (default relevant), buckets, outcome and the status and outcome vocabularies. Every identity is a 404 catalog_object_not_found. Filters are validated like Laravel: an unsupported filter, a non-integer or out-of-range page or page size, an empty identity, an unknown scope, status, outcome or bucket, and a bucket that disagrees with status_bucket are 422 validation_failed. The namespace comes from X-Namespace, else the namespace query, else default, case-folded; an unknown namespace is 404 namespace_not_found and a missing control-plane version 400. @own is the fixture's own namespace. Embedded applications have no HTTP catalog." +} diff --git a/tests/Fixtures/ServerParityBaselines/php-2.11.1.json b/tests/Fixtures/ServerParityBaselines/php-2.11.1.json new file mode 100644 index 000000000..3bbcc5871 --- /dev/null +++ b/tests/Fixtures/ServerParityBaselines/php-2.11.1.json @@ -0,0 +1,20 @@ +{ + "schema": "durable-workflow.server-port-artifacts/v1", + "server": { + "version": "2.11.1", + "source_commit": "5336a98dd256e046edfdb67c81764cde4babd360", + "images": { + "linux/amd64": "durableworkflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102", + "linux/arm64": "durableworkflow/server@sha256:7fed9bf49709f7cb2e5cfd96cf89c7c2dd596b926068fcb3390345e680cc2d6b" + } + }, + "workflow": "2.10.2", + "sdk_php": "2.2.6", + "sdk_python": "2.5.0", + "sdk_rust": "3.4.1", + "cli": "2.2.0", + "waterline": "2.3.1", + "sample_app_commit": "4427a537e092f9d92c96d2a4867b9186e0e75ccd", + "capacity_suite": "1.9.0", + "performance_status": "not_measured" +} diff --git a/tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs b/tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs new file mode 100644 index 000000000..eb454b44b --- /dev/null +++ b/tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkActivityLeaseRenewal} from '../../scripts/conformance/server-parity/activity-lease-renewal-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/activity-lease-renewal.json', import.meta.url))); +const definition = fixture.activity_lease_renewal; +const id = 'parity-v1-activity-lease-renewal'; + +// Shapes published Server 2.11.1 returned. +const claim = (suffix, n) => ({workflow_id: `${id}-${suffix}`, run_id: `run-${n}`, workflow_task_id: `wft-${n}`, + task_id: `task-${n}`, activity_attempt_id: `ATTEMPT-${n}`, activity_execution_id: `exec-${n}`, lease_owner: `${id}-${suffix}-worker`}); +const live = claim('live', 1); +const cancelled = claim('cancelled', 2); +const terminated = claim('terminated', 3); +const status = (claimed, fields = {}) => ({can_continue: true, cancel_requested: false, reason: null, heartbeat_recorded: false, + workflow_instance_id: claimed.workflow_id, workflow_run_id: claimed.run_id, workflow_task_id: claimed.task_id, + activity_execution_id: claimed.activity_execution_id, activity_attempt_id: claimed.activity_attempt_id, attempt_number: 1, + run_status: 'waiting', run_closed_reason: null, run_closed_at: null, activity_status: 'running', attempt_status: 'running', + task_status: 'leased', lease_owner: claimed.lease_owner, lease_expires_at: '2026-10-22T20:52:13.554963Z', last_heartbeat_at: null, + heartbeat_deadline_at: '2026-10-22T20:48:11.939526Z', ...fields}); +const reply = (code, response) => ({status: code, response: {...response, protocol_version: '1.20'}, lease_renewal_capability: true}); +const renewed = (fields = {}) => reply(200, {...status(live), lease_renewed: true, server_time: '2026-10-22T20:47:13.592509Z', + task_id: live.task_id, worker_session: null, ...fields}); +const refusal = (code, task, attempt, error, reason, extra = {}) => reply(code, {task_id: task, activity_attempt_id: attempt, error, reason, ...extra}); +const stopped = (claimed, runStatus, reason) => reply(409, {...status(claimed, {can_continue: false, cancel_requested: true, reason, + run_status: runStatus, run_closed_reason: runStatus, run_closed_at: '2026-10-22T20:48:18.603147Z', activity_status: 'cancelled', + attempt_status: 'cancelled', task_status: 'cancelled', lease_expires_at: null}), lease_renewed: false}); +const history = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted']; +const observation = () => structuredClone({mode: 'http', activity_lease_renewal: {applicable: true, namespace: `${id}-lease`, live, + status_before: reply(200, {task_id: live.task_id, can_continue: true, lease_expires_at: '2026-10-22T20:52:11.939673Z', + last_heartbeat_at: null, deadlines: {start_to_close: '2026-10-22T20:52:11.939526Z', heartbeat: '2026-10-22T20:48:11.939526Z'}}), + history_before: [...history], renewed: renewed(), + status_after: reply(200, {task_id: live.task_id, can_continue: true, lease_expires_at: '2026-10-22T20:52:13.554963Z', last_heartbeat_at: null}), + history_after: [...history], + renewed_on_protocol_1_19: renewed(), extra_field: renewed(), + foreign_owner: refusal(409, live.task_id, live.activity_attempt_id, 'Activity attempt lease is owned by another worker.', + 'lease_owner_mismatch', {lease_owner: live.lease_owner}), + unknown_attempt: refusal(404, live.task_id, 'no-such-attempt', 'Activity attempt not found.', 'attempt_not_found'), + unknown_task: refusal(404, 'no-such-task', live.activity_attempt_id, 'Activity task not found.', 'task_not_found'), + task_mismatch: refusal(409, live.workflow_task_id, live.activity_attempt_id, 'Activity attempt is not leased for this task.', 'task_mismatch'), + missing_fields: reply(422, definition.validation.missing_fields), + non_string_fields: reply(422, definition.validation.non_string_fields), + heartbeat: reply(200, {task_id: live.task_id, heartbeat_recorded: true, last_heartbeat_at: '2026-10-22T20:47:14.723462Z'}), + renewed_after_heartbeat: renewed({last_heartbeat_at: '2026-10-22T20:47:14.723462Z', heartbeat_deadline_at: '2026-10-22T20:48:14.723462Z'}), + completed: reply(200, {outcome: 'completed', recorded: true}), + renewed_after_completion: reply(409, {...status(live, {can_continue: false, reason: 'attempt_closed', activity_status: 'completed', + attempt_status: 'completed', task_status: 'completed', lease_expires_at: null, last_heartbeat_at: '2026-10-22T20:47:14.723462Z', + heartbeat_deadline_at: '2026-10-22T20:48:14.723462Z'}), lease_renewed: false}), + cancelled_claim: cancelled, cancel: 200, renewed_after_cancel: stopped(cancelled, 'cancelled', 'run_cancelled'), + terminated_claim: terminated, terminate: 200, renewed_after_terminate: stopped(terminated, 'terminated', 'run_terminated')}}); + +test('lease renewal accepts the published shapes', () => { + assert.deepStrictEqual(checkActivityLeaseRenewal(fixture, observation(), id), {scope: 'activity_lease_renewal', lease_seconds: 300}); +}); + +test('lease renewal accepts a native lease exactly 300 s after its millisecond renewal', () => { + const state = observation(); + const renewal = state.activity_lease_renewal.renewed.response; + renewal.server_time = '2026-10-22T20:47:13.592000Z'; + renewal.lease_expires_at = '2026-10-22T20:52:13.592000Z'; + state.activity_lease_renewal.status_after.response.lease_expires_at = '2026-10-22T20:52:13.592Z'; + assert.doesNotThrow(() => checkActivityLeaseRenewal(fixture, state, id)); +}); + +test('embedded lease renewal is explicitly inapplicable', () => { + const projection = checkActivityLeaseRenewal(fixture, + {mode: 'embedded', activity_lease_renewal: {applicable: false, reason: 'embedded_has_no_worker_protocol'}}, id); + assert.deepStrictEqual(projection, {scope: 'activity_lease_renewal', lease_seconds: 300}); + assert.throws(() => checkActivityLeaseRenewal(fixture, {mode: 'embedded', activity_lease_renewal: {applicable: true}}, id)); +}); + +for (const [label, mutate] of [ + ['an unadvertised capability', state => { state.renewed.lease_renewal_capability = null; }], + ['an older protocol version', state => { state.foreign_owner.response.protocol_version = '1.19'; }], + ['a refused live renewal', state => { state.renewed.status = 409; }], + ['a recorded heartbeat', state => { state.renewed.response.heartbeat_recorded = true; }], + ['a missing server time', state => { delete state.renewed.response.server_time; }], + ['a millisecond lease', state => { state.renewed.response.lease_expires_at = '2026-10-22T20:52:13.554Z'; }], + ['a lease shorter than the window', state => { state.renewed.response.lease_expires_at = '2026-10-22T20:51:13.554963Z'; }], + ['a lease longer than the window', state => { state.renewed.response.lease_expires_at = '2026-10-22T20:52:14.554963Z'; }], + ['a moved progress deadline', state => { state.renewed.response.heartbeat_deadline_at = '2026-10-22T20:48:13.592509Z'; }], + ['added history', state => { state.history_after.push('ActivityHeartbeatRecorded'); }], + ['an unrenewed status read', state => { state.status_after.response.lease_expires_at = '2026-10-22T20:52:11.939673Z'; }], + ['a session object', state => { state.renewed.response.worker_session = {}; }], + ['a refused 1.19 request', state => { state.renewed_on_protocol_1_19.status = 422; }], + ['a refused unruled field', state => { state.extra_field.status = 422; }], + ['a foreign owner accepted', state => { state.foreign_owner.status = 200; }], + ['a foreign owner without its owner', state => { delete state.foreign_owner.response.lease_owner; }], + ['an unknown attempt as 409', state => { state.unknown_attempt.status = 409; }], + ['an unknown task message', state => { state.unknown_task.response.error = 'Task not found.'; }], + ['a task mismatch as 404', state => { state.task_mismatch.status = 404; }], + ['a generic validation refusal', state => { state.missing_fields.response = {reason: 'invalid_request', protocol_version: '1.20'}; }], + ['a lost heartbeat', state => { state.renewed_after_heartbeat.response.last_heartbeat_at = null; }], + ['a heartbeat deadline renewed', state => { state.renewed_after_heartbeat.response.heartbeat_deadline_at = '2026-10-22T20:48:15.000000Z'; }], + ['a renewed closed attempt', state => { state.renewed_after_completion.status = 200; }], + ['a closed attempt keeping its lease', state => { state.renewed_after_completion.response.lease_expires_at = '2026-10-22T20:52:14.958720Z'; }], + ['a closed attempt with a server time', state => { state.renewed_after_completion.response.server_time = '2026-10-22T20:47:14.987586Z'; }], + ['a closed attempt with a task id', state => { state.renewed_after_completion.response.task_id = live.task_id; }], + ['a cancellation not requested', state => { state.renewed_after_cancel.response.cancel_requested = false; }], + ['a cancelled attempt keeping its lease', state => { state.renewed_after_cancel.response.lease_expires_at = '2026-10-22T20:52:16.801401Z'; }], + ['a terminated run as cancelled', state => { state.renewed_after_terminate.response.reason = 'run_cancelled'; }], + ['a millisecond close time', state => { state.renewed_after_terminate.response.run_closed_at = '2026-10-22T20:48:18.603Z'; }], +]) { + test(`lease renewal rejects ${label}`, () => { + const value = observation(); + mutate(value.activity_lease_renewal); + assert.throws(() => checkActivityLeaseRenewal(fixture, value, id)); + }); +} diff --git a/tests/Unit/ServerParityNamespaceDeleteContractTest.mjs b/tests/Unit/ServerParityNamespaceDeleteContractTest.mjs new file mode 100644 index 000000000..d8dc16523 --- /dev/null +++ b/tests/Unit/ServerParityNamespaceDeleteContractTest.mjs @@ -0,0 +1,52 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkNamespaceDelete} from '../../scripts/conformance/server-parity/namespace-delete-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/namespace-delete.json', import.meta.url))); +const namespace = 'test-delete'; +const gone = {status: 404, response: {message: `Namespace [${namespace}] not found.`, reason: 'namespace_not_found', namespace}}; +// What published PHP 2.5.14 answered for this fixture. +const model = () => structuredClone({mode: 'http', namespace_delete: {applicable: true, namespace, + created: {name: namespace, description: 'delete me λ', retention_mode: 'bounded', retention_days: 30, status: 'active'}, + search_attribute: 201, started: 'pending', + deleted: {status: 200, response: {name: namespace, status: 'deleted', deleted: {workflow_commands: 1, + workflow_history_events: 2, workflow_tasks: 1, search_attribute_definitions: 1, workflow_run_summaries: 1, + workflow_runs: 1, workflow_instances: 1}}}, + described_after: gone, deleted_again: gone, + sdk_delete_again: {class: 'DurableWorkflow\\Exception\\ServerException', status: 404, reason: 'namespace_not_found'}, + own_run: 'TEST', restarted: 'TEST-doomed'}}); +const projection = {scope: 'namespace_delete', deleted: fixture.namespace_delete.deleted}; + +test('published PHP deletion passes', () => assert.deepStrictEqual(checkNamespaceDelete(fixture, model(), 'TEST'), projection)); +test("PHP's asynchronous projections may be counted", () => { + const observation = model(); + Object.assign(observation.namespace_delete.deleted.response.deleted, {workflow_run_timeline_entries: 2, + workflow_run_lineage_entries: 1}); + checkNamespaceDelete(fixture, observation, 'TEST'); +}); +test('embedded explicitly declares HTTP namespaces inapplicable', () => assert.deepStrictEqual(checkNamespaceDelete(fixture, + {mode: 'embedded', namespace_delete: {applicable: false, reason: 'embedded_has_no_http_namespaces'}}, 'test'), projection)); +for (const [label, mutate] of [ + ['a missing observation', s => { for (const key of Object.keys(s)) delete s[key]; }], + ['another namespace', s => s.namespace = 'default'], + ['a refused delete', s => s.deleted = {status: 503, response: {reason: 'external_payload_storage_driver_unavailable'}}], + ['another status word', s => s.deleted.response.status = 'removed'], + ['an owned table left behind', s => delete s.deleted.response.deleted.workflow_tasks], + ['a count off by one', s => s.deleted.response.deleted.workflow_history_events = 3], + ['a zero count reported', s => s.deleted.response.deleted.workflow_memos = 0], + ['an extra table deleted', s => s.deleted.response.deleted.workflow_schedules = 1], + ['a zero asynchronous projection reported', s => s.deleted.response.deleted.workflow_run_timeline_entries = 0], + ['a deleted namespace still described', s => s.described_after = {status: 200, response: s.created}], + ['a second delete accepted', s => s.deleted_again = s.deleted], + ['another not-found message', s => s.deleted_again.response.message = 'Namespace not found.'], + ['the SDK accepting a second delete', s => s.sdk_delete_again = {accepted: true}], + ["another namespace's run removed", s => s.own_run = null], + ['the workflow ID still reserved', s => s.restarted = null], +]) { + test(`rejects ${label}`, () => { + const observation = model(); + mutate(observation.namespace_delete); + assert.throws(() => checkNamespaceDelete(fixture, observation, 'TEST')); + }); +} diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index e49a20545..e855bf2c3 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -122,6 +122,85 @@ test('a task PHP\'s maintenance loop already redispatched passes', () => { for (const check of stage.checks) if (['task_transport', 'routing_health'].includes(check.name)) check.status = 'ok'; assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); }); +const setChecks = (stage, statuses) => { + for (const check of stage.checks) if (check.name in statuses) check.status = statuses[check.name]; +}; +test('published PHP health read while its maintenance loop repairs the task passes', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {durable_resume_paths: 'warning'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('published PHP health read after its maintenance loop redispatched the task passes', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {task_transport: 'ok', routing_health: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects native health reporting a repair native never runs', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = 'rust'; + setChecks(stage, {durable_resume_paths: 'warning'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects native health reporting a redispatch native never runs', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = 'rust'; + setChecks(stage, {task_transport: 'ok', routing_health: 'ok'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects PHP health behind metrics that show the task redispatched', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: 0, unhealthy: 0, max_repair_count: 1}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: false, oldest_unhealthy_at: false}); + stage.sections.backlog.counts.unhealthy_tasks = 0; + setChecks(stage, {durable_resume_paths: 'warning'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +// Metrics read while PHP's repair is committed but the task not yet +// redispatched: still overdue, repaired once, and the run needs repair. +const midRepair = (observation, runtime = 'php') => { + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = runtime; + stage.sections.tasks.counts.max_repair_count = 1; + stage.sections.runs.counts.repair_needed = 1; + stage.sections.runs.timestamps.oldest_repair_needed_at = true; + stage.sections.backlog.counts.repair_needed_runs = 1; + return stage; +}; +test('published PHP metrics read while its maintenance loop repairs the task pass', () => { + const observation = model(); + setChecks(midRepair(observation), {durable_resume_paths: 'warning'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('published PHP health read after the repair its metrics caught midway passes', () => { + const observation = model(); + setChecks(midRepair(observation), {task_transport: 'ok', routing_health: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health behind metrics caught mid-repair', () => { + const observation = model(); + midRepair(observation); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects native metrics showing a repair native never runs', () => { + const observation = model(); + setChecks(midRepair(observation, 'rust'), {durable_resume_paths: 'warning'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a run needing repair without a repaired task', () => { + const observation = model(); + const stage = midRepair(observation); + stage.sections.tasks.counts.max_repair_count = 0; + setChecks(stage, {durable_resume_paths: 'warning'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a repair warning at another stage', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting, {durable_resume_paths: 'warning'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); test('rejects a repaired task still reported overdue', () => { const observation = model(); observation.operator_metrics.stages.waiting_after_redispatch.sections.tasks.counts.max_repair_count = 1; diff --git a/tests/Unit/ServerParitySchedulePoliciesContractTest.mjs b/tests/Unit/ServerParitySchedulePoliciesContractTest.mjs index aa36002c6..ccb8ce5ce 100644 --- a/tests/Unit/ServerParitySchedulePoliciesContractTest.mjs +++ b/tests/Unit/ServerParitySchedulePoliciesContractTest.mjs @@ -48,3 +48,25 @@ for (const [label, mutate] of [ assert.throws(() => checkSchedulePolicies(fixture, observation, workflowId)); }); } +test('a status difference names the step and its response', () => { + const observation = model(); + observation.schedule_policies.steps[47] = {request: 'POST schedules/quota/trigger', status: 404, + response: {reason: 'schedule_not_found', schedule_id: 'quota'}}; + assert.throws(() => checkSchedulePolicies(fixture, observation, workflowId), + /step 47 POST schedules\/quota\/trigger: 404, expected 200: \{"reason":"schedule_not_found","schedule_id":"quota"\}/); +}); +test('an outcome difference shows the trigger responses', () => { + const observation = model(); + Object.assign(at(observation, 'POST schedules/policy-skip/trigger')[0].response, {outcome: 'skipped', reason: 'overlap_policy_skip'}); + assert.throws(() => checkSchedulePolicies(fixture, observation, workflowId), + /skip trigger outcomes \["skipped","skipped","skipped"\], expected \["triggered","skipped","skipped"\]: .*"reason":"overlap_policy_skip"/); +}); +test('a fifth sequence in its own namespace passes, a sixth does not', () => { + const fifth = model(); + const replayed = JSON.parse(JSON.stringify(fifth.schedule_policies).replaceAll('test-schedule-policies-schedules', 'test-schedule-policies-schedules-r5')); + fifth.schedule_policies = {...replayed, attempts: 5}; + assert.deepStrictEqual(checkSchedulePolicies(fixture, fifth, workflowId), projection); + const sixth = model(); + sixth.schedule_policies = {...JSON.parse(JSON.stringify(sixth.schedule_policies).replaceAll('test-schedule-policies-schedules', 'test-schedule-policies-schedules-r6')), attempts: 6}; + assert.throws(() => checkSchedulePolicies(fixture, sixth, workflowId), /recorded sequence attempts/); +}); diff --git a/tests/Unit/ServerParityServiceCatalogContractTest.mjs b/tests/Unit/ServerParityServiceCatalogContractTest.mjs new file mode 100644 index 000000000..23f4e6f98 --- /dev/null +++ b/tests/Unit/ServerParityServiceCatalogContractTest.mjs @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkServiceCatalog} from '../../scripts/conformance/server-parity/service-catalog-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/service-catalog.json', import.meta.url))); +const id = 'parity-v1-service-catalog'; +const own = `${id}-catalog`; +const observation = () => { + const responses = {}; + for (const {label} of fixture.service_catalog.requests) { + responses[label] = JSON.parse(JSON.stringify(fixture.service_catalog.expected[label]).split('"@own"').join(JSON.stringify(own))); + } + return {mode: 'http', service_catalog: {applicable: true, namespace: own, responses}}; +}; + +test('the service catalog accepts the published answers', () => { + assert.deepStrictEqual(checkServiceCatalog(fixture, observation(), id), {scope: 'service_catalog', cases: 37}); +}); + +test('the fixture covers lists, identities, filters and namespace selection', () => { + const {expected} = fixture.service_catalog; + const statuses = Object.values(expected).map(([status]) => status); + for (const status of [200, 400, 404, 422]) { + assert.ok(statuses.includes(status), `a ${status} case`); + } + assert.deepStrictEqual(expected['header second, query default'][1].namespace, '@own'); + assert.deepStrictEqual(expected['calls list'][1].outcome_buckets.policy.length, 5); +}); + +test('the embedded service catalog is explicitly inapplicable', () => { + checkServiceCatalog(fixture, {mode: 'embedded', service_catalog: {applicable: false, reason: 'embedded_has_no_http_service_catalog'}}, id); + assert.throws(() => checkServiceCatalog(fixture, {mode: 'embedded', service_catalog: {applicable: true}}, id)); +}); + +for (const [label, mutate] of [ + ['a missing response', responses => { delete responses['calls list']; }], + ['an extra response', responses => { responses.extra = [200, {}]; }], + ['a route that is not implemented', responses => { responses['endpoints list'] = [501, {reason: 'rust_capability_not_implemented'}]; }], + ['a non-empty page', responses => { responses['services list'][1].total = 1; }], + ['a default page size other than 50', responses => { responses['operations list'][1].per_page = 25; }], + ['the header namespace losing to the query', responses => { responses['header default, query second'][1].namespace = own; }], + ['an uncased namespace', responses => { responses['header upper'][1].namespace = own.toUpperCase(); }], + ['an identity found', responses => { responses['calls show'] = [200, {}]; }], + ['a generic not found', responses => { responses['endpoints show'][1].reason = 'not_found'; }], + ['a min message for per_page', responses => { responses['endpoints per0'][1].message = 'The per page field must be at least 1.'; }], + ['an accepted unknown filter', responses => { responses['endpoints unknown'] = [200, {}]; }], + ['an accepted page on a detail', responses => { responses['show with page'] = [404, {}]; }], + ['an unchecked bucket disagreement', responses => { responses['calls bucket mismatch'] = [200, {}]; }], + ['a bucket not echoed as status_bucket', responses => { responses['calls bucket'][1].status_bucket = null; }], + ['a missing outcome vocabulary', responses => { delete responses['calls filters'][1].outcomes; }], +]) { + test(`the service catalog rejects ${label}`, () => { + const value = observation(); + mutate(value.service_catalog.responses); + assert.throws(() => checkServiceCatalog(fixture, value, id)); + }); +} diff --git a/tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs b/tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs index 7880c7321..da497a13d 100644 --- a/tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs +++ b/tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs @@ -10,7 +10,7 @@ const workflow = 'test-signals-before-worker-before-worker'; const namespace = 'test-signals-before-worker-unregistered'; const queue = 'test-signals-before-worker-unserved'; const EVENTS = ['StartAccepted', 'WorkflowStarted', 'SignalReceived', 'SignalReceived', 'SignalReceived', 'SignalReceived', - 'TerminateRequested', 'WorkflowTerminated']; + 'TerminateRequested', 'WorkflowTerminated', 'SignalRejected', 'SignalRejected', 'SignalRejected', 'SignalRejected']; function model() { // Requests are recorded as the fixture's templates; responses carry the run's names. const steps = expected.map(step => ({...step, response: JSON.parse(JSON.stringify(step.response) diff --git a/tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs b/tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs index 169888729..d775df3a8 100644 --- a/tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs +++ b/tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs @@ -116,6 +116,17 @@ test('modeled live lease replacement requires unchanged original history and fen assert.deepStrictEqual(checkLive(liveModel()),checkWorkerDeregistration(liveFixture, {mode:'embedded',worker_deregistration:{applicable:false,reason:'embedded_has_no_http_worker_registration_lifecycle'}},id)); }); +test('a live release may change only the current definition routing of the run read', () => { + const s=liveModel(); + const routing=runnable=>({status:'not_worker_pinned',blocked:false,runnable_task:runnable}); + s.live_replacement.before_replacement={...s.live_replacement.before_replacement,execution:{definition_routing:routing(false)}}; + for (const key of ['after_replacement','after_superseded','after_stale_before_poll']) { + s.live_replacement[key]={...s.live_replacement[key],execution:{definition_routing:routing(true)}}; + } + assert.doesNotThrow(()=>checkLive(s)); + s.live_replacement.after_replacement.execution.status='completed'; + assert.throws(()=>checkLive(s)); +}); const liveMutations={ 'missing live replacement evidence':s=>{delete s.live_replacement;}, 'live token reused':s=>{s.live_replacement.initial_registration.registration_token=tokens[0];}, diff --git a/tests/Unit/ServerParityWorkerListingContractTest.mjs b/tests/Unit/ServerParityWorkerListingContractTest.mjs index fe3c89e7a..8a9d500f2 100644 --- a/tests/Unit/ServerParityWorkerListingContractTest.mjs +++ b/tests/Unit/ServerParityWorkerListingContractTest.mjs @@ -24,13 +24,13 @@ const firstWorker = {worker_id: first, namespace: 'default', task_queue: queues[ max_concurrent_workflow_tasks: capacity.workflow, max_concurrent_activity_tasks: capacity.activity, max_concurrent_worker_sessions: capacity.sessions, task_slots: {...definition.stored_task_slots, workflow_capacity: capacity.workflow, activity_capacity: capacity.activity, session_capacity: capacity.sessions}, - process_metrics: {process_uptime_seconds: 0, process_id: 17}, heartbeat_interval_seconds: null, status: 'active', + process_metrics: {process_uptime_seconds: 0, process_id: 17, _worker_protocol_version: '1.19'}, heartbeat_interval_seconds: null, status: 'active', last_heartbeat_at: at, registered_at: at}; const secondWorker = {...firstWorker, worker_id: second, task_queue: queues[1], build_id: null, workflow_definition_fingerprints: [], supported_activity_types: [], max_concurrent_workflow_tasks: 1, max_concurrent_activity_tasks: 1, max_concurrent_worker_sessions: 10, task_slots: {workflow_available: null, activity_available: null, session_available: null, - workflow_capacity: 1, activity_capacity: 1, session_capacity: 10}, process_metrics: null}; + workflow_capacity: 1, activity_capacity: 1, session_capacity: 10}, process_metrics: {_worker_protocol_version: '1.19'}}; const listing = workers => ({stale_after_seconds: definition.stale_after_seconds, workers}); const described = {...firstWorker, updated_at: at, stale_after_seconds: definition.stale_after_seconds}; const refusal = (reason, error, remediation) => ({status: 409, response: {error, reason, @@ -48,7 +48,7 @@ const observation = () => structuredClone({mode: 'http', worker_listing: {applic 'Advertise the matching definition fingerprint or restart with a new worker_id.'), after_refusals: described, after_reregistration: {...described, task_slots: {...described.task_slots, workflow_available: null, - activity_available: null, session_available: null}, process_metrics: null}}}); + activity_available: null, session_available: null}, process_metrics: {_worker_protocol_version: '1.19'}}}}); test('worker listing accepts the published facts in any equal-heartbeat order', () => { assert.deepStrictEqual(checkWorkerListing(fixture, observation(), id),