CI: run the six deadline-health cells in turn on one runner

.github/workflows/server-parity.yml

@@ -16,6 +16,7 @@ on:
       - tests/Fixtures/ServerParityNormalization/**
       - tests/Unit/ServerParityRunnerTest.mjs
       - tests/Unit/ServerParitySdkDirectionLanesTest.mjs
+      - tests/Unit/ServerParityCodecDeadlineCellsTest.mjs
       - tests/Unit/ServerParityArtifactSummaryTest.mjs
       - tests/Unit/ServerParityQueryObserverTest.mjs
       - tests/Unit/ServerParityCodecDeadlineLoadTest.mjs
@@ -232,16 +233,15 @@ jobs:
           build_container="$RESOURCE_SCOPE-build"
           docker rm -f "$build_container" >/dev/null 2>&1 || true
 
+  # The six deadline-health cells take about a minute each, beside database
+  # parts that take half an hour, so one runner runs them in turn: a failed
+  # cell still lets the others run, and the job fails after all of them.
   codec-deadline-health:
     needs: [published-sdk-adapters, codec-native-build]
     runs-on: ubuntu-latest
-    timeout-minutes: 10
-    strategy:
-      fail-fast: false
-      matrix:
-        database: [sqlite, postgres-16, postgres-17, mysql-8.0, mysql-8.4, mariadb-10.11]
+    timeout-minutes: 25
     env:
-      RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-${{ matrix.database }}
+      RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
       - name: Fetch this run's optimized native binary
@@ -261,47 +261,46 @@ jobs:
           set -euo pipefail
           (cd codec-native-build; sha256sum -c SHA256SUMS)
           chmod +x codec-native-build/server
-          bash scripts/conformance/server-parity/codec-operations-cell.sh \
-            '${{ matrix.database }}' cpu-cell codec-native-build corrected-marker-adapters/php codec-evidence
-      - name: Retire this cell's exact owned resources
+          bash scripts/conformance/server-parity/codec-deadline-cells.sh
+      - name: Retire every cell's exact owned resources
         if: always()
         run: |
-          scope="$RESOURCE_SCOPE"
-          native_server="$scope-native"
-          db="$scope-db"
-          probe="$scope-ready"
-          observer="$RESOURCE_SCOPE-native-codec-observer"
-          docker rm -f "$observer" "$probe" "$native_server" >/dev/null 2>&1 || true
-          docker stop --time 10 "$db" >/dev/null 2>&1 || true
-          docker rm "$db" >/dev/null 2>&1 || true
-          docker network rm "$scope" >/dev/null 2>&1 || true
-          if test -d codec-evidence/state; then rm -r -- codec-evidence/state; fi
-      - name: Publish the bounded outcome
+          for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do
+            scope="$RESOURCE_SCOPE-$database"
+            docker rm -f "$scope-native-codec-observer" "$scope-ready" "$scope-native" >/dev/null 2>&1 || true
+            docker stop --time 10 "$scope-db" >/dev/null 2>&1 || true
+            docker rm "$scope-db" >/dev/null 2>&1 || true
+            docker network rm "$scope" >/dev/null 2>&1 || true
+          done
+      - name: Publish the bounded outcomes
         if: always()
         run: |
-          if test -f codec-evidence/database-audit.json; then
-            printf 'Database: %s\n\n' '${{ matrix.database }}' >> "$GITHUB_STEP_SUMMARY"
-            cat codec-evidence/database-audit.json >> "$GITHUB_STEP_SUMMARY"
-          else
-            printf 'CPU/deadline original did not qualify. See this job diagnostics.\n' >> "$GITHUB_STEP_SUMMARY"
-          fi
+          for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do
+            if test -f "codec-evidence/$database/database-audit.json"; then
+              printf 'Database: %s\n\n' "$database" >> "$GITHUB_STEP_SUMMARY"
+              cat "codec-evidence/$database/database-audit.json" >> "$GITHUB_STEP_SUMMARY"
+            else
+              printf 'Database: %s: the CPU/deadline original did not qualify. See this job diagnostics.\n\n' "$database" >> "$GITHUB_STEP_SUMMARY"
+            fi
+          done
       - name: Keep small outcomes and failure diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: codec-deadline-health-${{ matrix.database }}
+          name: codec-deadline-health
           path: |
-            codec-evidence/build.json
-            codec-evidence/database*.json
-            codec-evidence/database-version.txt
-            codec-evidence/codec-*-contract.txt
-            codec-evidence/codec-*-controls.txt
-            codec-evidence/limits.json
-            codec-evidence/*state.json
-            codec-evidence/server-log.txt
-            codec-evidence/server-ready-errors.txt
-            codec-evidence/profile.txt
-            codec-evidence/failed.json
+            codec-evidence/*.status
+            codec-evidence/*/build.json
+            codec-evidence/*/database*.json
+            codec-evidence/*/database-version.txt
+            codec-evidence/*/codec-*-contract.txt
+            codec-evidence/*/codec-*-controls.txt
+            codec-evidence/*/limits.json
+            codec-evidence/*/*state.json
+            codec-evidence/*/server-log.txt
+            codec-evidence/*/server-ready-errors.txt
+            codec-evidence/*/profile.txt
+            codec-evidence/*/failed.json
           retention-days: 1
           if-no-files-found: warn
 
@@ -1801,7 +1800,7 @@ jobs:
           bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
-            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt
+            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt
           docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
             --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt

scripts/conformance/server-parity/codec-deadline-cells.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Run every deadline-health cell in turn on this runner. Each cell owns its
+# resources under "$RESOURCE_SCOPE-<database>" and tries to stop its server
+# and database when it exits, failed or not. Before removing a cell's state
+# and starting the next cell, the runner confirms that both of those exact
+# containers are stopped or absent. If it cannot, it keeps the state, runs no
+# later cell and fails; the job's always() step removes every cell's
+# containers and networks. A cell that failed with confirmed cleanup still
+# lets the others run, and the script fails after all of them.
+set -euo pipefail
+if [[ -z "${RESOURCE_SCOPE:-}" ]]; then
+  echo 'Set the owning isolated resource scope' >&2
+  exit 2
+fi
+databases=(sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11)
+
+# Each named container is absent, or present and not running.
+actors_stopped() {
+  local name state
+  for name in "$@"; do
+    if ! state=$(docker inspect --format '{{.State.Running}}' "$name" 2>&1); then
+      if [[ "$state" == *"No such object"* || "$state" == *"No such container"* ]]; then
+        continue
+      fi
+      echo "::error::Could not confirm that $name stopped: $state" >&2
+      return 1
+    fi
+    if [[ "$state" != false ]]; then
+      echo "::error::$name is still running" >&2
+      return 1
+    fi
+  done
+}
+
+mkdir -p codec-evidence
+failed=0
+halted=
+for database in "${databases[@]}"; do
+  if [[ -n "$halted" ]]; then
+    printf 'skipped after %s\n' "$halted" > "codec-evidence/$database.status"
+    continue
+  fi
+  scope="$RESOURCE_SCOPE-$database"
+  status=0
+  # The cell's unique scope admits only lowercase letters, digits and dashes.
+  RESOURCE_SCOPE="$scope" bash scripts/conformance/server-parity/codec-operations-cell.sh \
+    "$database" "cpu-${database//./-}" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$?
+  printf '%s\n' "$status" > "codec-evidence/$database.status"
+  if ! actors_stopped "$scope-native" "$scope-db"; then
+    echo "::error::Deadline-health cell $database left its cleanup unconfirmed; keeping its state and running no later cell" >&2
+    halted="$database"
+    failed=1
+    continue
+  fi
+  if [[ -d "codec-evidence/$database/state" ]]; then
+    rm -r -- "codec-evidence/$database/state"
+  fi
+  if [[ "$status" != 0 ]]; then
+    echo "::error::Deadline-health cell $database exited $status" >&2
+    failed=1
+  fi
+done
+exit "$failed"

scripts/conformance/server-parity/codec-operations-cell.sh

@@ -39,6 +39,11 @@ db="$scope-db"
 probe="$scope-ready"
 cleanup() {
   docker logs --tail 40 "$native_server" > "$output/server-log.txt" 2>&1 || true
+  # Stop this cell's server and database on every exit, failure included, so
+  # a failed cell never keeps them running while later cells qualify. The
+  # job's always() step removes them.
+  docker stop --time 10 "$native_server" >/dev/null 2>&1 || true
+  docker stop --time 10 "$db" >/dev/null 2>&1 || true
 }
 trap cleanup EXIT
 docker network create "$scope" >/dev/null