diff --git a/.github/workflows/deploy-brio-identity-db.yml b/.github/workflows/deploy-brio-identity-db.yml index cf0f1ea..5267070 100644 --- a/.github/workflows/deploy-brio-identity-db.yml +++ b/.github/workflows/deploy-brio-identity-db.yml @@ -60,6 +60,20 @@ jobs: printf '%s\n' "${SSH_KNOWN_HOSTS}" > "${ssh_dir}/known_hosts" chmod 0600 "${ssh_dir}"/* + - name: Acquire the cross-host Brio deployment lease + shell: bash + env: + BRIO_LEASE_REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + BRIO_LEASE_REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + BRIO_LEASE_REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + run: | + set -euo pipefail + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + [[ "${owner}" =~ ^[0-9a-f]{64}$ ]] + printf 'BRIO_OPERATION_LEASE_OWNER=%s\n' "${owner}" >> "${GITHUB_ENV}" + export BRIO_LEASE_SSH_DIRECTORY="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + scripts/brio-operation-lease-remote.sh acquire "${owner}" + - name: Prepare standalone DB-VM bundle and secrets shell: bash env: @@ -115,9 +129,11 @@ jobs: target="${REMOTE_USER}@${REMOTE_HOST}" printf -v remote_bundle_q %q "${remote_bundle}" printf -v remote_runtime_q %q "${remote_runtime}" + printf -v lease_owner_q %q "${BRIO_OPERATION_LEASE_OWNER}" + lease_status="sudo -n /usr/local/libexec/makepad/brio-operation-lease status ${lease_owner_q} deployment >/dev/null" # Both remote paths are fixed, run-scoped, and escaped locally with printf %q. # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "${target}" "if [ -e ${remote_bundle_q} ] || [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote identity deployment path.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + ssh "${ssh_opts[@]}" "${target}" "${lease_status} && if [ -e ${remote_bundle_q} ] || [ -e ${remote_runtime_q} ]; then echo 'Refusing existing remote identity deployment path.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" scp "${scp_opts[@]}" -r "${bundle_dir}/." "${target}:${remote_bundle}/" printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" printf -v db_env_q %q "${remote_bundle}/envs/production/.env.db" @@ -134,7 +150,7 @@ jobs: # Validated values are escaped locally with printf %q. # shellcheck disable=SC2029 ssh "${ssh_opts[@]}" "${target}" \ - "BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes ${deploy_script_q} ${remote_bundle_q} ${remote_runtime_q} ${db_hostname_q} ${cidr_q}" + "BRIO_OPERATION_LEASE_OWNER=${lease_owner_q} BRIO_IDENTITY_DB_DEPLOY_CONFIRM=restart-standalone-postgres-for-brio-staging BRIO_IDENTITY_DB_BACKUP_RESTORE_CONFIRMED=yes ${deploy_script_q} ${remote_bundle_q} ${remote_runtime_q} ${db_hostname_q} ${cidr_q}" - name: Create canonical standalone deployment evidence shell: bash @@ -195,9 +211,26 @@ jobs: ssh_opts=(-F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=${ssh_dir}/known_hosts" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes -i "${ssh_dir}/id_ed25519" -p "${remote_port}") printf -v remote_runtime_q %q "${remote_runtime}" printf -v remote_bundle_q %q "${remote_bundle}" + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + printf -v lease_owner_q %q "${owner}" # The fixed job path is escaped locally with printf %q. # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${remote_runtime_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_runtime_q} ]; then rm -f ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem; if [ -f ${remote_runtime_q}/RECOVERY_REQUIRED ] && [ ! -L ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${remote_runtime_q} -depth -delete; fi; elif [ -e ${remote_runtime_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "sudo -n /usr/local/libexec/makepad/brio-operation-lease status ${lease_owner_q} deployment >/dev/null && if [ -L ${remote_runtime_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_runtime_q} ]; then rm -f ${remote_runtime_q}/keycloak-brio-staging-app-password ${remote_runtime_q}/keycloak-brio-staging-backup-password ${remote_runtime_q}/brio-backup-recipient-cert.pem; if [ -f ${remote_runtime_q}/RECOVERY_REQUIRED ] && [ ! -L ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${remote_runtime_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${remote_runtime_q} -depth -delete; fi; elif [ -e ${remote_runtime_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + + - name: Release the cross-host Brio deployment lease + if: always() + shell: bash + env: + BRIO_LEASE_REMOTE_HOST: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_HOST }} + BRIO_LEASE_REMOTE_PORT: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_PORT }} + BRIO_LEASE_REMOTE_USER: ${{ secrets.BRIO_IDENTITY_DB_DEPLOY_SSH_USER }} + run: | + set -euo pipefail + ssh_directory="${RUNNER_TEMP}/postgres-identity-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_directory}/id_ed25519" && -r "${ssh_directory}/known_hosts" ]] || exit 0 + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + export BRIO_LEASE_SSH_DIRECTORY="${ssh_directory}" + scripts/brio-operation-lease-remote.sh release "${owner}" - name: Remove local job-scoped deployment material if: always() diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index fa14b50..6bd4ebe 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -56,6 +56,20 @@ jobs: printf '%s\n' "${DEPLOY_SSH_KNOWN_HOSTS}" > "${ssh_directory}/known_hosts" chmod 0600 "${ssh_directory}/known_hosts" + - name: Acquire the cross-host Brio deployment lease + shell: bash + env: + BRIO_LEASE_REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} + BRIO_LEASE_REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} + BRIO_LEASE_REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} + run: | + set -euo pipefail + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + [[ "${owner}" =~ ^[0-9a-f]{64}$ ]] + printf 'BRIO_OPERATION_LEASE_OWNER=%s\n' "${owner}" >> "${GITHUB_ENV}" + export BRIO_LEASE_SSH_DIRECTORY="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + scripts/brio-operation-lease-remote.sh acquire "${owner}" + - name: Materialize job-scoped Brio canary inputs if: inputs.environment == 'canary' shell: bash @@ -206,9 +220,11 @@ jobs: remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" printf -v remote_bundle_q %q "${remote_bundle}" printf -v remote_parent_q %q "${REMOTE_DIR}/.deploy" + printf -v lease_owner_q %q "${BRIO_OPERATION_LEASE_OWNER}" + lease_status="sudo -n /usr/local/libexec/makepad/brio-operation-lease status ${lease_owner_q} deployment >/dev/null" # The normalized, job-scoped paths are escaped locally with printf %q. # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "${remote_target}" "install -d -m 0755 ${remote_parent_q} && if [ -e ${remote_bundle_q} ]; then echo 'Refusing existing remote deployment bundle.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" + ssh "${ssh_opts[@]}" "${remote_target}" "${lease_status} && install -d -m 0755 ${remote_parent_q} && if [ -e ${remote_bundle_q} ]; then echo 'Refusing existing remote deployment bundle.' >&2; exit 1; fi && install -d -m 0700 ${remote_bundle_q}" scp "${scp_opts[@]}" -r "${bundle_root}/." "${remote_target}:${remote_bundle}/" # Install the host-side expiry guard before any job credential is transferred. printf -v cleaner_q %q "${remote_bundle}/scripts/ensure-brio-tmp-cleaner.sh" @@ -248,9 +264,9 @@ jobs: # Values are intentionally expanded locally and shell-escaped with %q. # shellcheck disable=SC2029 if [[ "${{ inputs.environment }}" == "canary" ]]; then - ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${remote_runtime_q}" + ssh "${ssh_opts[@]}" "${remote_target}" "BRIO_OPERATION_LEASE_OWNER=${lease_owner_q} ${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${remote_runtime_q}" else - ssh "${ssh_opts[@]}" "${remote_target}" "${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${deploy_env_q} ${remote_runtime_q}" + ssh "${ssh_opts[@]}" "${remote_target}" "BRIO_OPERATION_LEASE_OWNER=${lease_owner_q} ${remote_script_q} ${remote_bundle_q} ${stack_name_q} ${deploy_env_q} ${remote_runtime_q}" fi - name: Remove remote job-scoped deployment material @@ -280,9 +296,26 @@ jobs: remote_bundle="${REMOTE_DIR}/.deploy/postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" printf -v runtime_dir_q %q "${runtime_dir}" printf -v remote_bundle_q %q "${remote_bundle}" + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + printf -v lease_owner_q %q "${owner}" # The fixed job paths are escaped locally with printf %q. # shellcheck disable=SC2029 - ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "if [ -L ${runtime_dir_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${runtime_dir_q} ]; then rm -f ${runtime_dir_q}/postgres-superuser-password ${runtime_dir_q}/brio-staging-app-password ${runtime_dir_q}/brio-staging-backup-password ${runtime_dir_q}/postgres-ca.pem ${runtime_dir_q}/postgres-server-cert.pem ${runtime_dir_q}/postgres-server-key.pem ${runtime_dir_q}/brio-backup-recipient-cert.pem ${runtime_dir_q}/vif-db-password; if [ -f ${runtime_dir_q}/RECOVERY_REQUIRED ] && [ ! -L ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${runtime_dir_q} -depth -delete; fi; elif [ -e ${runtime_dir_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + ssh "${ssh_opts[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "sudo -n /usr/local/libexec/makepad/brio-operation-lease status ${lease_owner_q} deployment >/dev/null && if [ -L ${runtime_dir_q} ]; then echo 'Unsafe runtime symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${runtime_dir_q} ]; then rm -f ${runtime_dir_q}/postgres-superuser-password ${runtime_dir_q}/brio-staging-app-password ${runtime_dir_q}/brio-staging-backup-password ${runtime_dir_q}/postgres-ca.pem ${runtime_dir_q}/postgres-server-cert.pem ${runtime_dir_q}/postgres-server-key.pem ${runtime_dir_q}/brio-backup-recipient-cert.pem ${runtime_dir_q}/vif-db-password; if [ -f ${runtime_dir_q}/RECOVERY_REQUIRED ] && [ ! -L ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Recovery evidence retained; runtime cleanup intentionally skipped.' >&2; elif [ -e ${runtime_dir_q}/RECOVERY_REQUIRED ]; then echo 'Unsafe recovery marker; refusing runtime cleanup.' >&2; exit 1; else find ${runtime_dir_q} -depth -delete; fi; elif [ -e ${runtime_dir_q} ]; then echo 'Unexpected runtime path type; refusing cleanup.' >&2; exit 1; fi; if [ -L ${remote_bundle_q} ]; then echo 'Unsafe bundle symlink; refusing cleanup.' >&2; exit 1; elif [ -d ${remote_bundle_q} ]; then find ${remote_bundle_q} -depth -delete; elif [ -e ${remote_bundle_q} ]; then echo 'Unexpected bundle path type; refusing cleanup.' >&2; exit 1; fi" + + - name: Release the cross-host Brio deployment lease + if: always() + shell: bash + env: + BRIO_LEASE_REMOTE_HOST: ${{ secrets.DEPLOY_SSH_HOST }} + BRIO_LEASE_REMOTE_PORT: ${{ secrets.DEPLOY_SSH_PORT }} + BRIO_LEASE_REMOTE_USER: ${{ secrets.DEPLOY_SSH_USER }} + run: | + set -euo pipefail + ssh_directory="${RUNNER_TEMP}/postgres-deploy-ssh-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ -r "${ssh_directory}/id_ed25519" && -r "${ssh_directory}/known_hosts" ]] || exit 0 + owner=$(python3 scripts/derive-brio-operation-owner.py deployment) + export BRIO_LEASE_SSH_DIRECTORY="${ssh_directory}" + scripts/brio-operation-lease-remote.sh release "${owner}" - name: Remove job-scoped deployment material if: always() diff --git a/README.md b/README.md index 8b7edd6..74d97d7 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ GitHub environment variables. The exact Brio inventory is: | Canonical Proton Pass item | Protected GitHub environment | Exact mirrored fields | | --- | --- | --- | | `Hetzner App Server makepad` | `canary` and `production` | native fields `host`, `port`, `user`, `private_key`, and `known_hosts` map to the five `DEPLOY_SSH_*` destinations for the application Swarm manager required by `manual-deploy.yml` | -| `Hetzner Database Server makepad` | `staging-brio-identity-db` and `keycloak-cohort-restore` | canonical custom fields `DEPLOY_SSH_HOST`, `DEPLOY_SSH_PORT`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_PRIVATE_KEY`, and `DEPLOY_SSH_KNOWN_HOSTS` map to the workflow aliases `BRIO_IDENTITY_DB_DEPLOY_SSH_*` and `KEYCLOAK_COHORT_DB_SSH_*` only in their named standalone DB-host environments | +| `PostgreSQL · Brio identity database deployment SSH` | `staging-brio-identity-db` and `keycloak-cohort-restore` | canonical custom fields `DEPLOY_SSH_HOST`, `DEPLOY_SSH_PORT`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_PRIVATE_KEY`, and `DEPLOY_SSH_KNOWN_HOSTS` map to the workflow aliases `BRIO_IDENTITY_DB_DEPLOY_SSH_*` and `KEYCLOAK_COHORT_DB_SSH_*` only in their named standalone DB-host environments; the prior key remains outside the active inventory for rollback | | `PostgreSQL · shared Swarm deployment` | `canary` and `production` | current workflow-compatible protected fields for the PostgreSQL remote directory, stack, Catwlk network, and production-only VIF database name, role, network, and password; exact destinations are in `deploy/credential-inventory.json` | | `Le Petit Coin GitHub Deploy Secrets` | `canary` and `production` | canonical `DEPLOY_DB_NETWORK` maps to the PostgreSQL workflow alias `DEPLOY_LE_PETIT_COIN_DB_NETWORK`, keeping both stacks on the same application-owned database overlay | | `Brio Staging - PostgreSQL` | `canary`; Keycloak passwords only in `staging-brio-identity-db` | `canary` secrets `DEPLOY_BRIO_STAGING_DB_NETWORK`, `POSTGRES_CANARY_SUPERUSER_PASSWORD`, `BRIO_STAGING_DB_PASSWORD`, and `BRIO_STAGING_BACKUP_DB_PASSWORD`; only `KEYCLOAK_BRIO_STAGING_DB_PASSWORD` and `KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD` are mirrored to `staging-brio-identity-db` | @@ -248,6 +248,7 @@ GitHub environment variables. The exact Brio inventory is: | `PostgreSQL · JIT Launcher App` | repository policy only | public repository variable `POSTGRES_CI_LAUNCHER_APP_SENDER_ID`; private App fields remain on the controller host only | | `PostgreSQL · JIT hypervisor attestation` | repository policy only | public repository variables `POSTGRES_CI_ATTESTATION_PUBLIC_KEY` and `POSTGRES_CI_APPROVED_BASE_IMAGE_SHA256`; the signing key remains on the hypervisor only | | `PostgreSQL · GitHub repository variable bootstrap` | operator workstation only | field `repository_variable_admin_token` is supplied process-locally to `gh` only during the explicit four-variable sync; `owner` and `expires_at` remain operator verification records | +| `Brio · operation lease coordinator` | root-only on each Brio deployment host | fields `coordinator_json`, `ssh_private_key`, `ssh_known_hosts`, and `ssh_public_key`; none is mirrored to GitHub | The first two name-only entries are environment-scope duplicates: their active workflow destinations are the identically named `canary` secrets, while the @@ -435,6 +436,51 @@ selected-repository access to `Makepad-fr/postgres` only. The public repository runner groups continue to allow public repositories, but remain selected to this one exact repository and protected workflow set. +### Brio deployment/evidence exclusion lease + +Every PostgreSQL manual deployment and standalone Brio identity-database +deployment participates in the same host-local exclusion lease as Brio, +Keycloak, MailDev, and Nginx. A workflow hashes its immutable repository, run, +attempt, and commit identity into one public 64-character owner, then asks the +target host's root coordinator to acquire `app`, `identity`, and `database` in +that fixed order. Teardown releases `database`, `identity`, and `app`. A +four-hour expiry bounds a runner or network failure; reacquiring with the same +owner and kind is idempotent and never extends that expiry. Jobs remain capped +well below four hours. + +The local endpoint is `/usr/local/libexec/makepad/brio-operation-lease`. It +accepts only `acquire|status|release <64-lowercase-hex-owner> +`, serializes through the root-owned mode-`0600` guard, +and rejects unsafe runtime directories, symlinks, hard links, permissions, +owners, or malformed state. Deployment entrypoints require a matching local +`status OWNER deployment` immediately before their first provider mutation. +Thus release evidence holding `kind=evidence` blocks deployment, and a +deployment blocks browser evidence before either can mutate its first node. + +Bootstrap each deployment host once from reviewed, root-only material. For the +database host, invoke: + +```sh +sudo scripts/install-brio-operation-lease.sh \ + '' \ + '/secure/operator-path/operation-lease.pub' \ + '/secure/operator-path/coordinator.json' \ + '/secure/operator-path/operation-lease.key' \ + '/secure/operator-path/known_hosts' +``` + +The coordinator JSON must contain exactly the `app`, `identity`, and +`database` endpoints in that order and use the locked +`brio-operation-lease` SSH account. Stream the four fields from the canonical +`Brio · operation lease coordinator` Proton item into an owner-only tmpfs or +directly into the installer; never mirror them to GitHub or write them to a +runner workspace. The installer creates the endpoint account, forced-command +dispatcher, sudo policy, tmpfiles guard, root-only coordinator files, and this +host's exact `database` node identity. Complete this one-time bootstrap on all +three nodes before enabling the lease-gated workflows. A failed or partial +bootstrap must leave deployment disabled; it is not valid to bypass the +coordinator for initial rollout. + | Proton Pass item | Exact runtime fields and authority | | --- | --- | | `PostgreSQL · PR Checks App` | repository variable `POSTGRES_PR_CHECK_APP_ID` and protected `postgres-ci-attestation` secret `POSTGRES_PR_CHECK_APP_PRIVATE_KEY`; App installed only on this repository with Metadata read, Checks write, and organization self-hosted-runners read; it has no Actions permission | @@ -499,7 +545,7 @@ only to that environment: | Proton Pass item | Protected environment fields | | --- | --- | | `PostgreSQL · Keycloak cohort source reader` | `KEYCLOAK_COHORT_SOURCE_TOKEN`, dedicated token/App broker restricted to `Makepad-fr/keycloak` with Metadata and Contents read only | -| `Hetzner Database Server makepad` | canonical `DEPLOY_SSH_*` fields mirrored to `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, and `KEYCLOAK_COHORT_DB_SSH_USER`; dedicated non-root Docker-capable DB capture account only | +| `PostgreSQL · Brio identity database deployment SSH` | canonical `DEPLOY_SSH_*` fields mirrored to `KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY`, `KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS`, `KEYCLOAK_COHORT_DB_SSH_HOST`, `KEYCLOAK_COHORT_DB_SSH_PORT`, and `KEYCLOAK_COHORT_DB_SSH_USER`; dedicated non-root Docker-capable DB capture account only | | `Makepad Docker Hardened Images` | canonical `DOCKERHUB_USERNAME` and `DOCKERHUB_PRO_PAT` fields mirrored to `DHI_REGISTRY_USERNAME` and `DHI_REGISTRY_PASSWORD`, with read-only pull access to the exact reviewed Keycloak image | The source token has no Actions write, Checks, Administration, Environments, diff --git a/deploy/credential-inventory.json b/deploy/credential-inventory.json index 2802145..a46267e 100644 --- a/deploy/credential-inventory.json +++ b/deploy/credential-inventory.json @@ -55,11 +55,11 @@ {"environment":"production","kind":"secret","requirement":"required","destination":"DEPLOY_VIF_DB_USER","item":"PostgreSQL · shared Swarm deployment","field":"DEPLOY_VIF_DB_USER"}, {"environment":"production","kind":"secret","requirement":"required","destination":"DEPLOY_VIF_DB_PASSWORD","item":"PostgreSQL · shared Swarm deployment","field":"DEPLOY_VIF_DB_PASSWORD"}, - {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_HOST","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_HOST"}, - {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_PORT","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_PORT"}, - {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_USER","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_USER"}, - {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_PRIVATE_KEY"}, - {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_KNOWN_HOSTS"}, + {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_HOST","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_HOST"}, + {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_PORT","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_PORT"}, + {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_USER","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_USER"}, + {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_PRIVATE_KEY","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_PRIVATE_KEY"}, + {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_KNOWN_HOSTS"}, {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"KEYCLOAK_BRIO_STAGING_DB_PASSWORD","item":"Brio Staging - PostgreSQL","field":"KEYCLOAK_BRIO_STAGING_DB_PASSWORD"}, {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD","item":"Brio Staging - PostgreSQL","field":"KEYCLOAK_BRIO_STAGING_BACKUP_DB_PASSWORD"}, {"environment":"staging-brio-identity-db","kind":"secret","requirement":"required","destination":"BRIO_BACKUP_RECIPIENT_CERT_PEM","item":"Brio Staging - PKI and Backup Keys","field":"BRIO_BACKUP_RECIPIENT_CERT_PEM"}, @@ -69,11 +69,11 @@ {"environment":"release-brio-identity-db","kind":"secret","requirement":"required","destination":"KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN","item":"PostgreSQL · Brio identity release orchestrator","field":"KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN"}, {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_SOURCE_TOKEN","item":"PostgreSQL · Keycloak cohort source reader","field":"KEYCLOAK_COHORT_SOURCE_TOKEN"}, - {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_HOST","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_HOST"}, - {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_PORT","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_PORT"}, - {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_USER","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_USER"}, - {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_PRIVATE_KEY"}, - {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS","item":"Hetzner Database Server makepad","field":"DEPLOY_SSH_KNOWN_HOSTS"}, + {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_HOST","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_HOST"}, + {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_PORT","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_PORT"}, + {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_USER","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_USER"}, + {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_PRIVATE_KEY","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_PRIVATE_KEY"}, + {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"KEYCLOAK_COHORT_DB_SSH_KNOWN_HOSTS","item":"PostgreSQL · Brio identity database deployment SSH","field":"DEPLOY_SSH_KNOWN_HOSTS"}, {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"DHI_REGISTRY_USERNAME","item":"Makepad Docker Hardened Images","field":"DOCKERHUB_USERNAME"}, {"environment":"keycloak-cohort-restore","kind":"secret","requirement":"required","destination":"DHI_REGISTRY_PASSWORD","item":"Makepad Docker Hardened Images","field":"DOCKERHUB_PRO_PAT"}, @@ -99,6 +99,10 @@ {"boundary":"host-root-file","requirement":"required","destination":"POSTGRES_HOST_ALERT_URL_FILE","item":"PostgreSQL · CI hypervisor alert","field":"url"}, {"boundary":"operator-process-auth","requirement":"required","destination":"scripts/sync-github-environments.sh --sync-repository-variables process-local GH_TOKEN","item":"PostgreSQL · GitHub repository variable bootstrap","field":"repository_variable_admin_token"}, {"boundary":"operator-verification","requirement":"required","destination":"PostgreSQL repository-variable bootstrap expiry","item":"PostgreSQL · GitHub repository variable bootstrap","field":"expires_at"}, - {"boundary":"operator-verification","requirement":"required","destination":"PostgreSQL repository-variable bootstrap owner","item":"PostgreSQL · GitHub repository variable bootstrap","field":"owner"} + {"boundary":"operator-verification","requirement":"required","destination":"PostgreSQL repository-variable bootstrap owner","item":"PostgreSQL · GitHub repository variable bootstrap","field":"owner"}, + {"boundary":"host-root-file","requirement":"required","destination":"/etc/makepad/brio-operation-lease/coordinator.json","item":"Brio · operation lease coordinator","field":"coordinator_json"}, + {"boundary":"host-root-file","requirement":"required","destination":"/etc/makepad/brio-operation-lease/id_ed25519","item":"Brio · operation lease coordinator","field":"ssh_private_key"}, + {"boundary":"host-root-file","requirement":"required","destination":"/etc/makepad/brio-operation-lease/known_hosts","item":"Brio · operation lease coordinator","field":"ssh_known_hosts"}, + {"boundary":"host-root-file","requirement":"required","destination":"/var/lib/makepad/brio-operation-lease-user/.ssh/authorized_keys","item":"Brio · operation lease coordinator","field":"ssh_public_key"} ] } diff --git a/docs/credential-sync.md b/docs/credential-sync.md index 32b9831..b90d5ca 100644 --- a/docs/credential-sync.md +++ b/docs/credential-sync.md @@ -146,7 +146,7 @@ workflow and are deliberately absent from the reviewed inventory. ### `staging-brio-identity-db` -- `Hetzner Database Server makepad` supplies canonical custom fields +- `PostgreSQL · Brio identity database deployment SSH` supplies canonical custom fields `DEPLOY_SSH_HOST`, `DEPLOY_SSH_PORT`, `DEPLOY_SSH_USER`, `DEPLOY_SSH_PRIVATE_KEY`, and `DEPLOY_SSH_KNOWN_HOSTS`, which map to their `BRIO_IDENTITY_DB_DEPLOY_SSH_*` aliases. @@ -174,7 +174,7 @@ permission to recreate a missing name. - `release-brio-identity-db` receives only the dedicated `KEYCLOAK_RELEASE_ORCHESTRATOR_TOKEN` secret. - `keycloak-cohort-restore` receives its read-only Keycloak source token, the - five DB-capture SSH aliases from `Hetzner Database Server makepad`, and the + five DB-capture SSH aliases from `PostgreSQL · Brio identity database deployment SSH`, and the DHI pull username/token. - `postgres-ci-attestation` receives only the Checks App private key. The Launcher App private key and the Ed25519 signing key never enter Actions. @@ -222,6 +222,12 @@ Actions mirror: workstation; it is never installed on a runner or hypervisor. - The host alert URL remains a root-only file consumed by the systemd failure handler and is never mirrored to Actions. +- `Brio · operation lease coordinator` is the canonical Proton item for the + fixed three-node coordinator JSON, its Ed25519 SSH key pair, and its pinned + known-hosts content. The private key, coordinator JSON, and known-hosts file + are installed mode `0600` under `/etc/makepad/brio-operation-lease`; only the + public key reaches the locked lease endpoint account. None of these fields is + a GitHub secret or variable. Install a private key from `pass-cli` directly into a newly created owner-only file using a trusted root process. Do not use a command-line argument, dotenv diff --git a/scripts/brio-operation-lease-coordinator.py b/scripts/brio-operation-lease-coordinator.py new file mode 100755 index 0000000..26c44fb --- /dev/null +++ b/scripts/brio-operation-lease-coordinator.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Acquire the three host-local Brio leases in one immutable order.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any, NoReturn + + +CONFIG_DIRECTORY = Path("/etc/makepad/brio-operation-lease") +CONFIG_PATH = CONFIG_DIRECTORY / "coordinator.json" +PRIVATE_KEY_PATH = CONFIG_DIRECTORY / "id_ed25519" +KNOWN_HOSTS_PATH = CONFIG_DIRECTORY / "known_hosts" +NODE_ORDER = ("app", "identity", "database") +OWNER_PATTERN = re.compile(r"^[0-9a-f]{64}$") +HOST_PATTERN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$") +RESULT_KEYS = {"expires_at", "kind", "node", "owner", "released_at", "state", "version"} + + +class CoordinatorError(RuntimeError): + """A bounded coordination operation failed closed.""" + + +def fail(message: str, status: int = 78) -> NoReturn: + print(f"brio operation coordinator: {message}", file=sys.stderr) + raise SystemExit(status) + + +def validate_regular(path: Path, mode: int, maximum_size: int, *, read_contents: bool = True) -> bytes: + try: + metadata = path.lstat() + except OSError as error: + raise CoordinatorError(f"required root configuration is unavailable: {type(error).__name__}") from error + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_gid != 0 + or stat.S_IMODE(metadata.st_mode) != mode + or metadata.st_nlink != 1 + or metadata.st_size < 1 + or metadata.st_size > maximum_size + ): + raise CoordinatorError("required root configuration has unsafe metadata") + if not read_contents: + return b"" + flags = os.O_RDONLY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino): + raise CoordinatorError("root configuration changed while opening") + return os.read(descriptor, maximum_size + 1) + finally: + os.close(descriptor) + + +def load_config() -> list[dict[str, Any]]: + directory = CONFIG_DIRECTORY.lstat() + if ( + not stat.S_ISDIR(directory.st_mode) + or stat.S_ISLNK(directory.st_mode) + or directory.st_uid != 0 + or directory.st_gid != 0 + or stat.S_IMODE(directory.st_mode) != 0o700 + ): + raise CoordinatorError("coordinator configuration directory is unsafe") + try: + value = json.loads(validate_regular(CONFIG_PATH, 0o600, 2048)) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise CoordinatorError("coordinator configuration is malformed") from error + if not isinstance(value, dict) or set(value) != {"nodes", "user", "version"}: + raise CoordinatorError("coordinator configuration schema is invalid") + if value["version"] != 1 or value["user"] != "brio-operation-lease": + raise CoordinatorError("coordinator identity is invalid") + nodes = value["nodes"] + if not isinstance(nodes, list) or len(nodes) != 3: + raise CoordinatorError("coordinator node inventory is invalid") + for offset, node in enumerate(nodes): + if not isinstance(node, dict) or set(node) != {"host", "name", "port"}: + raise CoordinatorError("coordinator node schema is invalid") + if ( + node["name"] != NODE_ORDER[offset] + or not isinstance(node["host"], str) + or HOST_PATTERN.fullmatch(node["host"]) is None + or not isinstance(node["port"], int) + or isinstance(node["port"], bool) + or not 1 <= node["port"] <= 65535 + ): + raise CoordinatorError("coordinator node value is invalid") + validate_regular(PRIVATE_KEY_PATH, 0o600, 16_384, read_contents=False) + validate_regular(KNOWN_HOSTS_PATH, 0o600, 65_536, read_contents=False) + return nodes + + +def validate_result(raw: str, expected_node: str) -> dict[str, Any]: + if not raw or len(raw.encode("utf-8")) > 1024 or raw.count("\n") > 1: + raise CoordinatorError("lease endpoint returned an invalid response size") + try: + value = json.loads(raw) + except json.JSONDecodeError as error: + raise CoordinatorError("lease endpoint returned malformed JSON") from error + if not isinstance(value, dict) or set(value) != RESULT_KEYS: + raise CoordinatorError("lease endpoint returned an unexpected schema") + if ( + value["version"] != 1 + or value["node"] != expected_node + or not isinstance(value["owner"], str) + or OWNER_PATTERN.fullmatch(value["owner"]) is None + or value["kind"] not in {"deployment", "evidence"} + or value["state"] not in {"absent", "acquired", "busy", "expired", "held", "released"} + or (value["expires_at"] is not None and (not isinstance(value["expires_at"], int) or isinstance(value["expires_at"], bool))) + or (value["released_at"] is not None and (not isinstance(value["released_at"], int) or isinstance(value["released_at"], bool))) + ): + raise CoordinatorError("lease endpoint returned invalid values") + return value + + +def invoke(node: dict[str, Any], action: str, owner: str, kind: str) -> tuple[int, dict[str, Any]]: + command = [ + "/usr/bin/ssh", + "-F", "/dev/null", + "-T", + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "PasswordAuthentication=no", + "-o", "KbdInteractiveAuthentication=no", + "-o", "StrictHostKeyChecking=yes", + "-o", f"UserKnownHostsFile={KNOWN_HOSTS_PATH}", + "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "ConnectTimeout=10", + "-o", "ConnectionAttempts=1", + "-i", str(PRIVATE_KEY_PATH), + "-p", str(node["port"]), + f"brio-operation-lease@{node['host']}", + f"{action} {owner} {kind}", + ] + try: + completed = subprocess.run( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=20, + check=False, + env={"LANG": "C", "LC_ALL": "C", "PATH": "/usr/bin:/bin"}, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise CoordinatorError(f"{node['name']} lease endpoint is unavailable") from error + result = validate_result(completed.stdout, node["name"]) + return completed.returncode, result + + +def emit(result: dict[str, Any]) -> None: + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + + +def release_nodes(nodes: list[dict[str, Any]], owner: str, kind: str) -> bool: + succeeded = True + for node in reversed(nodes): + try: + status, result = invoke(node, "release", owner, kind) + emit(result) + if status != 0 or result["owner"] != owner or result["kind"] != kind or result["state"] not in {"released", "absent", "expired"}: + succeeded = False + except CoordinatorError: + succeeded = False + return succeeded + + +def main(argv: list[str]) -> int: + if os.geteuid() != 0: + fail("must run as root", 77) + if len(argv) != 4 or argv[1] not in {"acquire", "status", "release"}: + fail("usage: brio-operation-lease-coordinator acquire|status|release OWNER deployment|evidence", 64) + action, owner, kind = argv[1:] + if OWNER_PATTERN.fullmatch(owner) is None or kind not in {"deployment", "evidence"}: + fail("arguments do not match the bounded coordinator grammar", 64) + try: + nodes = load_config() + if action == "release": + if not release_nodes(nodes, owner, kind): + raise CoordinatorError("one or more lease releases failed") + return 0 + if action == "status": + for node in nodes: + status, result = invoke(node, action, owner, kind) + emit(result) + if status != 0 or result["owner"] != owner or result["kind"] != kind or result["state"] != "held": + raise CoordinatorError(f"{node['name']} lease is not held by the requested owner") + return 0 + + for offset, node in enumerate(nodes): + attempted_nodes = nodes[: offset + 1] + try: + status, result = invoke(node, action, owner, kind) + emit(result) + if status == 75 and result["state"] == "busy" and ( + result["owner"] != owner or result["kind"] != kind + ): + attempted_nodes = nodes[:offset] + if status != 0 or result["owner"] != owner or result["kind"] != kind or result["state"] not in {"acquired", "held"}: + raise CoordinatorError(f"{node['name']} lease acquisition was rejected") + except CoordinatorError as acquisition_error: + if not release_nodes(attempted_nodes, owner, kind): + raise CoordinatorError( + f"{node['name']} lease acquisition failed and partial lease cleanup failed" + ) from acquisition_error + raise + return 0 + except (CoordinatorError, OSError) as error: + fail(str(error)) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/brio-operation-lease-dispatch.py b/scripts/brio-operation-lease-dispatch.py new file mode 100755 index 0000000..ee082f7 --- /dev/null +++ b/scripts/brio-operation-lease-dispatch.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Forced-command adapter for the root-owned Brio operation lease.""" + +from __future__ import annotations + +import os +import re +import sys + + +LEASE_EXECUTABLE = "/usr/local/libexec/makepad/brio-operation-lease" +COMMAND_PATTERN = re.compile( + r"^(acquire|status|release) ([0-9a-f]{64}) (deployment|evidence)$" +) + + +def main() -> int: + if len(sys.argv) != 1: + print("brio lease dispatch: arguments are forbidden", file=sys.stderr) + return 64 + original = os.environ.get("SSH_ORIGINAL_COMMAND", "") + if len(original) > 96 or "\n" in original or "\r" in original or "\x00" in original: + print("brio lease dispatch: command is outside the bounded grammar", file=sys.stderr) + return 64 + match = COMMAND_PATTERN.fullmatch(original) + if match is None: + print("brio lease dispatch: command is outside the bounded grammar", file=sys.stderr) + return 64 + os.execv( + "/usr/bin/sudo", + ["sudo", "-n", "--", LEASE_EXECUTABLE, *match.groups()], + ) + return 70 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/brio-operation-lease-remote.sh b/scripts/brio-operation-lease-remote.sh new file mode 100755 index 0000000..dfc43b5 --- /dev/null +++ b/scripts/brio-operation-lease-remote.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +die() { + printf 'Brio operation lease remote client: %s\n' "$1" >&2 + exit 64 +} + +[[ $# == 2 ]] || die 'usage: brio-operation-lease-remote.sh acquire|status|release OWNER' +readonly action=$1 +readonly owner=$2 +[[ "${action}" =~ ^(acquire|status|release)$ ]] || die 'action is invalid' +[[ "${owner}" =~ ^[0-9a-f]{64}$ ]] || die 'owner is invalid' + +: "${BRIO_LEASE_REMOTE_HOST:?set BRIO_LEASE_REMOTE_HOST}" +: "${BRIO_LEASE_REMOTE_USER:?set BRIO_LEASE_REMOTE_USER}" +: "${BRIO_LEASE_SSH_DIRECTORY:?set BRIO_LEASE_SSH_DIRECTORY}" +[[ "${BRIO_LEASE_REMOTE_HOST}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}[A-Za-z0-9]$ ]] || die 'remote host is invalid' +[[ "${BRIO_LEASE_REMOTE_USER}" =~ ^[a-z_][a-z0-9_-]{0,31}$ ]] || die 'remote user is invalid' +readonly remote_port=${BRIO_LEASE_REMOTE_PORT:-22} +if [[ ! "${remote_port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( remote_port > 65535 )); then + die 'remote port is invalid' +fi +[[ "${BRIO_LEASE_SSH_DIRECTORY}" == /* && -d "${BRIO_LEASE_SSH_DIRECTORY}" && ! -L "${BRIO_LEASE_SSH_DIRECTORY}" ]] || die 'SSH directory is invalid' +readonly private_key=${BRIO_LEASE_SSH_DIRECTORY}/id_ed25519 +readonly known_hosts=${BRIO_LEASE_SSH_DIRECTORY}/known_hosts +for source in "${private_key}" "${known_hosts}"; do + [[ -f "${source}" && ! -L "${source}" && -r "${source}" ]] || die 'SSH material is unavailable or unsafe' + [[ "$(stat -c '%a' "${source}")" == 600 ]] || die 'SSH material must use mode 0600' +done + +exec /usr/bin/ssh \ + -F /dev/null -T \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o PasswordAuthentication=no \ + -o KbdInteractiveAuthentication=no \ + -o StrictHostKeyChecking=yes \ + -o "UserKnownHostsFile=${known_hosts}" \ + -o GlobalKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o ConnectionAttempts=1 \ + -i "${private_key}" \ + -p "${remote_port}" \ + "${BRIO_LEASE_REMOTE_USER}@${BRIO_LEASE_REMOTE_HOST}" \ + "/usr/bin/sudo -n /usr/local/libexec/makepad/brio-operation-lease-coordinator ${action} ${owner} deployment" diff --git a/scripts/brio-operation-lease.py b/scripts/brio-operation-lease.py new file mode 100755 index 0000000..a9be806 --- /dev/null +++ b/scripts/brio-operation-lease.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Root-owned, host-local exclusion lease for Brio mutations and evidence.""" + +from __future__ import annotations + +import fcntl +import json +import os +import re +import stat +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Callable, NoReturn + + +RUNTIME_DIRECTORY = Path("/run/makepad/brio-operation-lease") +GUARD_PATH = RUNTIME_DIRECTORY / "guard" +LEASE_PATH = RUNTIME_DIRECTORY / "lease" +NODE_PATH = Path("/etc/makepad/brio-operation-lease-node") +TTL_SECONDS = 14_400 +OWNER_PATTERN = re.compile(r"^[0-9a-f]{64}$") +KINDS = {"deployment", "evidence"} +ACTIONS = {"acquire", "status", "release"} +NODES = {"app", "identity", "database"} +STATE_KEYS = {"acquired_at", "expires_at", "kind", "node", "owner", "version"} + + +class LeaseError(RuntimeError): + """A fail-closed local lease validation or operation error.""" + + +def fail(message: str, status: int = 78) -> NoReturn: + print(f"brio operation lease: {message}", file=sys.stderr) + raise SystemExit(status) + + +def canonical_result( + *, node: str, owner: str, kind: str, state: str, + expires_at: int | None, released_at: int | None, +) -> str: + return json.dumps( + { + "expires_at": expires_at, + "kind": kind, + "node": node, + "owner": owner, + "released_at": released_at, + "state": state, + "version": 1, + }, + sort_keys=True, + separators=(",", ":"), + ) + + +class LeaseStore: + def __init__( + self, + runtime_directory: Path = RUNTIME_DIRECTORY, + node_path: Path = NODE_PATH, + *, + expected_uid: int = 0, + expected_gid: int = 0, + now: Callable[[], int] | None = None, + ) -> None: + self.runtime_directory = runtime_directory + self.guard_path = runtime_directory / "guard" + self.lease_path = runtime_directory / "lease" + self.node_path = node_path + self.expected_uid = expected_uid + self.expected_gid = expected_gid + self.now = now or (lambda: int(time.time())) + + def _validate_metadata(self, path: Path, *, directory: bool, mode: int) -> os.stat_result: + try: + metadata = path.lstat() + except OSError as error: + raise LeaseError(f"required local object is unavailable: {type(error).__name__}") from error + expected_type = stat.S_ISDIR if directory else stat.S_ISREG + if not expected_type(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise LeaseError("required local object has an unsafe type") + if metadata.st_uid != self.expected_uid or metadata.st_gid != self.expected_gid: + raise LeaseError("required local object has unsafe ownership") + if stat.S_IMODE(metadata.st_mode) != mode: + raise LeaseError("required local object has unsafe permissions") + if not directory and metadata.st_nlink != 1: + raise LeaseError("required local file has an unsafe link count") + return metadata + + def _read_node(self) -> str: + metadata = self._validate_metadata(self.node_path, directory=False, mode=0o600) + if metadata.st_size > 16: + raise LeaseError("node identity is oversized") + flags = os.O_RDONLY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(self.node_path, flags) + try: + observed = os.fstat(descriptor) + if (observed.st_dev, observed.st_ino) != (metadata.st_dev, metadata.st_ino): + raise LeaseError("node identity changed while opening") + raw = os.read(descriptor, 17) + finally: + os.close(descriptor) + if raw not in {f"{node}\n".encode("ascii") for node in NODES}: + raise LeaseError("node identity is invalid") + return raw[:-1].decode("ascii") + + def _read_lease(self, node: str) -> dict[str, Any] | None: + try: + metadata = self.lease_path.lstat() + except FileNotFoundError: + return None + except OSError as error: + raise LeaseError(f"lease state is unavailable: {type(error).__name__}") from error + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != self.expected_uid + or metadata.st_gid != self.expected_gid + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or metadata.st_size > 512 + ): + raise LeaseError("lease state metadata is unsafe") + flags = os.O_RDONLY | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(self.lease_path, flags) + try: + observed = os.fstat(descriptor) + if (observed.st_dev, observed.st_ino) != (metadata.st_dev, metadata.st_ino): + raise LeaseError("lease state changed while opening") + raw = os.read(descriptor, 513) + finally: + os.close(descriptor) + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise LeaseError("lease state is malformed") from error + if not isinstance(value, dict) or set(value) != STATE_KEYS: + raise LeaseError("lease state has an unexpected schema") + if ( + value["version"] != 1 + or value["node"] != node + or not isinstance(value["owner"], str) + or OWNER_PATTERN.fullmatch(value["owner"]) is None + or value["kind"] not in KINDS + or not isinstance(value["acquired_at"], int) + or isinstance(value["acquired_at"], bool) + or not isinstance(value["expires_at"], int) + or isinstance(value["expires_at"], bool) + or value["acquired_at"] < 0 + or value["expires_at"] - value["acquired_at"] != TTL_SECONDS + ): + raise LeaseError("lease state values are invalid") + return value + + def _replace_lease(self, encoded: bytes) -> None: + descriptor, temporary_name = tempfile.mkstemp(prefix=".lease-", dir=self.runtime_directory) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + written = 0 + while written < len(encoded): + written += os.write(descriptor, encoded[written:]) + os.fsync(descriptor) + observed = os.fstat(descriptor) + if ( + observed.st_uid != self.expected_uid + or observed.st_gid != self.expected_gid + or stat.S_IMODE(observed.st_mode) != 0o600 + or observed.st_nlink != 1 + ): + raise LeaseError("temporary lease state metadata is unsafe") + os.close(descriptor) + descriptor = -1 + os.replace(temporary, self.lease_path) + directory_descriptor = os.open(self.runtime_directory, os.O_RDONLY | os.O_CLOEXEC) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + finally: + if descriptor >= 0: + os.close(descriptor) + try: + temporary.unlink() + except FileNotFoundError: + pass + + def _write_lease(self, value: dict[str, Any]) -> None: + encoded = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + self._replace_lease(encoded) + + def _remove_lease(self) -> None: + try: + self.lease_path.unlink() + except FileNotFoundError as error: + raise LeaseError("lease state disappeared while held") from error + directory_descriptor = os.open(self.runtime_directory, os.O_RDONLY | os.O_CLOEXEC) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + + def operate(self, action: str, owner: str, kind: str) -> tuple[int, str]: + self._validate_metadata(self.runtime_directory, directory=True, mode=0o700) + node = self._read_node() + guard_metadata = self._validate_metadata(self.guard_path, directory=False, mode=0o600) + flags = os.O_RDWR | os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + guard_descriptor = os.open(self.guard_path, flags) + try: + opened_guard = os.fstat(guard_descriptor) + if (opened_guard.st_dev, opened_guard.st_ino) != (guard_metadata.st_dev, guard_metadata.st_ino): + raise LeaseError("lease guard changed while opening") + fcntl.flock(guard_descriptor, fcntl.LOCK_EX) + now = self.now() + if not isinstance(now, int) or isinstance(now, bool) or now < 0: + raise LeaseError("system clock is invalid") + active = self._read_lease(node) + if active is not None and ( + active["acquired_at"] > now or active["expires_at"] > now + TTL_SECONDS + ): + raise LeaseError("lease state is inconsistent with the system clock") + expired: dict[str, Any] | None = None + if active is not None and active["expires_at"] <= now: + expired = active + self._remove_lease() + active = None + + if action == "acquire": + if active is not None: + if active["owner"] == owner and active["kind"] == kind: + return 0, canonical_result( + node=node, owner=owner, kind=kind, state="held", + expires_at=active["expires_at"], released_at=None, + ) + return 75, canonical_result( + node=node, owner=active["owner"], kind=active["kind"], state="busy", + expires_at=active["expires_at"], released_at=None, + ) + value = { + "acquired_at": now, + "expires_at": now + TTL_SECONDS, + "kind": kind, + "node": node, + "owner": owner, + "version": 1, + } + self._write_lease(value) + return 0, canonical_result( + node=node, owner=owner, kind=kind, state="acquired", + expires_at=value["expires_at"], released_at=None, + ) + + if action == "status": + if active is not None: + if active["owner"] == owner and active["kind"] == kind: + return 0, canonical_result( + node=node, owner=owner, kind=kind, state="held", + expires_at=active["expires_at"], released_at=None, + ) + return 75, canonical_result( + node=node, owner=active["owner"], kind=active["kind"], state="busy", + expires_at=active["expires_at"], released_at=None, + ) + return 3, canonical_result( + node=node, owner=owner, kind=kind, + state="expired" if expired is not None else "absent", + expires_at=expired["expires_at"] if expired is not None else None, + released_at=now, + ) + + if active is not None: + if active["owner"] != owner or active["kind"] != kind: + return 75, canonical_result( + node=node, owner=active["owner"], kind=active["kind"], state="busy", + expires_at=active["expires_at"], released_at=None, + ) + self._remove_lease() + return 0, canonical_result( + node=node, owner=owner, kind=kind, state="released", + expires_at=active["expires_at"], released_at=now, + ) + return 0, canonical_result( + node=node, owner=owner, kind=kind, + state="expired" if expired is not None else "absent", + expires_at=expired["expires_at"] if expired is not None else None, + released_at=now, + ) + finally: + os.close(guard_descriptor) + + +def main(argv: list[str]) -> int: + if os.geteuid() != 0: + fail("must run as root", 77) + if len(argv) != 4 or argv[1] not in ACTIONS: + fail("usage: brio-operation-lease acquire|status|release OWNER deployment|evidence", 64) + action, owner, kind = argv[1:] + if OWNER_PATTERN.fullmatch(owner) is None or kind not in KINDS: + fail("arguments do not match the bounded lease grammar", 64) + try: + status_code, output = LeaseStore().operate(action, owner, kind) + except (LeaseError, OSError) as error: + fail(str(error)) + print(output) + return status_code + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/deploy-brio-canary-postgres.sh b/scripts/deploy-brio-canary-postgres.sh index e3326b2..3f97b3e 100755 --- a/scripts/deploy-brio-canary-postgres.sh +++ b/scripts/deploy-brio-canary-postgres.sh @@ -44,6 +44,15 @@ fi [[ "${runtime_dir}" =~ ^/tmp/postgres-brio-canary-runtime-[0-9]+-[0-9]+$ ]] || { echo "runtime-secret-dir must be a job-scoped /tmp/postgres-brio-canary-runtime-- path." >&2; exit 2; } case "${stack_name}" in ''|*[!a-zA-Z0-9_-]*) echo "stack-name contains unsupported characters." >&2; exit 2 ;; esac +require_brio_deployment_lease() { + [[ "${BRIO_OPERATION_LEASE_OWNER:-}" =~ ^[0-9a-f]{64}$ ]] || { + echo "BRIO_OPERATION_LEASE_OWNER must be the exact deployment lease owner." >&2 + exit 1 + } + /usr/bin/sudo -n /usr/local/libexec/makepad/brio-operation-lease \ + status "${BRIO_OPERATION_LEASE_OWNER}" deployment >/dev/null +} + cleanup_runtime() { local name for name in postgres-superuser-password brio-staging-app-password brio-staging-backup-password postgres-ca.pem postgres-server-cert.pem postgres-server-key.pem brio-backup-recipient-cert.pem; do @@ -510,6 +519,7 @@ docker network inspect "${db_network}" >/dev/null 2>&1 || { # A SIGKILL can leave immutable objects created by the interrupted run. Recover # its root-owned journal before inventorying those objects so the current # attempt cannot mistake interrupted-run state for its own validated pre-state. +require_brio_deployment_lease recover_incomplete_journals prevalidate_swarm_config "${tls_cert_config}" "${runtime_dir}/postgres-server-cert.pem" "TLS certificate" diff --git a/scripts/deploy-brio-identity-db-host.sh b/scripts/deploy-brio-identity-db-host.sh index 7b4e032..7a10a5f 100755 --- a/scripts/deploy-brio-identity-db-host.sh +++ b/scripts/deploy-brio-identity-db-host.sh @@ -62,6 +62,15 @@ fi exit 2 } +require_brio_deployment_lease() { + [[ "${BRIO_OPERATION_LEASE_OWNER:-}" =~ ^[0-9a-f]{64}$ ]] || { + echo "BRIO_OPERATION_LEASE_OWNER must be the exact deployment lease owner." >&2 + exit 1 + } + /usr/bin/sudo -n /usr/local/libexec/makepad/brio-operation-lease \ + status "${BRIO_OPERATION_LEASE_OWNER}" deployment >/dev/null +} + read_setting() { local name=$1 file=$2 value value=$(grep -E "^${name}=" "${file}" | tail -n 1 | cut -d= -f2-) @@ -499,6 +508,7 @@ openssl x509 -in "${server_cert_host_file}" -noout -checkend 604800 >/dev/null openssl x509 -in "${server_cert_host_file}" -noout -checkip "${db_hostname}" >/dev/null openssl verify -purpose sslserver -CAfile "${ca_host_file}" "${server_cert_host_file}" >/dev/null cert_key_hash=$(openssl x509 -in "${server_cert_host_file}" -pubkey -noout | openssl pkey -pubin -outform DER | sha256sum | cut -d' ' -f1) +require_brio_deployment_lease private_key_hash=$(docker run --rm --mount "type=bind,src=${server_key_host_file},dst=/runtime/server.key,readonly" \ "${validation_image}" openssl pkey -in /runtime/server.key -pubout -outform DER | sha256sum | cut -d' ' -f1) [[ "${cert_key_hash}" == "${private_key_hash}" ]] || { echo "Existing DB-VM server certificate and key do not match." >&2; exit 1; } @@ -551,6 +561,7 @@ fi prior_container_id=$(validate_postgres_target "${postgres_image}") prior_postgres_image=$(docker container inspect "${prior_container_id}" --format '{{.Config.Image}}') +require_brio_deployment_lease docker pull "${postgres_image}" >/dev/null docker pull "${validation_image}" >/dev/null diff --git a/scripts/deploy-postgres-stack.sh b/scripts/deploy-postgres-stack.sh index 0ba486f..4bfa27a 100755 --- a/scripts/deploy-postgres-stack.sh +++ b/scripts/deploy-postgres-stack.sh @@ -27,6 +27,15 @@ elif [[ $# -ne 3 ]]; then exit 2 fi +require_brio_deployment_lease() { + [[ "${BRIO_OPERATION_LEASE_OWNER:-}" =~ ^[0-9a-f]{64}$ ]] || { + echo "BRIO_OPERATION_LEASE_OWNER must be the exact deployment lease owner." >&2 + exit 1 + } + /usr/bin/sudo -n /usr/local/libexec/makepad/brio-operation-lease \ + status "${BRIO_OPERATION_LEASE_OWNER}" deployment >/dev/null +} + server_certificate= cleanup_deploy_material() { [[ -z "${server_certificate}" ]] || rm -f -- "${server_certificate}" @@ -260,6 +269,9 @@ ensure_internal_encrypted_overlay_network() { fi } +# This is the final gate before the first network creation or other provider +# mutation. Status is idempotent and never renews the fixed four-hour lease. +require_brio_deployment_lease ensure_encrypted_overlay_network "${db_network}" ensure_encrypted_overlay_network "${le_petit_coin_db_network}" if [[ "${vif_enabled}" == "1" ]]; then diff --git a/scripts/derive-brio-operation-owner.py b/scripts/derive-brio-operation-owner.py new file mode 100755 index 0000000..a1f91ed --- /dev/null +++ b/scripts/derive-brio-operation-owner.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Derive the public lease owner from immutable GitHub run identity.""" + +from __future__ import annotations + +import hashlib +import os +import re +import sys + + +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +RUN_NUMBER = re.compile(r"^[1-9][0-9]*$") +SHA = re.compile(r"^[0-9a-f]{40}$") + + +def required(name: str, pattern: re.Pattern[str]) -> str: + value = os.environ.get(name, "") + if pattern.fullmatch(value) is None: + raise SystemExit(f"Brio operation owner: {name} is invalid") + return value + + +def main(argv: list[str]) -> int: + if len(argv) != 2 or argv[1] not in {"deployment", "evidence"}: + raise SystemExit("usage: derive-brio-operation-owner.py deployment|evidence") + kind = argv[1] + run_id = required("GITHUB_RUN_ID", RUN_NUMBER) + sha = required("GITHUB_SHA", SHA) + if kind == "deployment": + repository = required("GITHUB_REPOSITORY", REPOSITORY) + run_attempt = required("GITHUB_RUN_ATTEMPT", RUN_NUMBER) + identity = f"deployment\0{repository}\0{run_id}\0{run_attempt}\0{sha}" + else: + identity = f"evidence\0{run_id}\0{sha}" + print(hashlib.sha256(identity.encode("ascii")).hexdigest()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/fixtures/brio-deployment-failure-fixture.sh b/scripts/fixtures/brio-deployment-failure-fixture.sh index d00bcf0..a0d3e41 100755 --- a/scripts/fixtures/brio-deployment-failure-fixture.sh +++ b/scripts/fixtures/brio-deployment-failure-fixture.sh @@ -4,6 +4,22 @@ set -euo pipefail repo=/repo mock_bin=/tmp/brio-mock-bin install -d -m 0700 "${mock_bin}" +export BRIO_OPERATION_LEASE_OWNER=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + +# Production hardcodes /usr/bin/sudo and the root-owned lease executable. This +# isolated container replaces sudo with an exact-grammar status stub so every +# failure scenario still proves that it crossed the lease gate first. +cat > /usr/bin/sudo <<'MOCK_SUDO' +#!/usr/bin/env bash +set -euo pipefail +[[ $# == 5 && "$1" == -n \ + && "$2" == /usr/local/libexec/makepad/brio-operation-lease \ + && "$3" == status \ + && "$4" =~ ^[0-9a-f]{64}$ \ + && "$5" == deployment ]] +printf '%s\n' "$4" >> /tmp/mock-brio-operation-lease-status.log +MOCK_SUDO +chmod 0755 /usr/bin/sudo cat > "${mock_bin}/docker" <<'MOCK' #!/usr/bin/env bash diff --git a/scripts/install-brio-operation-lease.sh b/scripts/install-brio-operation-lease.sh new file mode 100755 index 0000000..12fd951 --- /dev/null +++ b/scripts/install-brio-operation-lease.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly expected_node=database +readonly lease_user=brio-operation-lease +readonly lease_home=/var/lib/makepad/brio-operation-lease-user +readonly executable_directory=/usr/local/libexec/makepad +readonly lease_executable=${executable_directory}/brio-operation-lease +readonly dispatch_executable=${executable_directory}/brio-operation-lease-dispatch +readonly coordinator_executable=${executable_directory}/brio-operation-lease-coordinator +readonly config_directory=/etc/makepad/brio-operation-lease +readonly node_path=/etc/makepad/brio-operation-lease-node +readonly tmpfiles_path=/etc/tmpfiles.d/makepad-brio-operation-lease.conf +readonly sudoers_path=/etc/sudoers.d/brio-operation-lease-postgres +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +readonly repo_root + +die() { + printf 'Brio operation lease installer: %s\n' "$1" >&2 + exit 1 +} + +(( EUID == 0 )) || die 'run this installer as root on the PostgreSQL host' +[[ $# == 5 ]] || die 'usage: install-brio-operation-lease.sh DEPLOY_USER PUBLIC_KEY_FILE COORDINATOR_CONFIG PRIVATE_KEY_FILE KNOWN_HOSTS_FILE' +deploy_user=$1 +public_key_file=$2 +coordinator_config=$3 +private_key_file=$4 +known_hosts_file=$5 +[[ "${deploy_user}" =~ ^[a-z_][a-z0-9_-]{0,31}$ ]] || die 'deployment user is invalid' +id "${deploy_user}" >/dev/null 2>&1 || die 'deployment user does not exist' +[[ ! -e /run/makepad/brio-operation-lease/lease && ! -L /run/makepad/brio-operation-lease/lease ]] || \ + die 'install only when no Brio operation lease is active' + +for source_file in \ + "${repo_root}/scripts/brio-operation-lease.py" \ + "${repo_root}/scripts/brio-operation-lease-dispatch.py" \ + "${repo_root}/scripts/brio-operation-lease-coordinator.py" \ + "${public_key_file}" "${coordinator_config}" "${private_key_file}" "${known_hosts_file}"; do + [[ -f "${source_file}" && ! -L "${source_file}" ]] || die 'every installation source must be a regular non-symlink file' +done + +read -r key_type key_body key_extra <"${public_key_file}" +[[ "${key_type}" == ssh-ed25519 && "${key_body}" =~ ^[A-Za-z0-9+/]+={0,3}$ && -z "${key_extra:-}" ]] || \ + die 'lease endpoint public key must be one comment-free Ed25519 key' +[[ $(wc -l <"${public_key_file}") -eq 1 ]] || die 'lease endpoint public key must contain exactly one line' +grep -Fqx -- '-----BEGIN OPENSSH PRIVATE KEY-----' <(head -n 1 "${private_key_file}") || \ + die 'coordinator private key must be OpenSSH format' + +python3 - "${coordinator_config}" <<'PY' +import json +import pathlib +import re +import sys + +value = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +if not isinstance(value, dict) or set(value) != {"nodes", "user", "version"}: + raise SystemExit("invalid coordinator configuration schema") +if value["version"] != 1 or value["user"] != "brio-operation-lease": + raise SystemExit("invalid coordinator identity") +nodes = value["nodes"] +if not isinstance(nodes, list) or [entry.get("name") for entry in nodes if isinstance(entry, dict)] != ["app", "identity", "database"]: + raise SystemExit("coordinator nodes must use app, identity, database order") +host = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$") +for entry in nodes: + if set(entry) != {"host", "name", "port"} or not isinstance(entry["host"], str) or host.fullmatch(entry["host"]) is None: + raise SystemExit("invalid coordinator node") + if not isinstance(entry["port"], int) or isinstance(entry["port"], bool) or not 1 <= entry["port"] <= 65535: + raise SystemExit("invalid coordinator port") +PY + +if ! id "${lease_user}" >/dev/null 2>&1; then + useradd --system --create-home --home-dir "${lease_home}" --shell /bin/bash "${lease_user}" +fi +passwd --lock "${lease_user}" >/dev/null + +install -d -o root -g root -m 0755 /usr/local/libexec "${executable_directory}" /etc/makepad +install -d -o root -g root -m 0700 "${config_directory}" +install -o root -g root -m 0755 "${repo_root}/scripts/brio-operation-lease.py" "${lease_executable}" +install -o root -g root -m 0755 "${repo_root}/scripts/brio-operation-lease-dispatch.py" "${dispatch_executable}" +install -o root -g root -m 0755 "${repo_root}/scripts/brio-operation-lease-coordinator.py" "${coordinator_executable}" + +temporary_directory=$(mktemp -d) +[[ -d "${temporary_directory}" && ! -L "${temporary_directory}" ]] || die 'could not create private installation staging' +cleanup() { + find "${temporary_directory}" -depth -mindepth 1 -delete + rmdir "${temporary_directory}" +} +trap cleanup EXIT +printf '%s\n' "${expected_node}" >"${temporary_directory}/node" +printf '%s\n' \ + 'd /run/makepad 0755 root root -' \ + 'd /run/makepad/brio-operation-lease 0700 root root -' \ + 'f /run/makepad/brio-operation-lease/guard 0600 root root -' \ + >"${temporary_directory}/tmpfiles" +printf 'restrict,command="/usr/bin/sudo -n %s" %s %s\n' \ + "${dispatch_executable}" "${key_type}" "${key_body}" >"${temporary_directory}/authorized_keys" +printf 'Defaults!%s env_keep += "SSH_ORIGINAL_COMMAND"\n' "${dispatch_executable}" >"${temporary_directory}/sudoers" +printf '%s ALL=(root) NOPASSWD: %s\n' "${lease_user}" "${dispatch_executable}" >>"${temporary_directory}/sudoers" +printf '%s ALL=(root) NOPASSWD: %s *, %s status * deployment\n' \ + "${deploy_user}" "${coordinator_executable}" "${lease_executable}" >>"${temporary_directory}/sudoers" +chmod 0440 "${temporary_directory}/sudoers" +visudo -cf "${temporary_directory}/sudoers" >/dev/null + +install -o root -g root -m 0600 "${temporary_directory}/node" "${node_path}" +install -o root -g root -m 0600 "${coordinator_config}" "${config_directory}/coordinator.json" +install -o root -g root -m 0600 "${private_key_file}" "${config_directory}/id_ed25519" +install -o root -g root -m 0600 "${known_hosts_file}" "${config_directory}/known_hosts" +install -o root -g root -m 0644 "${temporary_directory}/tmpfiles" "${tmpfiles_path}" +install -o root -g root -m 0440 "${temporary_directory}/sudoers" "${sudoers_path}" +install -d -o "${lease_user}" -g "${lease_user}" -m 0700 "${lease_home}/.ssh" +install -o "${lease_user}" -g "${lease_user}" -m 0600 \ + "${temporary_directory}/authorized_keys" "${lease_home}/.ssh/authorized_keys" +systemd-tmpfiles --create "${tmpfiles_path}" + +[[ "$(stat -c '%U:%G:%a' /run/makepad/brio-operation-lease)" == root:root:700 ]] || die 'runtime directory metadata is unsafe' +[[ "$(stat -c '%U:%G:%a' /run/makepad/brio-operation-lease/guard)" == root:root:600 ]] || die 'runtime guard metadata is unsafe' +[[ ! -L /run/makepad/brio-operation-lease/guard ]] || die 'runtime guard must not be a symlink' +[[ ! -e /run/makepad/brio-operation-lease/lease && ! -L /run/makepad/brio-operation-lease/lease ]] || \ + die 'install only when no Brio operation lease is active' +for installed in "${node_path}" "${config_directory}/coordinator.json" "${config_directory}/id_ed25519" "${config_directory}/known_hosts"; do + [[ "$(stat -c '%U:%G:%a' "${installed}")" == root:root:600 && ! -L "${installed}" ]] || die 'installed root configuration metadata is unsafe' +done + +printf 'Installed the Brio operation lease endpoint and coordinator for node %s.\n' "${expected_node}" diff --git a/scripts/run-ci.sh b/scripts/run-ci.sh index 13f918b..eca3c2e 100755 --- a/scripts/run-ci.sh +++ b/scripts/run-ci.sh @@ -19,6 +19,8 @@ shellcheck \ scripts/install-keycloak-cohort-capture-host.sh \ scripts/brio-runtime-observe.sh \ scripts/install-brio-runtime-observer.sh \ + scripts/brio-operation-lease-remote.sh \ + scripts/install-brio-operation-lease.sh \ scripts/verify-brio-encrypted-restore.sh \ scripts/test-brio-bootstrap.sh \ scripts/test-brio-db-transaction.sh \ @@ -52,6 +54,10 @@ for source in ( "scripts/ci-base-image.py", "scripts/verify-postgres-ci-jit-result.py", "scripts/brio-postgres-control-receipt.py", + "scripts/brio-operation-lease.py", + "scripts/brio-operation-lease-dispatch.py", + "scripts/brio-operation-lease-coordinator.py", + "scripts/derive-brio-operation-owner.py", "scripts/test-brio-postgres-control-receipt.py", "scripts/reconcile-github-environment-main-policy.py", "scripts/test-github-environment-main-policy.py", @@ -61,6 +67,8 @@ for source in ( ): ast.parse(Path(source).read_text(), filename=source) PY +PYTHONDONTWRITEBYTECODE=1 python3 tests/test_brio_operation_lease.py +PYTHONDONTWRITEBYTECODE=1 python3 tests/test_brio_operation_lease_wiring.py PYTHONDONTWRITEBYTECODE=1 python3 scripts/test-github-environment-main-policy.py ./scripts/test-sync-github-environments.sh node --check scripts/publish-pr-ci-check.mjs diff --git a/scripts/test-sync-github-environments.sh b/scripts/test-sync-github-environments.sh index 32fdef7..4a845a3 100755 --- a/scripts/test-sync-github-environments.sh +++ b/scripts/test-sync-github-environments.sh @@ -273,8 +273,8 @@ assert_no_value_read_or_write() { jq -e ' .schemaVersion == 2 and ([.githubEntries[] | select((.environment == "canary" or .environment == "production") and (.destination | startswith("DEPLOY_SSH_"))) | .item] | unique) == ["Hetzner App Server makepad"] and - ([.githubEntries[] | select((.environment == "staging-brio-identity-db" or .environment == "keycloak-cohort-restore") and (.destination | test("SSH_(HOST|PORT|USER|PRIVATE_KEY|KNOWN_HOSTS)$"))) | .item] | unique) == ["Hetzner Database Server makepad"] and - ([.githubEntries[] | select(.item == "Hetzner App Server makepad" or .item == "Hetzner Database Server makepad") | {destination, field}] | unique | sort_by(.destination)) == ([ + ([.githubEntries[] | select((.environment == "staging-brio-identity-db" or .environment == "keycloak-cohort-restore") and (.destination | test("SSH_(HOST|PORT|USER|PRIVATE_KEY|KNOWN_HOSTS)$"))) | .item] | unique) == ["PostgreSQL · Brio identity database deployment SSH"] and + ([.githubEntries[] | select(.item == "Hetzner App Server makepad" or .item == "PostgreSQL · Brio identity database deployment SSH") | {destination, field}] | unique | sort_by(.destination)) == ([ {"destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_HOST","field":"DEPLOY_SSH_HOST"}, {"destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_KNOWN_HOSTS","field":"DEPLOY_SSH_KNOWN_HOSTS"}, {"destination":"BRIO_IDENTITY_DB_DEPLOY_SSH_PORT","field":"DEPLOY_SSH_PORT"}, diff --git a/scripts/validate-credential-inventory-contract.py b/scripts/validate-credential-inventory-contract.py index acc6605..b0da3b9 100755 --- a/scripts/validate-credential-inventory-contract.py +++ b/scripts/validate-credential-inventory-contract.py @@ -85,7 +85,7 @@ def github(item: str, **destinations: str) -> set[tuple[str, str, str, str]]: ), "staging-brio-identity-db": ( github( - "Hetzner Database Server makepad", + "PostgreSQL · Brio identity database deployment SSH", BRIO_IDENTITY_DB_DEPLOY_SSH_HOST="DEPLOY_SSH_HOST", BRIO_IDENTITY_DB_DEPLOY_SSH_PORT="DEPLOY_SSH_PORT", BRIO_IDENTITY_DB_DEPLOY_SSH_USER="DEPLOY_SSH_USER", @@ -126,7 +126,7 @@ def github(item: str, **destinations: str) -> set[tuple[str, str, str, str]]: KEYCLOAK_COHORT_SOURCE_TOKEN="KEYCLOAK_COHORT_SOURCE_TOKEN", ) | github( - "Hetzner Database Server makepad", + "PostgreSQL · Brio identity database deployment SSH", KEYCLOAK_COHORT_DB_SSH_HOST="DEPLOY_SSH_HOST", KEYCLOAK_COHORT_DB_SSH_PORT="DEPLOY_SSH_PORT", KEYCLOAK_COHORT_DB_SSH_USER="DEPLOY_SSH_USER", @@ -190,6 +190,10 @@ def github(item: str, **destinations: str) -> set[tuple[str, str, str, str]]: } EXPECTED_NON_GITHUB_ENTRIES = { + ("host-root-file", "/etc/makepad/brio-operation-lease/coordinator.json", "Brio · operation lease coordinator", "coordinator_json"), + ("host-root-file", "/etc/makepad/brio-operation-lease/id_ed25519", "Brio · operation lease coordinator", "ssh_private_key"), + ("host-root-file", "/etc/makepad/brio-operation-lease/known_hosts", "Brio · operation lease coordinator", "ssh_known_hosts"), + ("host-root-file", "/var/lib/makepad/brio-operation-lease-user/.ssh/authorized_keys", "Brio · operation lease coordinator", "ssh_public_key"), ("operator-verification", "PostgreSQL Checks App private-key fingerprint", "PostgreSQL · PR Checks App", "private_key_fingerprint"), ("host-root-setting", "/etc/makepad/postgres-ci/controller.env:POSTGRES_CI_LAUNCHER_APP_ID", "PostgreSQL · JIT Launcher App", "app_id"), ("host-root-setting", "/etc/makepad/postgres-ci/controller.env:POSTGRES_CI_LAUNCHER_APP_INSTALLATION_ID", "PostgreSQL · JIT Launcher App", "installation_id"), diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 7baa910..3bce3c5 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -825,7 +825,7 @@ require( ) require(all(entry.get("boundary") in {"host-root-file", "host-root-setting", "operator-stdin", "operator-verification", "operator-process-auth"} for entry in non_github_entries), "Non-GitHub credential boundary is invalid.") for canonical_item in { - "Hetzner App Server makepad", "Hetzner Database Server makepad", + "Hetzner App Server makepad", "PostgreSQL · Brio identity database deployment SSH", "Brio Staging - PostgreSQL", "Le Petit Coin GitHub Deploy Secrets", "PostgreSQL · shared Swarm deployment", pki_item, "PostgreSQL · Brio identity release orchestrator", @@ -833,6 +833,7 @@ for canonical_item in { "PostgreSQL · PR Checks App", "PostgreSQL · JIT Launcher App", "PostgreSQL · JIT hypervisor attestation", "PostgreSQL · GitHub repository variable bootstrap", + "Brio · operation lease coordinator", }: require(canonical_item in readme, f"README credential inventory is missing canonical Proton item {canonical_item}.") ssh_source_by_environment = { @@ -854,7 +855,7 @@ app_native_ssh_field_by_suffix = { "PRIVATE_KEY": "private_key", "KNOWN_HOSTS": "known_hosts", } for entry in github_entries: - if entry.get("item") not in {"Hetzner App Server makepad", "Hetzner Database Server makepad"}: + if entry.get("item") not in {"Hetzner App Server makepad", "PostgreSQL · Brio identity database deployment SSH"}: continue suffix = next((suffix for suffix in app_native_ssh_field_by_suffix if entry["destination"].endswith(f"SSH_{suffix}")), None) require(suffix is not None, f"Unexpected SSH destination {entry['destination']}.") @@ -925,6 +926,8 @@ for required in ( ): require(required in inventory_contract_validator, f"Credential inventory contract validator is missing: {required}") require("./scripts/test-sync-github-environments.sh" in ci_runner, "CI must run the credential sync behavioral test.") +require("tests/test_brio_operation_lease.py" in ci_runner, "CI must run the Brio operation lease contract tests.") +require("tests/test_brio_operation_lease_wiring.py" in ci_runner, "CI must run the Brio operation lease wiring tests.") require("pass-cli item view --item-title '' --field ''" in readme, "README must document stdin-only pass-cli credential synchronization.") require("| gh secret set '' --env '' --repo 'Makepad-fr/postgres'" in normalized_readme, "README must mirror workflow secrets only into protected GitHub environments.") for policy in ( diff --git a/tests/test_brio_operation_lease.py b/tests/test_brio_operation_lease.py new file mode 100644 index 0000000..e8389a2 --- /dev/null +++ b/tests/test_brio_operation_lease.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import pathlib +import stat +import tempfile +import unittest +from unittest import mock + + +ROOT = pathlib.Path(os.environ.get("BRIO_LEASE_CANDIDATE_ROOT", pathlib.Path(__file__).parents[1])).resolve() + + +def load_module(name: str, relative: str): + specification = importlib.util.spec_from_file_location(name, ROOT / relative) + if specification is None or specification.loader is None: + raise RuntimeError(f"cannot load {relative}") + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + return module + + +LEASE = load_module("brio_operation_lease", "scripts/brio-operation-lease.py") +DISPATCH = load_module("brio_operation_lease_dispatch", "scripts/brio-operation-lease-dispatch.py") +COORDINATOR = load_module("brio_operation_lease_coordinator", "scripts/brio-operation-lease-coordinator.py") + +OWNER_A = "a" * 64 +OWNER_B = "b" * 64 + + +class LeaseFixture(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temporary.name) + self.runtime = self.root / "run" / "makepad" / "brio-operation-lease" + self.runtime.mkdir(parents=True, mode=0o700) + self.runtime.chmod(0o700) + self.guard = self.runtime / "guard" + self.guard.write_bytes(b"") + self.guard.chmod(0o600) + self.lease = self.runtime / "lease" + self.node = self.root / "node" + self.node.write_text("app\n", encoding="ascii") + self.node.chmod(0o600) + self.clock = [1_000_000] + self.store = LEASE.LeaseStore( + self.runtime, + self.node, + expected_uid=os.getuid(), + expected_gid=os.getgid(), + now=lambda: self.clock[0], + ) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def result(self, action: str, owner: str = OWNER_A, kind: str = "deployment") -> tuple[int, dict]: + status_code, raw = self.store.operate(action, owner, kind) + self.assertEqual(raw, json.dumps(json.loads(raw), sort_keys=True, separators=(",", ":"))) + return status_code, json.loads(raw) + + def test_same_owner_acquire_and_status_are_idempotent_without_renewal(self) -> None: + status, acquired = self.result("acquire") + self.assertEqual((status, acquired["state"]), (0, "acquired")) + self.assertEqual(acquired["expires_at"], self.clock[0] + 14_400) + original_expiry = acquired["expires_at"] + + self.clock[0] += 300 + status, held = self.result("acquire") + self.assertEqual((status, held["state"], held["expires_at"]), (0, "held", original_expiry)) + status, held = self.result("status") + self.assertEqual((status, held["state"], held["expires_at"]), (0, "held", original_expiry)) + + def test_different_owner_or_kind_contends_until_exact_stale_takeover(self) -> None: + _, acquired = self.result("acquire") + expiry = acquired["expires_at"] + status, busy = self.result("acquire", OWNER_B) + self.assertEqual((status, busy["state"], busy["owner"]), (75, "busy", OWNER_A)) + status, busy = self.result("acquire", OWNER_A, "evidence") + self.assertEqual((status, busy["state"], busy["kind"]), (75, "busy", "deployment")) + + self.clock[0] = expiry + status, replacement = self.result("acquire", OWNER_B, "evidence") + self.assertEqual((status, replacement["state"], replacement["owner"]), (0, "acquired", OWNER_B)) + self.assertEqual(replacement["expires_at"], expiry + 14_400) + + def test_release_is_owner_bound_reverse_safe_and_idempotent(self) -> None: + self.result("acquire") + status, busy = self.result("release", OWNER_B) + self.assertEqual((status, busy["state"]), (75, "busy")) + status, released = self.result("release") + self.assertEqual((status, released["state"], released["released_at"]), (0, "released", self.clock[0])) + status, absent = self.result("release") + self.assertEqual((status, absent["state"]), (0, "absent")) + + def test_malformed_symlink_and_permission_drift_fail_closed(self) -> None: + lease_path = self.lease + lease_path.write_bytes(b"") + lease_path.chmod(0o600) + with self.assertRaisesRegex(LEASE.LeaseError, "malformed"): + self.store.operate("acquire", OWNER_A, "deployment") + + lease_path.write_text("not-json\n", encoding="ascii") + lease_path.chmod(0o600) + with self.assertRaisesRegex(LEASE.LeaseError, "malformed"): + self.store.operate("acquire", OWNER_A, "deployment") + + lease_path.unlink() + target = self.root / "outside" + target.write_text("{}", encoding="ascii") + lease_path.symlink_to(target) + with self.assertRaisesRegex(LEASE.LeaseError, "metadata is unsafe"): + self.store.operate("acquire", OWNER_A, "deployment") + + lease_path.unlink() + self.guard.chmod(0o644) + with self.assertRaisesRegex(LEASE.LeaseError, "unsafe permissions"): + self.store.operate("acquire", OWNER_A, "deployment") + self.guard.chmod(0o600) + self.runtime.chmod(0o755) + with self.assertRaisesRegex(LEASE.LeaseError, "unsafe permissions"): + self.store.operate("acquire", OWNER_A, "deployment") + + def test_pristine_absent_state_acquires_and_release_removes_state(self) -> None: + self.assertFalse(self.lease.exists()) + status, acquired = self.result("acquire") + self.assertEqual((status, acquired["state"]), (0, "acquired")) + self.assertTrue(self.lease.is_file()) + status, released = self.result("release") + self.assertEqual((status, released["state"]), (0, "released")) + self.assertFalse(self.lease.exists()) + + def test_recreated_volatile_runtime_does_not_restore_stale_lease(self) -> None: + self.result("acquire") + self.lease.unlink() + status, acquired = self.result("acquire", OWNER_B, "evidence") + self.assertEqual((status, acquired["state"], acquired["owner"]), (0, "acquired", OWNER_B)) + + def test_acquired_state_mode_drift_is_rejected(self) -> None: + self.result("acquire") + (self.runtime / "lease").chmod(0o644) + with self.assertRaisesRegex(LEASE.LeaseError, "metadata is unsafe"): + self.store.operate("status", OWNER_A, "deployment") + + def test_noncanonical_node_and_clock_inconsistent_state_fail_closed(self) -> None: + self.node.write_text("app", encoding="ascii") + with self.assertRaisesRegex(LEASE.LeaseError, "node identity is invalid"): + self.store.operate("acquire", OWNER_A, "deployment") + self.node.write_text("app\n", encoding="ascii") + self.result("acquire") + self.clock[0] -= 1 + with self.assertRaisesRegex(LEASE.LeaseError, "inconsistent with the system clock"): + self.store.operate("status", OWNER_A, "deployment") + + +class DispatchTests(unittest.TestCase): + def test_dispatch_executes_only_the_fixed_argv(self) -> None: + command = f"acquire {OWNER_A} evidence" + with mock.patch.object(DISPATCH.sys, "argv", ["dispatch"]), mock.patch.dict(os.environ, {"SSH_ORIGINAL_COMMAND": command}, clear=True), mock.patch.object( + DISPATCH.os, "execv", side_effect=RuntimeError("captured") + ) as execute: + with self.assertRaisesRegex(RuntimeError, "captured"): + DISPATCH.main() + execute.assert_called_once_with( + "/usr/bin/sudo", + [ + "sudo", "-n", "--", + "/usr/local/libexec/makepad/brio-operation-lease", + "acquire", OWNER_A, "evidence", + ], + ) + + def test_dispatch_rejects_shell_syntax_extra_arguments_and_uppercase_owner(self) -> None: + invalid = ( + f"acquire {OWNER_A} evidence; id", + f"acquire {OWNER_A} evidence extra", + f"acquire {OWNER_A.upper()} evidence", + f"acquire {OWNER_A} deployment\nstatus {OWNER_A} deployment", + "", + ) + for command in invalid: + with self.subTest(command=command), mock.patch.dict( + os.environ, {"SSH_ORIGINAL_COMMAND": command}, clear=True + ), mock.patch.object(DISPATCH.sys, "argv", ["dispatch"]), mock.patch.object(DISPATCH.os, "execv") as execute, contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(DISPATCH.main(), 64) + execute.assert_not_called() + + +class OwnerDerivationTests(unittest.TestCase): + def test_deployment_and_evidence_owners_are_deterministic_and_domain_separated(self) -> None: + owner_module = load_module("derive_brio_operation_owner", "scripts/derive-brio-operation-owner.py") + environment = { + "GITHUB_REPOSITORY": "Makepad-fr/postgres", + "GITHUB_RUN_ID": "12345", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_SHA": "1" * 40, + } + outputs: dict[str, str] = {} + for kind in ("deployment", "evidence"): + stream = io.StringIO() + with mock.patch.dict(os.environ, environment, clear=True), contextlib.redirect_stdout(stream): + self.assertEqual(owner_module.main(["owner", kind]), 0) + outputs[kind] = stream.getvalue().strip() + self.assertRegex(outputs[kind], r"^[0-9a-f]{64}$") + self.assertNotEqual(outputs["deployment"], outputs["evidence"]) + + repeated = io.StringIO() + with mock.patch.dict(os.environ, environment, clear=True), contextlib.redirect_stdout(repeated): + owner_module.main(["owner", "deployment"]) + self.assertEqual(repeated.getvalue().strip(), outputs["deployment"]) + + def test_owner_derivation_rejects_incomplete_or_malformed_identity(self) -> None: + owner_module = load_module("derive_brio_operation_owner_invalid", "scripts/derive-brio-operation-owner.py") + with mock.patch.dict(os.environ, {"GITHUB_RUN_ID": "0", "GITHUB_SHA": "A" * 40}, clear=True): + with self.assertRaises(SystemExit): + owner_module.main(["owner", "evidence"]) + + +def remote_result(node: str, owner: str, kind: str, state: str) -> dict: + return { + "expires_at": 1_014_400 if state not in {"absent", "expired"} else None, + "kind": kind, + "node": node, + "owner": owner, + "released_at": 1_000_100 if state in {"absent", "expired", "released"} else None, + "state": state, + "version": 1, + } + + +class CoordinatorTests(unittest.TestCase): + def setUp(self) -> None: + self.nodes = [ + {"name": "app", "host": "app.invalid", "port": 22}, + {"name": "identity", "host": "identity.invalid", "port": 22}, + {"name": "database", "host": "database.invalid", "port": 22}, + ] + + def run_main(self, action: str, invoke): + output = io.StringIO() + errors = io.StringIO() + with mock.patch.object(COORDINATOR.os, "geteuid", return_value=0), mock.patch.object( + COORDINATOR, "load_config", return_value=self.nodes + ), mock.patch.object(COORDINATOR, "invoke", side_effect=invoke), contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + try: + status = COORDINATOR.main(["coordinator", action, OWNER_A, "deployment"]) + except SystemExit as error: + status = int(error.code) + return status, output.getvalue(), errors.getvalue() + + def test_acquisition_and_release_use_fixed_opposite_orders(self) -> None: + calls: list[tuple[str, str]] = [] + + def invoke(node, action, owner, kind): + calls.append((action, node["name"])) + state = "acquired" if action == "acquire" else "released" + return 0, remote_result(node["name"], owner, kind, state) + + status, _, _ = self.run_main("acquire", invoke) + self.assertEqual(status, 0) + self.assertEqual(calls, [("acquire", "app"), ("acquire", "identity"), ("acquire", "database")]) + calls.clear() + status, _, _ = self.run_main("release", invoke) + self.assertEqual(status, 0) + self.assertEqual(calls, [("release", "database"), ("release", "identity"), ("release", "app")]) + + def test_explicit_foreign_contention_cleans_only_acquired_prefix(self) -> None: + calls: list[tuple[str, str]] = [] + + def invoke(node, action, owner, kind): + calls.append((action, node["name"])) + if action == "acquire" and node["name"] == "identity": + return 75, remote_result(node["name"], OWNER_B, "evidence", "busy") + state = "acquired" if action == "acquire" else "released" + return 0, remote_result(node["name"], owner, kind, state) + + status, _, errors = self.run_main("acquire", invoke) + self.assertEqual(status, 78) + self.assertEqual( + calls, + [ + ("acquire", "app"), + ("acquire", "identity"), + ("release", "app"), + ], + ) + self.assertIn("identity lease acquisition was rejected", errors) + self.assertNotIn("partial lease cleanup failed", errors) + + def test_partial_acquisition_cleanup_failure_is_surfaced(self) -> None: + calls: list[tuple[str, str]] = [] + + def invoke(node, action, owner, kind): + calls.append((action, node["name"])) + if action == "acquire" and node["name"] == "identity": + raise COORDINATOR.CoordinatorError("identity lease endpoint is unavailable") + if action == "release" and node["name"] == "app": + return 75, remote_result(node["name"], OWNER_B, "evidence", "busy") + state = "acquired" if action == "acquire" else "released" + return 0, remote_result(node["name"], owner, kind, state) + + status, _, errors = self.run_main("acquire", invoke) + self.assertEqual(status, 78) + self.assertEqual(calls[-2:], [("release", "identity"), ("release", "app")]) + self.assertIn("identity lease acquisition failed and partial lease cleanup failed", errors) + + def test_same_owner_other_kind_contention_does_not_release_current_node(self) -> None: + calls: list[tuple[str, str]] = [] + + def invoke(node, action, owner, kind): + calls.append((action, node["name"])) + if action == "acquire" and node["name"] == "identity": + return 75, remote_result(node["name"], owner, "evidence", "busy") + state = "acquired" if action == "acquire" else "released" + return 0, remote_result(node["name"], owner, kind, state) + + status, _, errors = self.run_main("acquire", invoke) + self.assertEqual(status, 78) + self.assertEqual(calls[-1], ("release", "app")) + self.assertNotIn(("release", "identity"), calls) + self.assertIn("identity lease acquisition was rejected", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_brio_operation_lease_wiring.py b/tests/test_brio_operation_lease_wiring.py new file mode 100644 index 0000000..bd24843 --- /dev/null +++ b/tests/test_brio_operation_lease_wiring.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +import pathlib +import re +import unittest + + +ROOT = pathlib.Path( + os.environ.get("BRIO_LEASE_CANDIDATE_ROOT", pathlib.Path(__file__).parents[1]) +).resolve() + + +class PostgreSQLOperationLeaseWiringTests(unittest.TestCase): + def test_postgres_mutations_are_bounded_by_cross_host_lease(self) -> None: + workflows = [ + (ROOT / ".github/workflows/manual-deploy.yml").read_text(encoding="utf-8"), + (ROOT / ".github/workflows/deploy-brio-identity-db.yml").read_text(encoding="utf-8"), + ] + for workflow in workflows: + with self.subTest(workflow=workflow[:40]): + timeout = re.search(r"timeout-minutes:\s*([0-9]+)", workflow) + self.assertIsNotNone(timeout) + self.assertLessEqual(int(timeout.group(1)), 210) + acquire = workflow.index("brio-operation-lease-remote.sh acquire") + first_remote_mutation = min( + position + for marker in ( + "install -d -m 0755 ${remote_parent_q}", + "install -d -m 0700 ${remote_bundle_q}", + ) + if (position := workflow.find(marker)) >= 0 + ) + release = workflow.index("brio-operation-lease-remote.sh release") + cleanup_names = ( + "Remove job-scoped deployment material", + "Remove local job-scoped deployment material", + ) + local_cleanup = min( + workflow.index(name) for name in cleanup_names if name in workflow + ) + self.assertLess(acquire, first_remote_mutation) + self.assertIn( + "brio-operation-lease status ${lease_owner_q} deployment", + workflow[: first_remote_mutation + 200], + ) + self.assertLess(release, local_cleanup) + + for relative in ( + "scripts/deploy-postgres-stack.sh", + "scripts/deploy-brio-canary-postgres.sh", + "scripts/deploy-brio-identity-db-host.sh", + ): + script = (ROOT / relative).read_text(encoding="utf-8") + self.assertIn("require_brio_deployment_lease", script) + self.assertIn("/usr/local/libexec/makepad/brio-operation-lease", script) + self.assertIn('status "${BRIO_OPERATION_LEASE_OWNER}" deployment', script) + + def test_remote_client_uses_only_the_bounded_coordinator_command(self) -> None: + client = (ROOT / "scripts/brio-operation-lease-remote.sh").read_text(encoding="utf-8") + self.assertIn('[[ "${action}" =~ ^(acquire|status|release)$ ]]', client) + self.assertIn('[[ "${owner}" =~ ^[0-9a-f]{64}$ ]]', client) + self.assertIn("brio-operation-lease-coordinator ${action} ${owner} deployment", client) + for forbidden in ("eval ", "bash -c", "sh -c", "docker "): + self.assertNotIn(forbidden, client) + + +if __name__ == "__main__": + unittest.main()