From 746e44fccf03b4b57f42619c90afbc13f3283df1 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 13 Aug 2026 12:41:14 +0200 Subject: [PATCH] Provision Jotwink databases and encrypted backups --- .github/workflows/manual-deploy.yml | 34 +++++++++ README.md | 77 ++++++++++++++++++- bootstrap/jotwink-databases.sql | 72 ++++++++++++++++++ compose.host.yml | 48 ++++++++++++ compose.yml | 4 + config/runtrace-pg_hba.conf | 13 +++- envs/canary/.env.db | 2 +- envs/production/.env.db | 10 ++- envs/production/compose.yml | 72 ++++++++++++++++++ scripts/run-jotwink-backup-loop.sh | 37 +++++++++ scripts/run-jotwink-backup.sh | 112 ++++++++++++++++++++++++++++ scripts/test-jotwink-backup.sh | 73 ++++++++++++++++++ scripts/validate-jotwink-config.sh | 55 ++++++++++++++ scripts/verify-jotwink-restore.sh | 66 ++++++++++++++++ 14 files changed, 668 insertions(+), 7 deletions(-) create mode 100644 bootstrap/jotwink-databases.sql create mode 100755 scripts/run-jotwink-backup-loop.sh create mode 100755 scripts/run-jotwink-backup.sh create mode 100755 scripts/test-jotwink-backup.sh create mode 100755 scripts/validate-jotwink-config.sh create mode 100755 scripts/verify-jotwink-restore.sh diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 32befcb..39a4a9e 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -56,6 +56,8 @@ jobs: cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf" cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh" cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh" + cp scripts/run-jotwink-backup.sh "${bundle_root}/scripts/run-jotwink-backup.sh" + cp scripts/run-jotwink-backup-loop.sh "${bundle_root}/scripts/run-jotwink-backup-loop.sh" cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <&2 + exit 1 + fi + if [[ ! -s "${jotwink_backup_password_file}" || -L "${jotwink_backup_password_file}" ]]; then + echo "Jotwink backup credential must be a non-empty, non-symlink file: ${jotwink_backup_password_file}" >&2 + exit 1 + fi + jotwink_backup_password_mode=$(stat -c '%a' "${jotwink_backup_password_file}") + jotwink_backup_password_uid=$(stat -c '%u' "${jotwink_backup_password_file}") + if [[ "${jotwink_backup_password_mode}" != "400" || "${jotwink_backup_password_uid}" != "70" ]]; then + echo "Jotwink backup credential must be owned by uid 70 with mode 0400." >&2 + exit 1 + fi + if ! docker config inspect "${jotwink_backup_encryption_cert_config}" >/dev/null 2>&1; then + echo "Jotwink backup recipient certificate config does not exist: ${jotwink_backup_encryption_cert_config}" >&2 + exit 1 + fi if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2 exit 1 diff --git a/README.md b/README.md index edac409..26b7d6d 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,12 @@ This repository owns the shared PostgreSQL server. Application repositories conn - `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the Vif, Makepad, Vestiaire, and Runtrace Keycloak databases - `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database - `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database +- `bootstrap/jotwink-databases.sql`: idempotent bootstrap for the isolated Jotwink application and identity databases - `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database - `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data - `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets +- `scripts/run-jotwink-backup.sh`: certificate-verified logical backup for Jotwink application and identity data +- `scripts/verify-jotwink-restore.sh`: destructive Jotwink restore verification against explicit non-production targets ## Networks @@ -57,7 +60,7 @@ Use the manual GitHub Actions workflow in this repository. The dedicated database VM currently runs standalone Docker Compose rather than joining the application Swarm. On that host, deploy the same TLS and backup policy with `compose.host.yml` after provisioning the certificate, key, CA, -password files, backup directory, and committed HBA policy: +password files, both backup directories, and committed HBA policy: ```bash docker compose --env-file envs/production/.env.db -f compose.host.yml config @@ -66,7 +69,10 @@ docker compose --env-file envs/production/.env.db -f compose.host.yml up -d --pu The host deployment preserves the existing host-network endpoint used by Keycloak while requiring TLS and SCRAM for `runtrace` and -`keycloak_runtrace`. Other databases keep their existing SCRAM transport policy. +`keycloak_runtrace`. Jotwink is stricter: `jotwink_app` can reach `jotwink` +only from the application WireGuard peer `10.80.0.1`, and +`keycloak_jotwink_app` can reach `keycloak_jotwink` only from the dedicated +Keycloak host. Other databases keep their existing SCRAM transport policy. Required environment secrets: @@ -97,7 +103,7 @@ docker config create makepad_postgres_tls_cert_v1 /secure/path/server.crt docker secret create makepad_postgres_tls_key_v1 /secure/path/server.key ``` -The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace` and `keycloak_runtrace` and requires SCRAM authentication over TLS for both; unrelated shared databases retain their current SCRAM transport policy during migration. +The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to application and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `jotwink`, and `keycloak_jotwink`; it also source-restricts the two Jotwink roles. Unrelated shared databases retain their current SCRAM transport policy during migration. Rotate the versioned HBA config name whenever this committed policy changes. The workflow deploys only the PostgreSQL stack. It validates the password file before deployment. If one of the configured database networks does not exist yet, it is created as an encrypted overlay on the manager before deployment. @@ -127,6 +133,45 @@ scripts/verify-runtrace-restore.sh /var/lib/makepad/postgres-backups/runtrace/