Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions .github/workflows/manual-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ jobs:
cp config/runtrace-pg_hba.conf "${bundle_root}/config/runtrace-pg_hba.conf"
cp scripts/run-runtrace-backup.sh "${bundle_root}/scripts/run-runtrace-backup.sh"
cp scripts/run-runtrace-backup-loop.sh "${bundle_root}/scripts/run-runtrace-backup-loop.sh"
cp scripts/run-jotwink-backup.sh "${bundle_root}/scripts/run-jotwink-backup.sh"
cp scripts/run-jotwink-backup-loop.sh "${bundle_root}/scripts/run-jotwink-backup-loop.sh"
cp "envs/${{ inputs.environment }}/compose.yml" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml"
cp "envs/${{ inputs.environment }}/.env.db" "${bundle_root}/envs/${{ inputs.environment }}/.env.db"
cat > "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" <<EOF
Expand Down Expand Up @@ -97,6 +99,8 @@ jobs:
scp "${scp_opts[@]}" "${bundle_root}/config/runtrace-pg_hba.conf" "${remote_target}:${REMOTE_DIR}/config/runtrace-pg_hba.conf"
scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup.sh"
scp "${scp_opts[@]}" "${bundle_root}/scripts/run-runtrace-backup-loop.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-runtrace-backup-loop.sh"
scp "${scp_opts[@]}" "${bundle_root}/scripts/run-jotwink-backup.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-jotwink-backup.sh"
scp "${scp_opts[@]}" "${bundle_root}/scripts/run-jotwink-backup-loop.sh" "${remote_target}:${REMOTE_DIR}/scripts/run-jotwink-backup-loop.sh"
scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/compose.yml" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/compose.yml"
scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.db" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.db"
scp "${scp_opts[@]}" "${bundle_root}/envs/${{ inputs.environment }}/.env.deploy" "${remote_target}:${REMOTE_DIR}/envs/${{ inputs.environment }}/.env.deploy"
Expand All @@ -118,6 +122,9 @@ jobs:
postgres_runtrace_hba_config=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-)
runtrace_backup_path=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-)
runtrace_backup_password_file=$(grep '^MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-)
jotwink_backup_path=$(grep '^MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-)
jotwink_backup_password_file=$(grep '^MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-)
jotwink_backup_encryption_cert_config=$(grep '^MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG=' "${db_env}" | tail -n 1 | cut -d= -f2-)
postgres_ca_cert_file=$(grep '^MAKEPAD_POSTGRES_CA_CERT_HOST_PATH=' "${db_env}" | tail -n 1 | cut -d= -f2-)
vif_enabled=0
if [[ "${deploy_env}" == "production" ]]; then
Expand All @@ -138,6 +145,9 @@ jobs:
if [[ "${deploy_env}" == "production" ]]; then
: "${runtrace_backup_path:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PATH is missing or empty in ${db_env}}"
: "${runtrace_backup_password_file:?MAKEPAD_POSTGRES_RUNTRACE_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}"
: "${jotwink_backup_path:?MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH is missing or empty in ${db_env}}"
: "${jotwink_backup_password_file:?MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH is missing or empty in ${db_env}}"
: "${jotwink_backup_encryption_cert_config:?MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG is missing or empty in ${db_env}}"
: "${postgres_ca_cert_file:?MAKEPAD_POSTGRES_CA_CERT_HOST_PATH is missing or empty in ${db_env}}"
fi
if [[ ! -s "${postgres_root_password_file}" ]]; then
Expand Down Expand Up @@ -173,6 +183,30 @@ jobs:
echo "Runtrace backup credential must be owned by uid 70 with mode 0400." >&2
exit 1
fi
if [[ ! -d "${jotwink_backup_path}" || -L "${jotwink_backup_path}" ]]; then
echo "Jotwink backup path must be a pre-provisioned non-symlink directory: ${jotwink_backup_path}" >&2
exit 1
fi
jotwink_backup_directory_mode=$(stat -c '%a' "${jotwink_backup_path}")
jotwink_backup_directory_uid=$(stat -c '%u' "${jotwink_backup_path}")
if [[ "${jotwink_backup_directory_mode}" != "700" || "${jotwink_backup_directory_uid}" != "70" ]]; then
echo "Jotwink backup path must be owned by uid 70 with mode 0700: ${jotwink_backup_path}" >&2
exit 1
fi
if [[ ! -s "${jotwink_backup_password_file}" || -L "${jotwink_backup_password_file}" ]]; then
echo "Jotwink backup credential must be a non-empty, non-symlink file: ${jotwink_backup_password_file}" >&2
exit 1
fi
jotwink_backup_password_mode=$(stat -c '%a' "${jotwink_backup_password_file}")
jotwink_backup_password_uid=$(stat -c '%u' "${jotwink_backup_password_file}")
if [[ "${jotwink_backup_password_mode}" != "400" || "${jotwink_backup_password_uid}" != "70" ]]; then
echo "Jotwink backup credential must be owned by uid 70 with mode 0400." >&2
exit 1
fi
if ! docker config inspect "${jotwink_backup_encryption_cert_config}" >/dev/null 2>&1; then
echo "Jotwink backup recipient certificate config does not exist: ${jotwink_backup_encryption_cert_config}" >&2
exit 1
fi
if [[ ! -s "${postgres_ca_cert_file}" || -L "${postgres_ca_cert_file}" ]] || ! grep -q -- '-----BEGIN CERTIFICATE-----' "${postgres_ca_cert_file}"; then
echo "PostgreSQL CA certificate must be a non-empty, non-symlink PEM file: ${postgres_ca_cert_file}" >&2
exit 1
Expand Down
77 changes: 74 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@ This repository owns the shared PostgreSQL server. Application repositories conn
- `bootstrap/keycloak-new-instances.sql`: idempotent SQL bootstrap for the Vif, Makepad, Vestiaire, and Runtrace Keycloak databases
- `bootstrap/keycloak-runtrace-app.sql`: targeted idempotent bootstrap for the Runtrace Keycloak database
- `bootstrap/runtrace-app.sql`: idempotent SQL bootstrap for the Runtrace application database
- `bootstrap/jotwink-databases.sql`: idempotent bootstrap for the isolated Jotwink application and identity databases
- `bootstrap/openpanel-app.sql`: idempotent SQL bootstrap for the OpenPanel application database
- `scripts/run-runtrace-backup.sh`: certificate-verified logical backup for Runtrace app and identity data
- `scripts/verify-runtrace-restore.sh`: destructive restore verification against explicit non-production targets
- `scripts/run-jotwink-backup.sh`: certificate-verified logical backup for Jotwink application and identity data
- `scripts/verify-jotwink-restore.sh`: destructive Jotwink restore verification against explicit non-production targets

## Networks

Expand Down Expand Up @@ -57,7 +60,7 @@ Use the manual GitHub Actions workflow in this repository.
The dedicated database VM currently runs standalone Docker Compose rather than
joining the application Swarm. On that host, deploy the same TLS and backup
policy with `compose.host.yml` after provisioning the certificate, key, CA,
password files, backup directory, and committed HBA policy:
password files, both backup directories, and committed HBA policy:

```bash
docker compose --env-file envs/production/.env.db -f compose.host.yml config
Expand All @@ -66,7 +69,10 @@ docker compose --env-file envs/production/.env.db -f compose.host.yml up -d --pu

The host deployment preserves the existing host-network endpoint used by
Keycloak while requiring TLS and SCRAM for `runtrace` and
`keycloak_runtrace`. Other databases keep their existing SCRAM transport policy.
`keycloak_runtrace`. Jotwink is stricter: `jotwink_app` can reach `jotwink`
only from the application WireGuard peer `10.80.0.1`, and
`keycloak_jotwink_app` can reach `keycloak_jotwink` only from the dedicated
Keycloak host. Other databases keep their existing SCRAM transport policy.

Required environment secrets:

Expand Down Expand Up @@ -97,7 +103,7 @@ docker config create makepad_postgres_tls_cert_v1 /secure/path/server.crt
docker secret create makepad_postgres_tls_key_v1 /secure/path/server.key
```

The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to Runtrace and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace` and `keycloak_runtrace` and requires SCRAM authentication over TLS for both; unrelated shared databases retain their current SCRAM transport policy during migration.
The names must match `MAKEPAD_POSTGRES_TLS_CERT_CONFIG` and `MAKEPAD_POSTGRES_TLS_KEY_SECRET` in the selected `.env.db`. Rotate by creating new versioned objects, updating those two names, and redeploying; never replace private-key material in place. Distribute only the issuing CA certificate to application and Keycloak hosts. The deployment creates the versioned `MAKEPAD_POSTGRES_RUNTRACE_HBA_CONFIG` from the committed policy when absent and rejects content drift under an existing name. The policy rejects plaintext connections to `runtrace`, `keycloak_runtrace`, `jotwink`, and `keycloak_jotwink`; it also source-restricts the two Jotwink roles. Unrelated shared databases retain their current SCRAM transport policy during migration. Rotate the versioned HBA config name whenever this committed policy changes.

The workflow deploys only the PostgreSQL stack. It validates the password file before deployment. If one of the configured database networks does not exist yet, it is created as an encrypted overlay on the manager before deployment.

Expand Down Expand Up @@ -127,6 +133,45 @@ scripts/verify-runtrace-restore.sh /var/lib/makepad/postgres-backups/runtrace/<t

Record the timestamp, artifact checksum, duration, and operator in Runtrace backup/restore evidence. The restore verifier intentionally refuses to run without the exact non-production replacement acknowledgement and validates that both durable state schemas exist after restore.

## Jotwink Backup And Restore

Jotwink has an independent backup service and health timestamp so a failed
Jotwink dump cannot be hidden by a successful Runtrace dump. It creates
custom-format dumps of `jotwink` and `keycloak_jotwink` every six hours,
validates each archive, encrypts it in ephemeral tmpfs with OpenSSL CMS and
AES-256-GCM before it reaches persistent storage, writes SHA-256 checksums, and
retains 35 days by default. Provision its root and a public recipient
certificate before starting `compose.host.yml`:

```bash
sudo install -d -o 70 -g 70 -m 0700 /var/lib/makepad/postgres-backups/jotwink
sudo install -d -o root -g root -m 0755 /etc/makepad/postgres-backup
sudo install -o root -g root -m 0444 jotwink-recipient.pem /etc/makepad/postgres-backup/jotwink-recipient.pem
```

For Swarm, create the versioned public certificate config named by
`MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_CONFIG`. Keep the matching
private key offline; it is used only during a controlled restore drill.

Replicate every completed timestamp directory and `last-success.json` to an
off-host, separately administered encrypted store. Alert before the health
timestamp exceeds two intervals. At least quarterly, and before launch or a
database upgrade, restore the newest artifacts into empty non-production
databases:

```bash
export PGSERVICEFILE=/etc/makepad/postgres-restore-services.conf
export JOTWINK_RESTORE_SERVICE=jotwink_restore_test
export KEYCLOAK_JOTWINK_RESTORE_SERVICE=keycloak_jotwink_restore_test
export JOTWINK_RESTORE_CONFIRM=replace-nonproduction-restore-targets
export JOTWINK_BACKUP_DECRYPTION_CERT=/secure/jotwink-recipient.pem
export JOTWINK_BACKUP_DECRYPTION_KEY=/secure/jotwink-recipient.key
scripts/verify-jotwink-restore.sh /var/lib/makepad/postgres-backups/jotwink/<timestamp>
```

Record the artifact checksum, elapsed time, recovery-point age, and operator.
The verifier checks both the app migration table and Keycloak realm table.

## Application Databases

Create one database and one dedicated user per application.
Expand All @@ -146,6 +191,13 @@ Runtrace application persistence uses:
| --- | --- | --- |
| Runtrace app | `runtrace` | `runtrace_app` |

Jotwink application and identity persistence use separate roles:

| Application | Database | Role |
| --- | --- | --- |
| Jotwink app | `jotwink` | `jotwink_app` |
| Jotwink Keycloak | `keycloak_jotwink` | `keycloak_jotwink_app` |

OpenPanel application persistence uses:

| Application | Database | Role |
Expand All @@ -162,6 +214,8 @@ Run the idempotent bootstrap with generated passwords. `POSTGRES_ADMIN_URL` must
: "${KEYCLOAK_RUNTRACE_DB_PASSWORD:?set KEYCLOAK_RUNTRACE_DB_PASSWORD to a generated password}"
: "${RUNTRACE_DB_PASSWORD:?set RUNTRACE_DB_PASSWORD to a generated password}"
: "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}"
: "${JOTWINK_DB_PASSWORD:?set JOTWINK_DB_PASSWORD to a generated password}"
: "${KEYCLOAK_JOTWINK_DB_PASSWORD:?set KEYCLOAK_JOTWINK_DB_PASSWORD to a generated password}"

psql "$POSTGRES_ADMIN_URL" \
-v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \
Expand All @@ -182,6 +236,21 @@ psql "$POSTGRES_ADMIN_URL" \
psql "$POSTGRES_ADMIN_URL" \
-v openpanel_app_password="$OPENPANEL_DB_PASSWORD" \
-f bootstrap/openpanel-app.sql

psql "$POSTGRES_ADMIN_URL" \
-v jotwink_app_password="$JOTWINK_DB_PASSWORD" \
-v keycloak_jotwink_app_password="$KEYCLOAK_JOTWINK_DB_PASSWORD" \
-f bootstrap/jotwink-databases.sql
```

Jotwink application traffic follows the existing WireGuard path from
`10.80.0.1` to the database host `10.80.0.2`. Both clients use certificate
verification; the server certificate SAN must include the exact hostname used
in these URLs:

```text
postgres://jotwink_app:<secret>@<db-vm-host>:5432/jotwink?sslmode=verify-full&sslrootcert=/etc/jotwink/postgres/ca.crt
postgres://keycloak_jotwink_app:<secret>@<db-vm-host>:5432/keycloak_jotwink?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt
```

The current production Keycloak environments connect with the DB VM host:
Expand Down Expand Up @@ -229,4 +298,6 @@ Run the local static checks before opening a deployment PR:
bash scripts/validate-postgres-config.sh
bash scripts/test-runtrace-tls-policy.sh
bash scripts/test-runtrace-backup.sh
bash scripts/validate-jotwink-config.sh
bash scripts/test-jotwink-backup.sh
```
72 changes: 72 additions & 0 deletions bootstrap/jotwink-databases.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
\set ON_ERROR_STOP on

-- Run this bootstrap with a PostgreSQL superuser connection. It provisions
-- the Jotwink product and Keycloak databases without embedding credentials.

\if :{?jotwink_app_password}
\else
\echo 'missing required psql variable: jotwink_app_password'
\quit 1
\endif
\if :{?keycloak_jotwink_app_password}
\else
\echo 'missing required psql variable: keycloak_jotwink_app_password'
\quit 1
\endif

SELECT CASE WHEN NULLIF(btrim(:'jotwink_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS jotwink_app_password_is_nonempty \gset
\if :jotwink_app_password_is_nonempty
\else
\echo 'empty required psql variable: jotwink_app_password'
\quit 1
\endif
SELECT CASE WHEN NULLIF(btrim(:'keycloak_jotwink_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_jotwink_app_password_is_nonempty \gset
\if :keycloak_jotwink_app_password_is_nonempty
\else
\echo 'empty required psql variable: keycloak_jotwink_app_password'
\quit 1
\endif

SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap'));

DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'jotwink_app') THEN
CREATE ROLE jotwink_app LOGIN;
END IF;
END;
$$;
ALTER ROLE jotwink_app LOGIN PASSWORD :'jotwink_app_password';
SELECT 'CREATE DATABASE jotwink OWNER jotwink_app'
WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'jotwink') \gexec
SELECT 'ALTER DATABASE jotwink OWNER TO jotwink_app'
WHERE EXISTS (
SELECT 1
FROM pg_database d
JOIN pg_roles r ON r.oid = d.datdba
WHERE d.datname = 'jotwink'
AND r.rolname <> 'jotwink_app'
) \gexec
GRANT CONNECT ON DATABASE jotwink TO jotwink_app;

DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_jotwink_app') THEN
CREATE ROLE keycloak_jotwink_app LOGIN;
END IF;
END;
$$;
ALTER ROLE keycloak_jotwink_app LOGIN PASSWORD :'keycloak_jotwink_app_password';
SELECT 'CREATE DATABASE keycloak_jotwink OWNER keycloak_jotwink_app'
WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_jotwink') \gexec
SELECT 'ALTER DATABASE keycloak_jotwink OWNER TO keycloak_jotwink_app'
WHERE EXISTS (
SELECT 1
FROM pg_database d
JOIN pg_roles r ON r.oid = d.datdba
WHERE d.datname = 'keycloak_jotwink'
AND r.rolname <> 'keycloak_jotwink_app'
) \gexec
GRANT CONNECT ON DATABASE keycloak_jotwink TO keycloak_jotwink_app;

SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('jotwink-databases-bootstrap'));
48 changes: 48 additions & 0 deletions compose.host.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,51 @@ services:
options:
max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"

jotwink_backup:
image: ${JOTWINK_BACKUP_IMAGE:-postgres:16-bookworm@sha256:64154d0babcb1741988719e703419af0382b19953706149f9872fbd0f438efa8}
user: "70:70"
network_mode: host
command:
- /usr/local/bin/run-jotwink-backup-loop.sh
environment:
PGHOST: 127.0.0.1
PGPORT: "5432"
PGUSER: ${POSTGRES_USER:-postgres}
PGSSLMODE: verify-full
PGSSLROOTCERT: /etc/postgresql/ca.crt
POSTGRES_SUPERUSER_PASSWORD_FILE: /run/secrets/postgres_backup_password
JOTWINK_BACKUP_ENCRYPTION_CERT: /run/config/jotwink_backup_recipient.pem
JOTWINK_BACKUP_ROOT: /backups
JOTWINK_BACKUP_INTERVAL_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_INTERVAL_SECONDS:-21600}
JOTWINK_BACKUP_RETRY_SECONDS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETRY_SECONDS:-300}
JOTWINK_BACKUP_RETENTION_DAYS: ${MAKEPAD_POSTGRES_JOTWINK_BACKUP_RETENTION_DAYS:-35}
read_only: true
tmpfs:
- /tmp:mode=0700,uid=70,gid=70
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
volumes:
- "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PATH:-/var/lib/makepad/postgres-backups/jotwink}:/backups"
- "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_PASSWORD_FILE_HOST_PATH:-/etc/makepad/secrets/postgres-backup-password}:/run/secrets/postgres_backup_password:ro"
- "${MAKEPAD_POSTGRES_CA_CERT_HOST_PATH:-/etc/makepad/tls/postgres/ca.crt}:/etc/postgresql/ca.crt:ro"
- "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_ENCRYPTION_CERT_HOST_PATH:-/etc/makepad/postgres-backup/jotwink-recipient.pem}:/run/config/jotwink_backup_recipient.pem:ro"
- "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup.sh}:/usr/local/bin/run-jotwink-backup.sh:ro"
- "${MAKEPAD_POSTGRES_JOTWINK_BACKUP_LOOP_SCRIPT_HOST_PATH:-/srv/makepad/postgres/scripts/run-jotwink-backup-loop.sh}:/usr/local/bin/run-jotwink-backup-loop.sh:ro"
healthcheck:
test: ["CMD", "/usr/local/bin/run-jotwink-backup-loop.sh", "healthcheck"]
interval: 5m
timeout: 10s
retries: 3
start_period: 10m
logging:
driver: json-file
options:
max-size: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_SIZE:-20m}"
max-file: "${MAKEPAD_POSTGRES_BACKUP_LOG_MAX_FILES:-5}"
Loading