feat(deletion): drain post-deletion access credentials

README.md

@@ -45,10 +45,12 @@ From a fresh clone:
 docker compose up --build
 ```
 
-The Compose entrypoint generates and persists a random creator-authority encryption
+The Compose entrypoint generates and persists a random runtime master
 key in the private `lock-home` volume. Set
-`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before startup only when you need to supply
-your own 32-byte base64url key.
+`PUBKY_LOCK_RUNTIME_MASTER_KEY` before startup only when you need to supply
+your own 32-byte unpadded-base64url key. A supplied override is atomically
+persisted to that volume, so a later startup without the environment variable
+continues using the same key rather than silently reverting to an older key.
 
 Verified browser-facing defaults for the basic `docker-compose.yml` stack are:
 

docker-compose.yml

@@ -49,7 +49,7 @@ services:
     network_mode: service:pubky-testnet
     environment:
       PUBKY_LOCK_DATABASE_URL: postgres://locks:locks@postgres:5432/locks_test
-      PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY: ${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-}
+      PUBKY_LOCK_RUNTIME_MASTER_KEY: ${PUBKY_LOCK_RUNTIME_MASTER_KEY:-}
     volumes:
       - lock-home:/var/lib/pubky-lock
     command: ["locks-server-compose-entrypoint.sh"]

docker/locks-server-compose-entrypoint.sh

@@ -5,26 +5,75 @@ service_home="${LOCKS_SERVICE_HOME:-/var/lib/pubky-lock/.pubky-lock}"
 generated_config="$service_home/config.toml"
 compose_config="${LOCKS_COMPOSE_CONFIG:-/var/lib/pubky-lock/config.compose.toml}"
 secret_path="$service_home/secret.sess"
-creator_authority_key_path="$service_home/creator-authority-encryption-key"
+runtime_master_key_path="$service_home/runtime-master-key"
+retired_creator_authority_key_path="$service_home/creator-authority-encryption-key"
 public_config="${LOCKS_PUBLIC_CONFIG:-/run/locks-public/config.toml}"
 
 mkdir -p "$service_home"
 
-if [ -z "${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-}" ]; then
-  if [ ! -f "$creator_authority_key_path" ]; then
-    echo "[locks-compose] generating creator-authority encryption key"
+if [ -f "$retired_creator_authority_key_path" ]; then
+  echo "[locks-compose] retired creator-authority key detected: $retired_creator_authority_key_path" >&2
+  echo "[locks-compose] stop the stack, discard and reacquire creator authority rows or recreate the local database, remove the retired key file, then restart" >&2
+  exit 1
+fi
+
+if [ -n "${PUBKY_LOCK_RUNTIME_MASTER_KEY:-}" ]; then
+  if ! printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" | grep -Eq '^[A-Za-z0-9_-]{43}$'; then
+    echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2
+    exit 1
+  fi
+  umask 077
+  decoded_key_path="$runtime_master_key_path.decoded.$$"
+  cleanup_decoded_key() {
+    rm -f "$decoded_key_path"
+  }
+  trap cleanup_decoded_key EXIT HUP INT TERM
+  if ! printf '%s=' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" \
+    | tr '_-' '/+' \
+    | base64 -d > "$decoded_key_path" 2>/dev/null \
+    || [ "$(wc -c < "$decoded_key_path" | tr -d ' ')" -ne 32 ]; then
+    echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2
+    exit 1
+  fi
+  canonical_runtime_master_key="$(
+    base64 < "$decoded_key_path" \
+      | tr '+/' '-_' \
+      | tr -d '=\n'
+  )"
+  if [ "$canonical_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then
+    echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2
+    exit 1
+  fi
+  cleanup_decoded_key
+  trap - EXIT HUP INT TERM
+  if [ -f "$runtime_master_key_path" ]; then
+    persisted_runtime_master_key="$(cat "$runtime_master_key_path")"
+    if [ "$persisted_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then
+      echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY does not match the persisted runtime master key" >&2
+      echo "[locks-compose] rotate only through an explicit data migration or reset that handles encrypted state" >&2
+      exit 1
+    fi
+  else
+    temporary_key_path="$runtime_master_key_path.tmp.$$"
+    printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" > "$temporary_key_path"
+    chmod 600 "$temporary_key_path"
+    mv "$temporary_key_path" "$runtime_master_key_path"
+  fi
+else
+  if [ ! -f "$runtime_master_key_path" ]; then
+    echo "[locks-compose] generating runtime master key"
     umask 077
-    temporary_key_path="$creator_authority_key_path.tmp.$$"
+    temporary_key_path="$runtime_master_key_path.tmp.$$"
     head -c 32 /dev/urandom \
       | base64 \
       | tr '+/' '-_' \
       | tr -d '=\n' > "$temporary_key_path"
     chmod 600 "$temporary_key_path"
-    mv "$temporary_key_path" "$creator_authority_key_path"
+    mv "$temporary_key_path" "$runtime_master_key_path"
   fi
 
-  PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(cat "$creator_authority_key_path")"
-  export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY
+  PUBKY_LOCK_RUNTIME_MASTER_KEY="$(cat "$runtime_master_key_path")"
+  export PUBKY_LOCK_RUNTIME_MASTER_KEY
 fi
 
 if [ ! -f "$generated_config" ] || [ ! -f "$secret_path" ]; then
@@ -83,7 +132,14 @@ frontend_session_code_ttl_seconds = 120
 allowed_return_origins = ["http://127.0.0.1:8080", "http://localhost:8080"]
 
 [secrets]
-creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY"
+runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY"
+
+[deletion]
+retry_max_attempts = 10
+retry_initial_backoff_seconds = 1
+retry_max_backoff_seconds = 300
+final_credential_issuance_window_seconds = 900
+final_read_window_seconds = 900
 
 [logging]
 level = "info,pubky::actors::session=warn"

docs/LOCAL_OPERATOR_DEMO.md

@@ -108,7 +108,7 @@ If `/creator/lock-service-config` or `/creator/priv-resources/content/<path>` re
 ## Prerequisites
 
 - A Postgres database reachable through `PUBKY_LOCK_DATABASE_URL`.
-- A 32-byte base64url creator-authority encryption key in `PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY`.
+- A 32-byte base64url runtime master key in `PUBKY_LOCK_RUNTIME_MASTER_KEY`.
 - `curl`, `jq`, and `python3` available in your shell.
 - A generated/default Lock Server config and secret under `~/.pubky-lock/`.
 
@@ -118,10 +118,10 @@ The database URL below is a local development example. Real credentials must com
 export PUBKY_LOCK_DATABASE_URL='postgres://locks:locks@localhost:55433/locks_test'
 ```
 
-Generate a local creator-authority encryption key for this shell before starting the server:
+Generate a local runtime master key for this shell before starting the server:
 
 ```bash
-export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(python3 - <<'PY'
+export PUBKY_LOCK_RUNTIME_MASTER_KEY="$(python3 - <<'PY'
 import base64, os
 print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('='))
 PY

docs/RUNTIME.md

@@ -182,14 +182,33 @@ Operator-facing readiness uses semantic storage labels:
 
 Postgres is private runtime storage for verification tasks, task claiming, access credentials, frontend sessions, and creator-granted homeserver session material. It is not storage for Pubky-owned content locks, guarded resources, Lock Service Pointers, or verified proof bundles.
 
-Creator-granted session material is encrypted before storage. The server-side encryption key comes from an env var named by config:
+Sensitive runtime material is encrypted before storage. A root key comes from an env var named by config:
 
 ```toml
 [secrets]
-creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY"
+runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY"
 ```
 
-The named env var must contain a 32-byte key encoded as base64url without padding:
+The named env var must contain a 32-byte key encoded as base64url without padding. Locks derives separate fixed-domain keys for creator-authority material and replayable final deletion credentials; the root key is never used directly as an AEAD key. Rotating it requires an explicit data migration. In the local Compose flow, an explicitly supplied `PUBKY_LOCK_RUNTIME_MASTER_KEY` is validated and persisted only when `.pubky-lock/runtime-master-key` does not yet exist. Once persisted, an override must match those exact bytes or startup fails closed; later starts without the override reuse the persisted key. Changing the key therefore requires an explicit encrypted-data migration or an intentional reset that discards the dependent ciphertext and its old key together.
+
+Migration `0016_content_lock_access_drains` fails closed when an older database contains a resumable `queued`, `running`, or `failed` deletion job, because Task 7 rows do not contain enough information to reconstruct cutoff credential classification safely. Failed jobs are resumable by graceful-deletion replay, so they cannot be treated as terminal for this upgrade. Before upgrading, stop new writes and check:
+
+```sql
+SELECT job_id, state, phase
+FROM content_lock_deletion_jobs
+WHERE state IN ('queued', 'running', 'failed');
+```
+
+If rows are returned, resume/retry them on the pre-0016 release until every deletion reaches `completed` and then retry the upgrade, or explicitly reset the pre-production environment: stop the stack, back up anything needed, recreate the Locks PostgreSQL database/volume, reconcile or republish any public tombstones/content locks, and reacquire creator authority. Do not bypass the guard by deleting only the job rows; that can strand external Pubky state and accepted obligations.
+
+The runtime-master-key cutover intentionally cannot decrypt creator-authority rows written with the retired Compose `creator-authority-encryption-key`. An existing Compose volume containing that file fails startup instead of silently stranding encrypted authority. To upgrade a local stack:
+
+1. Stop the stack and back up any data that must be retained.
+2. Either discard the existing creator-authority rows and reacquire authority after restart, or recreate the local PostgreSQL database/volume.
+3. Remove `.pubky-lock/creator-authority-encryption-key` from the Locks service volume.
+4. Restart. Compose creates `.pubky-lock/runtime-master-key`; keep that file stable with the database.
+
+Removing only the retired key file while retaining its encrypted creator-authority rows is unsupported.
 
 ```bash
 python3 - <<'PY'
@@ -198,6 +217,19 @@ print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('='))
 PY
 ```
 
+Deletion retry and final-access bounds are a closed configuration section:
+
+```toml
+[deletion]
+retry_max_attempts = 10
+retry_initial_backoff_seconds = 1
+retry_max_backoff_seconds = 300
+final_credential_issuance_window_seconds = 900
+final_read_window_seconds = 900
+```
+
+All values must be positive, initial backoff cannot exceed maximum backoff, and both final-access windows must be at most 3600 seconds.
+
 ## Development integration shape
 
 ```toml

docs/plans/2026-08-10-graceful-content-lock-deletion.md

@@ -14,7 +14,7 @@
 
 ## Status and provenance
 
-- Plan status: **accepted product design; Tasks 1, 2, 4, 5, and 6 committed; the omitted Task 3 prerequisite is implemented pending commit; Tasks 7–10 remain**.
+- Plan status: **accepted product design; Tasks 1–7 committed; Tasks 8–11 remain**.
 - Repository inspected: `/home/u/Projects/Synonym/Pubky/locks-public`.
 - Planning base when written: clean `master` at `ba49a77`.
 - There has been no production deployment. New persistence may require a clean pre-production database; no historical backfill is required.
@@ -66,6 +66,11 @@
 32. `force=true` against an active graceful job persists `force_requested`, revokes the current claim token/lease, requeues the same frozen job, and returns `202`; a fresh worker claim escalates asynchronously under exclusive action ownership, skips drains, deletes tombstone then content, and finishes forced.
 33. Graceful job insertion/resume and permanent force-receipt establishment acquire the same canonical per-lock PostgreSQL fence. The durable result is either an active graceful job or a permanent force receipt, never both. Failed graceful replay requeues the same job and frozen manifest. Force against a terminal job atomically replaces that operational row with the permanent receipt before synchronous external deletion.
 34. Any Content Lock fetched from Pubky for deletion must hash to the requested Lock ID and name the authenticated creator before its manifest is frozen or used for resource deletion.
+35. Runtime encryption uses one environment-only 32-byte unpadded-base64url master key selected by `secrets.runtime_master_key_env`. Creator-authority and final-credential encryption keys are derived from it with distinct fixed domain labels. The retired `creator_authority_key_env` key is rejected as unknown configuration; no compatibility alias is retained.
+36. The closed `[deletion]` configuration contract is `retry_max_attempts = 10`, `retry_initial_backoff_seconds = 1`, `retry_max_backoff_seconds = 300`, `final_credential_issuance_window_seconds = 900`, and `final_read_window_seconds = 900` by default. All values are positive; initial backoff cannot exceed maximum backoff; both credential windows are bounded to at most 3600 seconds. Retry jitter remains an implementation policy rather than a configurable field.
+37. Deletion admission immutably records whether each paid snapshot Bundle had any active credential at cutoff and enrolls every such ordinary credential with its original expiry. Enrolled ordinary credentials remain reusable against the frozen manifest until that expiry; they do not acquire one-shot resource-read rows. When the claimed job first enters `issue_final_credentials`, it persists `final_issuance_started_at`, `final_credential_issuance_deadline = final_issuance_started_at + final_credential_issuance_window`, and `final_read_deadline = final_credential_issuance_deadline + final_read_window` once; replay and later config changes never extend them. A paid snapshot resolved completed without an active ordinary credential at cutoff becomes durably final-credential eligible and receives exactly one encrypted replayable final credential expiring at `final_read_deadline`. Every final credential receives one claimable row per frozen manifest path. Final-read claims precede Pubky fetch, are released on pre-response failure, expire for crash recovery, and are consumed only after the complete HTTP response is constructed; consumption is permanent. Phase advancement waits until every enrolled ordinary credential is expired and every final resource is consumed or its credential/read window is expired.
+38. Ordinary credential insertion and deletion admission acquire the same canonical per-lock fence. Deletion-first rejects the insert; insertion-first is attached and classified at cutoff. Database lock order is canonical per-lock fence, deletion job row, snapshot/credential row, then resource-read row. No transaction spans Pubky I/O. Final read claims use fixed 30-second leases clamped to credential expiry; stale claim tokens cannot consume or release a reclaimed row.
+39. This pre-production migration intentionally has no creator-authority ciphertext compatibility path. Moving the same bytes to `runtime_master_key_env` changes the derived creator-authority key; existing local encrypted authority rows must be discarded and reacquired or the local database recreated.
 
 ### Source-derived constraints
 
@@ -458,7 +463,7 @@ cargo test --workspace --no-run
 
 **RED:** Cover exact encrypted replay, wrong-key/corrupt/version rejection, no secret Debug/log output, issuance/read deadlines, no deadline extension, existing/final access through the frozen manifest while the public path is a tombstone, denial outside the persisted drain, one concurrent success per path, claim release before response construction, consumption after construction, and automatic revocation when complete/expired.
 
-**GREEN:** Use versioned AEAD and domain-separated key derivation; retain lookup hashes. Resolve draining reads from the frozen manifest rather than parsing the tombstone. Do not store plaintext bearer.
+**GREEN:** Snapshot and enroll active credentials atomically at deletion admission, and initialize final-window timestamps once when entering final issuance under the deletion lease. Fence ordinary insertion against deletion admission. Use versioned AEAD and domain-separated key derivation; retain lookup hashes. Enroll cutoff-active credentials at their original expiry and create one final credential only for an eligible completed snapshot without one. Resolve draining reads from the frozen manifest rather than parsing the tombstone. Claim each credential/path before fetch, release on pre-response failure, consume only after the server constructs the complete response, and allow only expired claims to be reclaimed. Do not store plaintext bearer.
 
 **Suggested commit:** `feat(access): drain final deletion credentials`
 
@@ -560,9 +565,7 @@ Cross-service acceptance must additionally prove:
 
 ## Remaining implementation-contract gates
 
-These do not reopen accepted product semantics, but code must not start for the affected slice until both plans are patched identically:
-
-1. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima.
+None. The exact Locks-only deletion configuration and runtime-master-key contracts are fixed above. Paykit Server has no corresponding credential or deletion-worker configuration.
 
 ## Out of scope
 

examples/js-sdk/README.md

@@ -96,10 +96,10 @@ For a containerized local stack from the repository root:
 docker compose up --build
 ```
 
-On first startup, the Lock Server entrypoint generates a random creator-authority
-encryption key and persists it in the private `lock-home` volume. Later starts reuse
-that key. To supply your own 32-byte base64url key instead, export
-`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before running Compose.
+On first startup, the Lock Server entrypoint generates a random runtime master key
+and persists it in the private `lock-home` volume. Later starts reuse that key. To
+supply your own 32-byte base64url key instead, export
+`PUBKY_LOCK_RUNTIME_MASTER_KEY` before running Compose.
 
 The compose stack starts:
 
@@ -112,8 +112,8 @@ The compose stack starts:
 The Pubky testnet image is built from the public `pubky/pubky-homeserver` repository at
 the revision pinned in `docker-compose.yml`; no sibling checkout is required.
 
-Compose keeps the Lock Server identity, config, and generated creator-authority
-encryption key in the `lock-home` Docker volume and Postgres data in `postgres-data`.
+Compose keeps the Lock Server identity, config, and generated runtime master key in
+the `lock-home` Docker volume and Postgres data in `postgres-data`.
 To reset everything:
 
 ```bash